Merge pull request #39 from NASA-IMPACT/feat/kmz-inside-zip

campaigns-schema.yaml

@@ -30,6 +30,10 @@ deployments:
               required: false
               type: boolean
               description: Whether a .txt file should be processed as a .ict file.
+            filter_kmz:
+              required: false
+              type: boolean
+              description: Some zip files contain a kmz file that is the only file needed. In that case, it will only extract the kmz and uncompress it, resulting in a kml file.
             coords_divisor:
               required: false
               type: integer

campaigns/PODEX/deployments.yaml

@@ -0,0 +1,18 @@
+---
+deployments:
+- name: PODEX-D1_2013
+  platforms:
+  - name: ER-2
+    files:
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA/2013-01-14/IWG1.xml
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-14/IWG1.15Jan2013-0002
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-16/IWG1.16Jan2013-2108
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-18/IWG1.18Jan2013-2203
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-20/IWG1.20Jan2013-2149
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-22/IWG1.23Jan2013-0030
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-28/IWG1.28Jan2013-2322
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-01-31/IWG1.31Jan2013-2257
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-02-01/IWG1.01Feb2013-2317
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-02-03/IWG1.03Feb2013-1452
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-02-03/IWG1.03Feb2013-2312
+    - https://asp-archive.arc.nasa.gov/PODEX/N809NA//2013-02-06/IWG1.07Feb2013-0001

campaigns/Polar Winds/deployments.yaml

@@ -0,0 +1,37 @@
+---
+deployments:
+- name: Polar Winds-D1_2014
+  platforms:
+  - name: B-200
+    filter_kmz: true
+    files:
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141029-115154_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141030-132302_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141031-110108_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141103-135618_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141104-135816_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141105-141140_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141106-140534_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141107-141738_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141108-155441_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141110-142837_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141111-140700_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141112-110621_txt.zip
+    - https://asdc.larc.nasa.gov/data/PolarWindsI/DAWN_KingAirUC-12B_1/PolarWindsI_DAWN_KingAirUC-12B_1.20141113-140215_txt.zip
+- name: Polar Winds-D2_2015
+  platforms:
+  - name: DC-8
+    files:
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2014-10-25/IWG1.xml
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-07/IWG1.07May2015-2226
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-09/IWG1.09May2015-1401
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-11/IWG1.11May2015-1728
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-13/IWG1.13May2015-1509
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-15/IWG1.15May2015-2032
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-16/IWG1.16May2015-2115
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-17/IWG1.17May2015-2117
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-19/IWG1.19May2015-1708
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-21/IWG1.21May2015-2202
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-23/IWG1.23May2015-2001
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-24/IWG1.24May2015-1835
+    - https://asp-archive.arc.nasa.gov/N817NA/FY2015/2015-05-25/IWG1.25May2015-1729

task/get-files-list.js

@@ -0,0 +1,16 @@
+const { listLinks } = require('./src/list-links');
+
+const url = process.argv[2];
+const outputFile = process.argv[3];
+
+// this is a utility command to list the files URLs in some NASA websites,
+// like https://asp-archive.arc.nasa.gov/PODEX/N809NA/
+// It will create a txt with the URLs of the files listed in the
+// subpages of that webpage.
+// It also filters out all the files that don't contain the string "IWG1."
+// and the .xml files
+
+// Usage:
+// yarn list-files https://asp-archive.arc.nasa.gov/PODEX/N809NA/ podex.txt
+
+listLinks(url, outputFile);

task/package.json

@@ -6,14 +6,18 @@
   "scripts": {
     "download": "node downloadCampaign.js",
     "lint": "eslint --ext .js",
+    "list-files": "node get-files-list.js",
     "process": "node processAll.js",
     "test": "jest",
     "xls2csv": "node xls2csv.js"
   },
   "dependencies": {
     "@mapbox/geojson-merge": "^1.1.1",
     "@tmcw/togeojson": "^5.8.1",
-    "@turf/distance": "^6.5.0",
+    "@turf/distance": "^7.1.0",
+    "@turf/helpers": "^7.1.0",
+    "axios": "^1.7.7",
+    "cheerio": "^1.0.0",
     "csv2geojson": "^5.1.2",
     "d3-dsv": "1.0.1",
     "download": "8.0.0",

task/src/convert-kml.js

@@ -2,15 +2,23 @@ const toGeojson = require('@tmcw/togeojson');
 const fs = require('fs');
 const { DOMParser } = require('xmldom');
 const unzipper = require('unzipper');
+const { lineString } = require('@turf/helpers');
 const simplify = require('simplify-geojson');
 
 const kml2geojson = (filePath, properties) => {
   const file = fs.readFileSync(filePath);
   const kml = new DOMParser().parseFromString(file.toString());
   const geojson = toGeojson.kml(kml);
-  const features = geojson.features
-    .filter((f) => f.geometry.type === 'LineString')
-    .map((f) => ({ ...f, properties }));
+  let features;
+  // some kml files have all features as points, in that case,
+  // we need to convert it to a single LineString
+  if (geojson.features.every((f) => f.geometry.type === 'Point')) {
+    features = [lineString(geojson.features.map((f) => f.geometry.coordinates), properties)];
+  } else {
+    features = geojson.features
+      .filter((f) => f.geometry.type === 'LineString')
+      .map((f) => ({ ...f, properties }));
+  }
   return simplify({ ...geojson, features }, 0.001);
 };
 
@@ -37,7 +45,32 @@ const kmz2kml = (filePath) => new Promise((resolve, reject) => {
     .on('error', reject);
 });
 
+const extractKmlContent = (filePath) => new Promise((resolve, reject) => {
+  fs.createReadStream(filePath)
+    .pipe(unzipper.ParseOne(/\.kmz$/))
+    .pipe(unzipper.Parse())
+    .on('entry', (entry) => {
+      if (entry.path.indexOf('.kml') === -1) {
+        entry.autodrain();
+        return;
+      }
+      let data = '';
+
+      entry.on('error', reject);
+
+      entry.on('data', (chunk) => {
+        data += chunk;
+      });
+
+      entry.on('end', () => {
+        resolve(data);
+      });
+    })
+    .on('error', reject);
+});
+
 module.exports = {
   kml2geojson,
+  extractKmlContent,
   kmz2kml,
 };

task/src/download.js

@@ -3,11 +3,11 @@ const path = require('path');
 const unzipper = require('unzipper');
 const download = require('download');
 const { getPlatformConfig, readCampaignYaml, urlHasFileExtension } = require('./utils');
-const { kmz2kml } = require('./convert-kml');
+const { kmz2kml, extractKmlContent } = require('./convert-kml');
 
 const replaceSlash = (str) => str.replaceAll('/', '-');
 
-const downloadFile = async (url, dir) => {
+const downloadFile = async (url, dir, platformConfig) => {
   await download(
     url,
     dir,
@@ -16,8 +16,14 @@ const downloadFile = async (url, dir) => {
   // if the file is a zip, decompress it
   if (url.endsWith('.zip')) {
     const filePath = path.join(dir, path.basename(url));
-    const zip = await unzipper.Open.file(filePath);
-    await zip.extract({ path: dir });
+    // some zip files, have only a kmz file that needs to be extracted as kml
+    if (platformConfig.filter_kmz) {
+      const kml = await extractKmlContent(filePath);
+      fs.writeFileSync(filePath.replace('.zip', '.kml'), kml);
+    } else {
+      const zip = await unzipper.Open.file(filePath);
+      await zip.extract({ path: dir });
+    }
     fs.unlinkSync(filePath);
   }
   // The GRIP campaign has files without an extension and others with .dat that should be txt
@@ -62,7 +68,9 @@ const downloadPlatform = async (campaignPath, deployment, platform, files) => {
   const platformPath = path.join(campaignPath, deployment, platform);
   const platformConfig = getPlatformConfig(platformPath);
   await createDir(platformPath);
-  await Promise.all(files.map((file) => downloadFile(file, platformPath)));
+  await Promise.all(
+    files.map((file) => downloadFile(file, platformPath, platformConfig))
+  );
   // some .ict files have a .ER2 or .WB57 extension,
   // so we need to rename it after the download
   if (platformConfig.rename_as_ict) {

task/src/list-links.js

@@ -0,0 +1,38 @@
+const fs = require('fs');
+const axios = require('axios');
+const cheerio = require('cheerio');
+
+async function getLinks(url) {
+  /**
+   * Retrieves all the links from a given webpage.
+   * @param {string} url - The URL of the webpage to be accessed.
+   * @returns {string[]} - A list of all the links found on the webpage.
+   */
+  const response = await axios.get(url);
+  const $ = cheerio.load(response.data);
+  const links = $('a').map((_, link) => `${url}/${$(link).attr('href')}`).get();
+  return links;
+}
+
+function saveToCSV(links, outputFileName) {
+  /**
+   * Saves a list of links to a CSV file.
+   * @param {string[]} links - A list of links to be saved.
+   * @param {string} outputFileName - The name of the output CSV file.
+   */
+  fs.writeFileSync(outputFileName, links.join('\n'));
+}
+
+async function listLinks(url, outputFileName) {
+  const initialLinks = await getLinks(url);
+  const allLinks = await Promise.all(initialLinks.map((i) => getLinks(i)));
+  saveToCSV(
+    allLinks
+      .flat().filter((i) => i.includes('IWG1.') && !i.includes('.xml')),
+    outputFileName
+  );
+}
+
+module.exports = {
+  listLinks,
+};

task/src/utils.js

@@ -65,8 +65,7 @@ const divideCoordinates = (features, coordsDivisor) => features.map((i) => {
 });
 
 const urlHasFileExtension = (url) => {
-  // eslint-disable-next-line no-unused-vars
-  const [name, ext] = path.basename(url).split('.');
+  const ext = path.basename(url).split('.').slice(-1)[0];
   return ext && [3, 4].includes(ext.length);
 };
 

task/tests/utils.test.js

@@ -33,6 +33,9 @@ describe('getPlatformConfig', () => {
 
 describe('urlHasFileExtension', () => {
   it('return true if an URL ends with any 3 or 4 letters file extension', () => {
+    expect(urlHasFileExtension(
+      'https://nasa.gov/PolarWindsI_DAWN_KingAirUC-12B_1.20141113-140215_txt.zip'
+    )).toBeTruthy();
     expect(urlHasFileExtension('https://a.com/b/c/d.ext')).toBeTruthy();
     expect(urlHasFileExtension('https://a.com/b/c/d.WB57')).toBeTruthy();
     expect(urlHasFileExtension('https://a.com/b/c/d.WB57')).toBeTruthy();

task/tests/validate-yaml.test.js

@@ -20,6 +20,7 @@ const structure = array(
         tsv_format: optional(boolean()),
         rename_as_ict: optional(boolean()),
         process_as_ict: optional(boolean()),
+        filter_kmz: optional(boolean()),
         coords_divisor: optional(number()),
         data_start_line_fix: optional(number()),
         header_content: optional(string()),