Add AfriSAR campaign and handle kml/kmz files

campaigns/AfriSAR/deployments.yaml

@@ -0,0 +1,25 @@
+---
+deployments:
+- name: AfriSAR-D1_2016
+  platforms:
+  - name: B-200
+    files:
+    - https://lvis.gsfc.nasa.gov/images/kmz/traj57438_LVIS.kmz
+    - https://lvis.gsfc.nasa.gov/images/kmz/traj57440_LVIS.kmz
+    - https://lvis.gsfc.nasa.gov/images/kmz/traj57441_LVIS.kmz
+    - https://lvis.gsfc.nasa.gov/images/kmz/traj57443_LVIS.kmz
+    - https://lvis.gsfc.nasa.gov/images/kmz/traj57449_LVIS.kmz
+    - https://lvis.gsfc.nasa.gov/images/kmz/traj57450_LVIS.kmz
+    - https://lvis.gsfc.nasa.gov/images/kmz/traj57451_LVIS.kmz
+    - https://lvis.gsfc.nasa.gov/images/kmz/traj57454_LVIS.kmz
+    - https://lvis.gsfc.nasa.gov/images/kmz/traj57455_LVIS.kmz
+  - name: G-III
+    files:
+    - https://uavsar.jpl.nasa.gov/flightplans/flt16008/PLAN_16008.kml
+    - https://uavsar.jpl.nasa.gov/flightplans/flt16009/PLAN_16009.kml
+    - https://uavsar.jpl.nasa.gov/flightplans/flt16010/PLAN_16010.kml
+    - https://uavsar.jpl.nasa.gov/flightplans/flt16011/PLAN_16011.kml
+    - https://uavsar.jpl.nasa.gov/flightplans/flt16012/PLAN_16012.kml
+    - https://uavsar.jpl.nasa.gov/flightplans/flt16013/PLAN_16013.kml
+    - https://uavsar.jpl.nasa.gov/flightplans/flt16014/PLAN_16014.kml
+    - https://uavsar.jpl.nasa.gov/flightplans/flt16015/PLAN_16015.kml

task/package.json

@@ -12,8 +12,8 @@
   },
   "dependencies": {
     "@mapbox/geojson-merge": "^1.1.1",
+    "@tmcw/togeojson": "^5.8.1",
     "@turf/distance": "^6.5.0",
-    "adm-zip": "^0.5.12",
     "csv2geojson": "^5.1.2",
     "d3-dsv": "1.0.1",
     "download": "8.0.0",
@@ -24,8 +24,10 @@
     "simplify-geojson": "^1.0.5",
     "slugify": "^1.6.6",
     "superstruct": "^1.0.4",
+    "unzipper": "^0.12.3",
     "xlsx": "^0.18.5",
     "xml2js": "^0.6.2",
+    "xmldom": "^0.6.0",
     "yaml": "2.3.4"
   },
   "devDependencies": {

task/processAll.js

@@ -4,7 +4,7 @@ const fs = require('fs');
 const { findDirectories, findFiles } = require('./src/find');
 const { exportHeaders } = require('./src/headers');
 const { makeCSV } = require('./makeCSV');
-const { convertAll } = require('./convertAll');
+const { makePlatformGeoJSON } = require('./processPlatform');
 const { convert } = require('./convert');
 const { mergeGeoJSONCollection } = require('./src/process');
 
@@ -14,7 +14,7 @@ const platforms = findDirectories(campaignPath, 2);
 platforms.forEach((p) => {
   exportHeaders(p);
   makeCSV(p);
-  convertAll(p);
+  makePlatformGeoJSON(p);
 });
 
 // convert the static CSV file to GeoJSON

task/convertAll.js → task/processPlatform.js

@@ -8,17 +8,27 @@ const {
   mergeGeoJSONCollection,
 } = require('./src/process');
 const { getPlatformConfig } = require('./src/utils');
+const { kml2geojson } = require('./src/convert-kml');
 
-const convertAll = (dir) => {
+const makePlatformGeoJSON = (dir) => {
   const properties = getPropertiesFromPath(dir);
   const platformConfig = getPlatformConfig(dir);
 
   const files = fs.readdirSync(dir);
-  const collection = files
+  let collection;
+  // convert CSV files to GeoJSON
+  collection = files
     .filter((f) => f.endsWith('.csv') && !f.endsWith('headers.csv'))
     .map((f) => path.join(dir, f))
     .map((f) => convertToGeoJSON(f, properties, platformConfig.coords_divisor));
 
+  // if the platform has
+  if (!collection.length && files.every((f) => f.endsWith('.kml'))) {
+    collection = files
+      .map((f) => path.join(dir, f))
+      .map((f) => kml2geojson(f, properties));
+  }
+
   const resultFile = path.join(
     dir,
     slugify(`${properties.deployment}-${properties.platform_name}.geojson`)
@@ -28,5 +38,5 @@ const convertAll = (dir) => {
 };
 
 module.exports = {
-  convertAll,
+  makePlatformGeoJSON,
 };

task/src/convert-kml.js

@@ -0,0 +1,43 @@
+const toGeojson = require('@tmcw/togeojson');
+const fs = require('fs');
+const { DOMParser } = require('xmldom');
+const unzipper = require('unzipper');
+const simplify = require('simplify-geojson');
+
+const kml2geojson = (filePath, properties) => {
+  const file = fs.readFileSync(filePath);
+  const kml = new DOMParser().parseFromString(file.toString());
+  const geojson = toGeojson.kml(kml);
+  const features = geojson.features
+    .filter((f) => f.geometry.type === 'LineString')
+    .map((f) => ({ ...f, properties }));
+  return simplify({ ...geojson, features }, 0.001);
+};
+
+const kmz2kml = (filePath) => new Promise((resolve, reject) => {
+  fs.createReadStream(filePath)
+    .pipe(unzipper.Parse())
+    .on('entry', (entry) => {
+      if (entry.path.indexOf('.kml') === -1) {
+        entry.autodrain();
+        return;
+      }
+      let data = '';
+
+      entry.on('error', reject);
+
+      entry.on('data', (chunk) => {
+        data += chunk;
+      });
+
+      entry.on('end', () => {
+        resolve(data);
+      });
+    })
+    .on('error', reject);
+});
+
+module.exports = {
+  kml2geojson,
+  kmz2kml,
+};

task/src/download.js

@@ -1,8 +1,9 @@
 const fs = require('fs');
 const path = require('path');
-const AdmZip = require('adm-zip');
+const unzipper = require('unzipper');
 const download = require('download');
 const { getPlatformConfig, readCampaignYaml, urlHasFileExtension } = require('./utils');
+const { kmz2kml } = require('./convert-kml');
 
 const replaceSlash = (str) => str.replaceAll('/', '-');
 
@@ -15,15 +16,21 @@ const downloadFile = async (url, dir) => {
   // if the file is a zip, decompress it
   if (url.endsWith('.zip')) {
     const filePath = path.join(dir, path.basename(url));
-    const zip = new AdmZip(filePath);
-    zip.extractAllTo(dir);
+    const zip = await unzipper.Open.file(filePath);
+    await zip.extract({ path: dir });
     fs.unlinkSync(filePath);
   }
   // The GRIP campaign has files without an extension and others with .dat that should be txt
   if (!urlHasFileExtension(url) || url.endsWith('.dat')) {
     const filePath = path.join(dir, path.basename(url));
     fs.renameSync(filePath, `${filePath}.txt`);
   }
+  if (url.endsWith('.kmz')) {
+    const filePath = path.join(dir, path.basename(url));
+    const kml = await kmz2kml(filePath);
+    fs.writeFileSync(filePath.replace('.kmz', '.kml'), kml);
+    fs.unlinkSync(filePath);
+  }
 };
 
 const downloadCampaign = (campaignPath) => {

task/yarn.lock

@@ -674,6 +674,11 @@
   dependencies:
     "@sinonjs/commons" "^3.0.0"
 
+"@tmcw/togeojson@^5.8.1":
+  version "5.8.1"
+  resolved "https://registry.yarnpkg.com/@tmcw/togeojson/-/togeojson-5.8.1.tgz#6cbcc9b1484ed28e71bdd4b5f96ceae540a4533f"
+  integrity sha512-2YNrbis3l5kS0XrYwiHEZcGwiRp0MJ5CvwGwtMWp2z2tsVlskeec2qgvKHnF0RCwI5GnjrrBOoKsWfndEnd3LA==
+
 "@turf/distance@^6.5.0":
   version "6.5.0"
   resolved "https://registry.yarnpkg.com/@turf/distance/-/distance-6.5.0.tgz#21f04d5f86e864d54e2abde16f35c15b4f36149a"
@@ -820,11 +825,6 @@ adler-32@~1.3.0:
   resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.3.1.tgz#1dbf0b36dda0012189a32b3679061932df1821e2"
   integrity sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==
 
-adm-zip@^0.5.12:
-  version "0.5.12"
-  resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.12.tgz#87786328e91d54b37358d8a50f954c4cd73ba60b"
-  integrity sha512-6TVU49mK6KZb4qG6xWaaM4C7sA/sgUMLy/JYMOzkcp3BvVLpW0fXDFQiIzAuxFCt/2+xD7fNIiPFAoLZPhVNLQ==
-
 ajv@^6.12.4:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
@@ -1058,6 +1058,11 @@ bl@^1.0.0:
     readable-stream "^2.3.5"
     safe-buffer "^5.1.1"
 
+bluebird@~3.7.2:
+  version "3.7.2"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
+  integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
+
 brace-expansion@^1.1.7:
   version "1.1.11"
   resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@@ -1568,6 +1573,13 @@ [email protected]:
     p-event "^2.1.0"
     pify "^4.0.1"
 
+duplexer2@~0.1.4:
+  version "0.1.4"
+  resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1"
+  integrity sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==
+  dependencies:
+    readable-stream "^2.0.2"
+
 duplexer3@^0.1.4:
   version "0.1.5"
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.5.tgz#0b5e4d7bad5de8901ea4440624c8e1d20099217e"
@@ -2052,6 +2064,15 @@ fs-constants@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
   integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==
 
+fs-extra@^11.2.0:
+  version "11.2.0"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.2.0.tgz#e70e17dfad64232287d01929399e0ea7c86b0e5b"
+  integrity sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==
+  dependencies:
+    graceful-fs "^4.2.0"
+    jsonfile "^6.0.1"
+    universalify "^2.0.0"
+
 fs.realpath@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
@@ -2237,7 +2258,7 @@ got@^8.3.1:
     url-parse-lax "^3.0.0"
     url-to-options "^1.0.1"
 
-graceful-fs@^4.1.10, graceful-fs@^4.2.9:
+graceful-fs@^4.1.10, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.2, graceful-fs@^4.2.9:
   version "4.2.11"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3"
   integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==
@@ -3081,6 +3102,15 @@ json5@^2.2.3:
   resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
   integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==
 
+jsonfile@^6.0.1:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"
+  integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==
+  dependencies:
+    universalify "^2.0.0"
+  optionalDependencies:
+    graceful-fs "^4.1.6"
+
 jsonparse@^1.2.0:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280"
@@ -3728,7 +3758,7 @@ read-pkg@^5.2.0:
     parse-json "^5.0.0"
     type-fest "^0.6.0"
 
-readable-stream@^2.0.0, readable-stream@^2.3.0, readable-stream@^2.3.5:
+readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stream@^2.3.0, readable-stream@^2.3.5:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b"
   integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==
@@ -4425,6 +4455,22 @@ undici-types@~5.26.4:
   resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
   integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
 
+universalify@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.1.tgz#168efc2180964e6386d061e094df61afe239b18d"
+  integrity sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==
+
+unzipper@^0.12.3:
+  version "0.12.3"
+  resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.12.3.tgz#31958f5eed7368ed8f57deae547e5a673e984f87"
+  integrity sha512-PZ8hTS+AqcGxsaQntl3IRBw65QrBI6lxzqDEL7IAo/XCEqRTKGfOX56Vea5TH9SZczRVxuzk1re04z/YjuYCJA==
+  dependencies:
+    bluebird "~3.7.2"
+    duplexer2 "~0.1.4"
+    fs-extra "^11.2.0"
+    graceful-fs "^4.2.2"
+    node-int64 "^0.4.0"
+
 update-browserslist-db@^1.0.13:
   version "1.0.13"
   resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4"
@@ -4568,6 +4614,11 @@ xmlbuilder@~11.0.0:
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
   integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
 
+xmldom@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/xmldom/-/xmldom-0.6.0.tgz#43a96ecb8beece991cef382c08397d82d4d0c46f"
+  integrity sha512-iAcin401y58LckRZ0TkI4k0VSM1Qg0KGSc3i8rU+xrxe19A/BN1zHyVSJY7uoutVlaTSzYyk/v5AmkewAP7jtg==
+
 xtend@^4.0.0:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"