Convert GPX/KML to geojson

.eslintrc

@@ -0,0 +1,17 @@
+{
+  "rules": {
+    "indent": [2, 2],
+    "quotes": [2, "single"],
+    "no-console": [0],
+    "semi": [2, "always"]
+  },
+  "env": {
+      "node": true
+  },
+  "globals": {
+      "process": true,
+      "module": true,
+      "require": true
+  },
+  "extends": "eslint:recommended"
+}

.gitignore

@@ -1,2 +1,3 @@
+node_modules
 preprocessors/test*
 test/fixtures/*.aux.xml

package.json

@@ -4,7 +4,7 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "pretest": "jshint index.js bin parts preprocessors test && jscs index.js bin parts preprocessors test",
+    "pretest": "jshint index.js bin parts preprocessors test && eslint index.js bin parts preprocessors test && jscs index.js bin parts preprocessors test",
     "test": "tape test/*.test.js",
     "coverage": "istanbul cover tape test/*.test.js && coveralls < ./coverage/lcov.info"
   },
@@ -23,18 +23,21 @@
   "homepage": "https://github.com/mapbox/preprocessorcerer",
   "devDependencies": {
     "concat-stream": "^1.4.8",
+    "eslint": "^1.6.0",
     "jscs": "^1.10.0",
     "jshint": "^2.6.0",
     "tape": "^3.4.0",
     "tilelive": "^5.7.0",
     "istanbul": "~0.3.17",
     "coveralls": "~2.11.2",
-    "checksum": "^0.1.1"
+    "checksum": "^0.1.1",
+    "rimraf": "^2.5.2"
   },
   "dependencies": {
     "bytetiff": "~0.3.0",
     "gdal": "~0.8.0",
     "mapbox-file-sniff": "~0.4.0",
+    "mapnik-omnivore": "~7.3.0",
     "mapnik": "~3.5.0",
     "mkdirp": "^0.5.0",
     "mbtiles": "^0.8.0",
@@ -55,6 +58,7 @@
     "preset": "airbnb",
     "requireCamelCaseOrUpperCaseIdentifiers": null,
     "disallowMultipleVarDecl": true,
-    "requireMultipleVarDecl": null
+    "requireMultipleVarDecl": null,
+    "validateLineBreaks": null
   }
 }

parts/mbtiles-byTiles.js

@@ -18,11 +18,15 @@ module.exports = function splitBySize(filepath, info, callback) {
         }
         else if (err) return callback(err);
 
-        var maxTilesPerJob = 100000;
-        var result = tilesResult.count + gridsResult.count;
-        if (!result) return callback(new Error('no tiles or grids'));
+        src.close(function(err) {
+          if (err) return callback(err);
+          src = null;
+          var maxTilesPerJob = 100000;
+          var result = tilesResult.count + gridsResult.count;
+          if (!result) return callback(new Error('no tiles or grids'));
 
-        callback(null, Math.min(50, Math.ceil(result / maxTilesPerJob)));
+          callback(null, Math.min(50, Math.ceil(result / maxTilesPerJob)));
+        });
       });
     });
   });

preprocessors/geojson-bom.preprocessor.js

@@ -32,7 +32,10 @@ module.exports.criteria = function(infile, info, callback) {
     fs.read(fd, buf, 0, 3, 0, function(err) {
       if (err) return callback(err);
       var strip = buf[0] === 0xef && buf[1] === 0xbb && buf[2] == 0xbf;
-      callback(null, strip);
+      fs.close(fd, function(err) {
+        if (err) return callback(err);
+        callback(null, strip);
+      });
     });
   });
 };

preprocessors/index.js

@@ -10,7 +10,9 @@ var preprocessors = [
   'tif-reproject.preprocessor',
   'shp-index.preprocessor',
   'geojson-bom.preprocessor',
-  'spatial-index.preprocessor'
+  'spatial-index.preprocessor',
+  'togeojson-kml.preprocessor',
+  'togeojson-gpx.preprocessor'
 ];
 
 // Loads each *.preprocessor.js file and builds an array of them

preprocessors/shp-index.preprocessor.js

@@ -2,7 +2,7 @@ var path = require('path');
 var fs = require('fs');
 var spawn = require('child_process').spawn;
 var mapnik = require('mapnik');
-var shapeindex = path.resolve(mapnik.module_path, 'shapeindex');
+var shapeindex = path.resolve(mapnik.module_path, 'shapeindex' + (process.platform === 'win32' ? '.exe' : ''));
 if (!fs.existsSync(shapeindex)) {
   throw new Error('shapeindex does not exist at ' + shapeindex);
 }

preprocessors/spatial-index.preprocessor.js

@@ -34,6 +34,7 @@ module.exports = function(infile, outdir, callback) {
         .on('exit', function() {
           // If error printed to --validate-features log
           if (data.indexOf('Error') != -1) {
+            console.log(data);
             callback('Invalid geojson feature');
           }
           else callback();

preprocessors/tif-reproject.preprocessor.js

@@ -27,6 +27,9 @@ module.exports.criteria = function(filepath, info, callback) {
   }
   catch (err) { return callback(err); }
 
+  ds.close();
+  ds = null;
+
   if (projection === sm) callback(null, false);
   else callback(null, true);
 };

preprocessors/tif-toBytes.preprocessor.js

@@ -21,6 +21,10 @@ module.exports.criteria = function(filepath, info, callback) {
   try { band = ds.bands.get(1); }
   catch (err) { return callback(err); }
 
-  if (band.dataType === gdal.GDT_UInt16) return callback(null, true);
-  callback(null, false);
+  var is16 = band.dataType === gdal.GDT_UInt16;
+
+  ds.close();
+  ds = null;
+
+  callback(null, is16);
 };

preprocessors/togeojson-gpx.preprocessor.js

@@ -0,0 +1,156 @@
+var gdal = require('gdal');
+var fs = require('fs');
+var mkdirp = require('mkdirp');
+var queue = require('queue-async');
+var spawn = require('child_process').spawn;
+var path = require('path');
+var digest = require('mapnik-omnivore').digest;
+var mapnik = require('mapnik');
+var mapnik_index = path.resolve(mapnik.module_path, 'mapnik-index' + (process.platform === 'win32' ? '.exe' : ''));
+if (!fs.existsSync(mapnik_index)) {
+  throw new Error('mapnik-index does not exist at ' + mapnik_index);
+}
+
+//disable in production
+//gdal.verbose();
+
+module.exports = function(infile, outdirectory, callback) {
+  mkdirp(outdirectory, function(err) {
+    if (err) return callback(err);
+
+    var geojson_files = [];
+    var ds_gpx;
+    var full_feature_cnt = 0;
+    var wgs84 = gdal.SpatialReference.fromEPSG(4326);
+
+    try {
+      ds_gpx = gdal.open(infile);
+    }
+    catch (err) {
+      return callback(new Error(err));
+    }
+
+    ds_gpx.layers.forEach(function(lyr_gpx) {
+      //drop point layers as they can get really huge
+      if (lyr_gpx.name === 'track_points' || lyr_gpx.name === 'route_points') {
+        return;
+      }
+
+      var feat_cnt = lyr_gpx.features.count(true);
+      if (feat_cnt === 0) {
+        return;
+      }
+
+      var geojson;
+      var lyr_name;
+      var out_ds;
+      var out_name;
+
+      try {
+        lyr_name = lyr_gpx.name;
+        out_name = path.join(outdirectory, lyr_name + '.geojson');
+        out_ds = gdal.open(out_name, 'w', 'GeoJSON');
+        geojson = out_ds.layers.create(lyr_name, wgs84, lyr_gpx.geomType);
+      }
+      catch (err) {
+        return callback(new Error(err));
+      }
+
+      lyr_gpx.features.forEach(function(gpx_feat) {
+        //skip null or empty geometries
+        var geom = gpx_feat.getGeometry();
+        if (!geom) {
+          return;
+        } else {
+          if (geom.isEmpty()) {
+            return;
+          }
+
+          if (!geom.isValid()) {
+            return;
+          }
+        }
+
+        geojson.features.add(gpx_feat);
+        full_feature_cnt++;
+      });
+
+      geojson.flush();
+      out_ds.flush();
+      out_ds.close();
+
+      //release objects to be able to index
+      geojson = null;
+      out_ds = null;
+
+      geojson_files.push(out_name);
+    });
+
+    ds_gpx.close();
+    if (full_feature_cnt === 0) {
+      return callback(new Error('GPX does not contain any valid features.'));
+    }
+
+    // Create metadata file for original gpx source
+    var metadatafile = path.join(outdirectory, '/metadata.json');
+    digest(infile, function(err, metadata) {
+      if (err) return callback(err);
+      fs.writeFile(metadatafile, JSON.stringify(metadata), function(err) {
+        if (err) return callback(err);
+        return createIndices(callback);
+      });
+    });
+
+    function createIndices(callback) {
+      // create mapnik index for each geojson layer
+      var q = queue();
+      geojson_files.forEach(function(gj) {
+        q.defer(createIndex, gj);
+      });
+
+      q.awaitAll(function(err) {
+        if (err) return callback(err);
+        return callback();
+      });
+    }
+
+    function createIndex(layerfile, callback) {
+      // Finally, create an .index file in the output dir (if layer is greater than index_worthy_size).
+      // mapnik-index will automatically add ".index" to the end of the original filename
+      fs.stat(layerfile, function(err, stats) {
+        if (err) return callback(err);
+
+        // check size is warrants creating an index
+        if (stats.size >= module.exports.index_worthy_size) {
+          var data = '';
+          var p = spawn(mapnik_index, [layerfile, '--validate-features'])
+            .once('error', callback)
+            .on('exit', function() {
+              // If error printed to --validate-features log
+              if (data.indexOf('Error') != -1) {
+                return callback(data);
+              }
+              else return callback();
+            });
+
+          p.stderr.on('data', function(d) {
+            d.toString();
+            data += d;
+          });
+        } else {
+          return callback();
+        }
+      });
+    }
+  });
+};
+
+module.exports.description = 'Convert GPX to GeoJSON';
+module.exports.index_worthy_size = 10 * 1024 * 1024; // 10 MB
+
+module.exports.criteria = function(filepath, info, callback) {
+
+  if (info.filetype !== 'gpx') return callback(null, false);
+
+  callback(null, true);
+};