diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..e395732 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,17 @@ +{ + "rules": { + "indent": [2, 2], + "quotes": [2, "single"], + "no-console": [0], + "semi": [2, "always"] + }, + "env": { + "node": true + }, + "globals": { + "process": true, + "module": true, + "require": true + }, + "extends": "eslint:recommended" +} diff --git a/.gitignore b/.gitignore index 54b024b..9ed8cc7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +node_modules preprocessors/test* test/fixtures/*.aux.xml diff --git a/package.json b/package.json index b285441..de37926 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "", "main": "index.js", "scripts": { - "pretest": "jshint index.js bin parts preprocessors test && jscs index.js bin parts preprocessors test", + "pretest": "jshint index.js bin parts preprocessors test && eslint index.js bin parts preprocessors test && jscs index.js bin parts preprocessors test", "test": "tape test/*.test.js", "coverage": "istanbul cover tape test/*.test.js && coveralls < ./coverage/lcov.info" }, @@ -23,18 +23,21 @@ "homepage": "https://github.com/mapbox/preprocessorcerer", "devDependencies": { "concat-stream": "^1.4.8", + "eslint": "^1.6.0", "jscs": "^1.10.0", "jshint": "^2.6.0", "tape": "^3.4.0", "tilelive": "^5.7.0", "istanbul": "~0.3.17", "coveralls": "~2.11.2", - "checksum": "^0.1.1" + "checksum": "^0.1.1", + "rimraf": "^2.5.2" }, "dependencies": { "bytetiff": "~0.3.0", "gdal": "~0.8.0", "mapbox-file-sniff": "~0.4.0", + "mapnik-omnivore": "~7.4.0", "mapnik": "~3.5.0", "mkdirp": "^0.5.0", "mbtiles": "^0.8.0", @@ -55,6 +58,7 @@ "preset": "airbnb", "requireCamelCaseOrUpperCaseIdentifiers": null, "disallowMultipleVarDecl": true, - "requireMultipleVarDecl": null + "requireMultipleVarDecl": null, + "validateLineBreaks": null } } diff --git a/parts/mbtiles-byTiles.js b/parts/mbtiles-byTiles.js index 9705e88..0c3439f 100644 --- a/parts/mbtiles-byTiles.js +++ b/parts/mbtiles-byTiles.js @@ -18,11 +18,15 @@ module.exports = function splitBySize(filepath, info, callback) { } else if (err) return callback(err); - var maxTilesPerJob = 100000; - var result = tilesResult.count + gridsResult.count; - if (!result) return callback(new Error('no tiles or grids')); + src.close(function(err) { + if (err) return callback(err); + src = null; + var maxTilesPerJob = 100000; + var result = tilesResult.count + gridsResult.count; + if (!result) return callback(new Error('no tiles or grids')); - callback(null, Math.min(50, Math.ceil(result / maxTilesPerJob))); + callback(null, Math.min(50, Math.ceil(result / maxTilesPerJob))); + }); }); }); }); diff --git a/preprocessors/geojson-bom.preprocessor.js b/preprocessors/geojson-bom.preprocessor.js index 196fafb..39c5992 100644 --- a/preprocessors/geojson-bom.preprocessor.js +++ b/preprocessors/geojson-bom.preprocessor.js @@ -32,7 +32,10 @@ module.exports.criteria = function(infile, info, callback) { fs.read(fd, buf, 0, 3, 0, function(err) { if (err) return callback(err); var strip = buf[0] === 0xef && buf[1] === 0xbb && buf[2] == 0xbf; - callback(null, strip); + fs.close(fd, function(err) { + if (err) return callback(err); + callback(null, strip); + }); }); }); }; diff --git a/preprocessors/index.js b/preprocessors/index.js index 88b8afd..209b169 100644 --- a/preprocessors/index.js +++ b/preprocessors/index.js @@ -10,7 +10,9 @@ var preprocessors = [ 'tif-reproject.preprocessor', 'shp-index.preprocessor', 'geojson-bom.preprocessor', - 'spatial-index.preprocessor' + 'spatial-index.preprocessor', + 'togeojson-kml.preprocessor', + 'togeojson-gpx.preprocessor' ]; // Loads each *.preprocessor.js file and builds an array of them diff --git a/preprocessors/shp-index.preprocessor.js b/preprocessors/shp-index.preprocessor.js index 103bc36..cf83a8e 100644 --- a/preprocessors/shp-index.preprocessor.js +++ b/preprocessors/shp-index.preprocessor.js @@ -2,7 +2,7 @@ var path = require('path'); var fs = require('fs'); var spawn = require('child_process').spawn; var mapnik = require('mapnik'); -var shapeindex = path.resolve(mapnik.module_path, 'shapeindex'); +var shapeindex = path.resolve(mapnik.module_path, 'shapeindex' + (process.platform === 'win32' ? '.exe' : '')); if (!fs.existsSync(shapeindex)) { throw new Error('shapeindex does not exist at ' + shapeindex); } diff --git a/preprocessors/tif-reproject.preprocessor.js b/preprocessors/tif-reproject.preprocessor.js index 6f69ca5..27cbcc6 100644 --- a/preprocessors/tif-reproject.preprocessor.js +++ b/preprocessors/tif-reproject.preprocessor.js @@ -27,6 +27,9 @@ module.exports.criteria = function(filepath, info, callback) { } catch (err) { return callback(err); } + ds.close(); + ds = null; + if (projection === sm) callback(null, false); else callback(null, true); }; diff --git a/preprocessors/tif-toBytes.preprocessor.js b/preprocessors/tif-toBytes.preprocessor.js index 2f89e43..fcd41f6 100644 --- a/preprocessors/tif-toBytes.preprocessor.js +++ b/preprocessors/tif-toBytes.preprocessor.js @@ -21,6 +21,10 @@ module.exports.criteria = function(filepath, info, callback) { try { band = ds.bands.get(1); } catch (err) { return callback(err); } - if (band.dataType === gdal.GDT_UInt16) return callback(null, true); - callback(null, false); + var is16 = band.dataType === gdal.GDT_UInt16; + + ds.close(); + ds = null; + + callback(null, is16); }; diff --git a/preprocessors/togeojson-gpx.preprocessor.js b/preprocessors/togeojson-gpx.preprocessor.js new file mode 100644 index 0000000..ada823f --- /dev/null +++ b/preprocessors/togeojson-gpx.preprocessor.js @@ -0,0 +1,156 @@ +var gdal = require('gdal'); +var fs = require('fs'); +var mkdirp = require('mkdirp'); +var queue = require('queue-async'); +var spawn = require('child_process').spawn; +var path = require('path'); +var digest = require('mapnik-omnivore').digest; +var mapnik = require('mapnik'); +var mapnik_index = path.resolve(mapnik.module_path, 'mapnik-index' + (process.platform === 'win32' ? '.exe' : '')); +if (!fs.existsSync(mapnik_index)) { + throw new Error('mapnik-index does not exist at ' + mapnik_index); +} + +//disable in production +//gdal.verbose(); + +module.exports = function(infile, outdirectory, callback) { + mkdirp(outdirectory, function(err) { + if (err) return callback(err); + + var geojson_files = []; + var ds_gpx; + var full_feature_cnt = 0; + var wgs84 = gdal.SpatialReference.fromEPSG(4326); + + try { + ds_gpx = gdal.open(infile); + } + catch (err) { + return callback(new Error(err)); + } + + ds_gpx.layers.forEach(function(lyr_gpx) { + //drop point layers as they can get really huge + if (lyr_gpx.name === 'track_points' || lyr_gpx.name === 'route_points') { + return; + } + + var feat_cnt = lyr_gpx.features.count(true); + if (feat_cnt === 0) { + return; + } + + var geojson; + var lyr_name; + var out_ds; + var out_name; + + try { + lyr_name = lyr_gpx.name; + out_name = path.join(outdirectory, lyr_name + '.geojson'); + out_ds = gdal.open(out_name, 'w', 'GeoJSON'); + geojson = out_ds.layers.create(lyr_name, wgs84, lyr_gpx.geomType); + } + catch (err) { + return callback(new Error(err)); + } + + lyr_gpx.features.forEach(function(gpx_feat) { + //skip null or empty geometries + var geom = gpx_feat.getGeometry(); + if (!geom) { + return; + } else { + if (geom.isEmpty()) { + return; + } + + if (!geom.isValid()) { + return; + } + } + + geojson.features.add(gpx_feat); + full_feature_cnt++; + }); + + geojson.flush(); + out_ds.flush(); + out_ds.close(); + + //release objects to be able to index + geojson = null; + out_ds = null; + + geojson_files.push(out_name); + }); + + ds_gpx.close(); + if (full_feature_cnt === 0) { + return callback(new Error('GPX does not contain any valid features.')); + } + + // Create metadata file for original gpx source + var metadatafile = path.join(outdirectory, '/metadata.json'); + digest(infile, function(err, metadata) { + if (err) return callback(err); + fs.writeFile(metadatafile, JSON.stringify(metadata), function(err) { + if (err) return callback(err); + return createIndices(callback); + }); + }); + + function createIndices(callback) { + // create mapnik index for each geojson layer + var q = queue(); + geojson_files.forEach(function(gj) { + q.defer(createIndex, gj); + }); + + q.awaitAll(function(err) { + if (err) return callback(err); + return callback(); + }); + } + + function createIndex(layerfile, callback) { + // Finally, create an .index file in the output dir (if layer is greater than index_worthy_size). + // mapnik-index will automatically add ".index" to the end of the original filename + fs.stat(layerfile, function(err, stats) { + if (err) return callback(err); + + // check size is warrants creating an index + if (stats.size >= module.exports.index_worthy_size) { + var data = ''; + var p = spawn(mapnik_index, [layerfile, '--validate-features']) + .once('error', callback) + .on('exit', function() { + // If error printed to --validate-features log + if (data.indexOf('Error') != -1) { + return callback(data); + } + else return callback(); + }); + + p.stderr.on('data', function(d) { + d.toString(); + data += d; + }); + } else { + return callback(); + } + }); + } + }); +}; + +module.exports.description = 'Convert GPX to GeoJSON'; +module.exports.index_worthy_size = 10 * 1024 * 1024; // 10 MB + +module.exports.criteria = function(filepath, info, callback) { + + if (info.filetype !== 'gpx') return callback(null, false); + + callback(null, true); +}; diff --git a/preprocessors/togeojson-kml.preprocessor.js b/preprocessors/togeojson-kml.preprocessor.js new file mode 100644 index 0000000..dc2fda7 --- /dev/null +++ b/preprocessors/togeojson-kml.preprocessor.js @@ -0,0 +1,190 @@ +var gdal = require('gdal'); +var fs = require('fs'); +var mkdirp = require('mkdirp'); +var queue = require('queue-async'); +var path = require('path'); +var util = require('util'); +var digest = require('mapnik-omnivore').digest; +var mapnik = require('mapnik'); +var spawn = require('child_process').spawn; +var mapnik_index = path.resolve(mapnik.module_path, 'mapnik-index' + (process.platform === 'win32' ? '.exe' : '')); +if (!fs.existsSync(mapnik_index)) { + throw new Error('mapnik-index does not exist at ' + mapnik_index); +} + +//disable in production +//gdal.verbose(); + +module.exports = function(infile, outdirectory, callback) { + + mkdirp(outdirectory, function(err) { + if (err) return callback(err); + + var geojson_files = []; + var wgs84; + var ds_kml; + var lyr_cnt; + var full_feature_cnt; + + try { + wgs84 = gdal.SpatialReference.fromEPSG(4326); + ds_kml = gdal.open(infile); + lyr_cnt = ds_kml.layers.count(); + full_feature_cnt = 0; + } + catch (err) { + return callback(new Error(err)); + } + + if (lyr_cnt < 1) { + ds_kml.close(); + return callback(new Error('KML does not contain any layers.')); + } + + if (lyr_cnt > module.exports.max_layer_count) { + ds_kml.close(); + return callback(new Error(util.format('%d layers found. Maximum of %d layers allowed.', lyr_cnt, module.exports.max_layer_count))); + } + + var duplicate_lyr_msg = layername_count(ds_kml); + if (duplicate_lyr_msg) { + ds_kml.close(); + return callback(new Error(duplicate_lyr_msg)); + } + + ds_kml.layers.forEach(function(lyr_kml) { + var feat_cnt = lyr_kml.features.count(true); + if (feat_cnt === 0) return; + + //strip kml from layer name. features at the root get the KML filename as layer name + var out_ds; + var geojson; + var lyr_name = lyr_kml.name + .replace(/.kml/gi, '') + .replace(/[ \\/&?]/g, '_') + .replace(/[();:,\]\[{}]/g, ''); + var out_name = path.join(outdirectory, lyr_name + '.geojson'); + + try { + out_ds = gdal.open(out_name, 'w', 'GeoJSON'); + geojson = out_ds.layers.create(lyr_name, wgs84, lyr_kml.geomType); + } + catch (err) { + return callback(new Error(err)); + } + + lyr_kml.features.forEach(function(kml_feat) { + var geom = kml_feat.getGeometry(); + if (!geom) return; + else { + if (geom.isEmpty()) return; + if (!geom.isValid()) return; + } + + geojson.features.add(kml_feat); + full_feature_cnt++; + }); + + geojson.flush(); + out_ds.flush(); + out_ds.close(); + + //release objects to be able to index + geojson = null; + out_ds = null; + + geojson_files.push(out_name); + }); + + ds_kml.close(); + if (full_feature_cnt === 0) { + return callback(new Error('KML does not contain any valid features')); + } + + // Create metadata file for original gpx source + var metadatafile = path.join(outdirectory, '/metadata.json'); + digest(infile, function(err, metadata) { + fs.writeFile(metadatafile, JSON.stringify(metadata), function(err) { + if (err) return callback(err); + return createIndices(callback); + }); + }); + + function createIndices(callback) { + // create mapnik index for each geojson layer + var q = queue(); + geojson_files.forEach(function(gj) { + q.defer(createIndex, gj); + }); + + q.awaitAll(function(err) { + if (err) return callback(err); + return callback(); + }); + } + + function createIndex(layerfile, callback) { + // Finally, create an .index file in the output dir (if layer is greater than index_worthy_size). + // mapnik-index will automatically add ".index" to the end of the original filename + fs.stat(layerfile, function(err, stats) { + if (err) return callback(err); + + // check size is warrants creating an index + if (stats.size >= module.exports.index_worthy_size) { + var data = ''; + var p = spawn(mapnik_index, [layerfile, '--validate-features']) + .once('error', callback) + .on('exit', function() { + // If error printed to --validate-features log + if (data.indexOf('Error') != -1) { + return callback(data); + } + else return callback(); + }); + + p.stderr.on('data', function(d) { + d.toString(); + data += d; + }); + } else { + return callback(); + } + }); + } + }); +}; + +module.exports.description = 'Convert KML to GeoJSON'; + +module.exports.criteria = function(filepath, info, callback) { + + if (info.filetype !== 'kml') return callback(null, false); + + callback(null, true); +}; + +function layername_count(ds) { + var lyr_name_cnt = {}; + ds.layers.forEach(function(lyr) { + var lyr_name = lyr.name; + if (lyr_name in lyr_name_cnt) { + lyr_name_cnt[lyr_name]++; + } else { + lyr_name_cnt[lyr_name] = 1; + } + }); + + var err = ''; + for (var name in lyr_name_cnt) { + var cnt = lyr_name_cnt[name]; + if (cnt > 1) { + err += util.format('%s\'%s\' found %d times', err.length > 0 ? ', ' : '', name, cnt); + } + } + + return err.length > 0 ? 'Duplicate layer names! ' + err : null; +} + +//expose this as ENV option? +module.exports.max_layer_count = 15; +module.exports.index_worthy_size = 10 * 1024 * 1024; // 10 MB diff --git a/test/end2end.test.js b/test/end2end.test.js index 812e019..f68b097 100644 --- a/test/end2end.test.js +++ b/test/end2end.test.js @@ -2,7 +2,9 @@ var test = require('tape'); var path = require('path'); var os = require('os'); var fs = require('fs'); +var rimraf = require('rimraf'); var crypto = require('crypto'); +var queue = require('queue-async'); var gdal = require('gdal'); var preprocess = require('..'); var exec = require('child_process').exec; @@ -13,7 +15,15 @@ var MBtiles = require('mbtiles'); // with random names will litter the fixture directory var fixtureDir = path.resolve(__dirname, 'fixtures', 'end2end'); var tmpdir = path.join(os.tmpdir(), crypto.randomBytes(8).toString('hex')); -var cmd = ['cp -r', fixtureDir, tmpdir].join(' '); +var copy_cmd = 'cp -r'; +if (process.platform === 'win32') { + //during tests %PATH% env var is completely stripped. add Windows system dir back to get xcopy + process.env.Path += ';' + path.join(process.env.windir, 'system32'); + copy_cmd = 'xcopy /s /y'; + tmpdir += '\\'; +} + +var cmd = [copy_cmd, fixtureDir, tmpdir].join(' '); exec(cmd, function(err) { if (err) throw err; @@ -42,17 +52,34 @@ exec(cmd, function(err) { }; }); + var q = queue(); fixtures.forEach(function(fixture) { - test('[end2end ' + fixture.name + ']', function(assert) { - preprocess(fixture.filepath, function(err, outfile, parts, descriptions) { - assert.ifError(err, 'preprocessed'); - assert.deepEqual(descriptions, fixture.descriptions, 'expected preprocessorcery performed'); - outputCheck(outfile, fixture.type, assert, function() { - assert.end(); + q.defer(function(next) { + test('[end2end ' + fixture.name + ']', function(assert) { + preprocess(fixture.filepath, function(err, outfile, parts, descriptions) { + assert.ifError(err, 'preprocessed'); + assert.deepEqual(descriptions, fixture.descriptions, 'expected preprocessorcery performed'); + outputCheck(outfile, fixture.type, assert, function() { + assert.end(); + next(); + }); }); }); }); }); + + q.defer(function(cb) { + test('[end2end delete temporary data]', function(assert) { + rimraf(tmpdir, function(err) { + assert.end(err); + cb(err); + }); + }); + }); + + q.awaitAll(function(err) { + if (err) console.error(err); + }); }); // Given only a filetype, what assertions can we make about the outfile? @@ -78,6 +105,8 @@ function outputCheck(outfile, type, assert, callback) { assert.ok(!layer.srs.isSame(mercator), layer.name + ' projected to spherical mercator'); }); + ds.close(); + ds = null; return callback(); } @@ -89,7 +118,10 @@ function outputCheck(outfile, type, assert, callback) { var fd = fs.openSync(outfile, 'r'); fs.readSync(fd, buf, 0, 3, 0); assert.notOk(buf[0] === 0xef && buf[1] === 0xbb && buf[2] == 0xbf, 'no BOM'); + fs.closeSync(fd); + ds.close(); + ds = null; return callback(); } @@ -101,11 +133,13 @@ function outputCheck(outfile, type, assert, callback) { // each band is 8-bit and has overviews ds.bands.forEach(function(band) { - assert.equal(band.dataType, gdal.GDT_Byte, 'band ' + band.id + ' is 8-bit'); + assert.equal(band.dataType, gdal.GDT_Byte, 'band ' + band.id + ' is 8-bit'); // assert.ok(band.overviews.count() >= 10, 'band ' + band.id + ' has overviews'); }); + ds.close(); + ds = null; return callback(); } diff --git a/test/fixtures/gpx/fail-corrupted-file.gpx b/test/fixtures/gpx/fail-corrupted-file.gpx new file mode 100644 index 0000000..b8ffc1c --- /dev/null +++ b/test/fixtures/gpx/fail-corrupted-file.gpx @@ -0,0 +1,5 @@ + + + + + + + + + + + diff --git a/test/fixtures/gpx/ok-valid-file.gpx b/test/fixtures/gpx/ok-valid-file.gpx new file mode 100644 index 0000000..e8f407a --- /dev/null +++ b/test/fixtures/gpx/ok-valid-file.gpx @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/test/fixtures/kml/fail-duplicate-layer-names.kml b/test/fixtures/kml/fail-duplicate-layer-names.kml new file mode 100644 index 0000000..1127f0e --- /dev/null +++ b/test/fixtures/kml/fail-duplicate-layer-names.kml @@ -0,0 +1,19 @@ + + + + duplicate layer names + 1 + + duplicate layer name + + + duplicate layer name + + + layer 2 + + + layer 2 + + + diff --git a/test/fixtures/kml/fail-empty-features-only.kml b/test/fixtures/kml/fail-empty-features-only.kml new file mode 100644 index 0000000..dee4289 --- /dev/null +++ b/test/fixtures/kml/fail-empty-features-only.kml @@ -0,0 +1,88 @@ + + + + empty-features-only.kml + + + normal + #s_ylw-pushpin + + + highlight + #s_ylw-pushpin_hl0 + + + + + + + + + normal + #s_ylw-pushpin0 + + + highlight + #s_ylw-pushpin_hl + + + + empty-features-only + 1 + + empty polygon + #m_ylw-pushpin + + 1 + + + + + + + + + + empty line + #m_ylw-pushpin0 + + 1 + + + + + + + diff --git a/test/fixtures/kml/fail-invalid.kml b/test/fixtures/kml/fail-invalid.kml new file mode 100644 index 0000000..367da2f --- /dev/null +++ b/test/fixtures/kml/fail-invalid.kml @@ -0,0 +1,6 @@ + + + + invalid + 1 + \ No newline at end of file diff --git a/test/fixtures/kml/fail-more-than-15-layers.kml b/test/fixtures/kml/fail-more-than-15-layers.kml new file mode 100644 index 0000000..602a80d --- /dev/null +++ b/test/fixtures/kml/fail-more-than-15-layers.kml @@ -0,0 +1,90 @@ + + + + more than 15 layers + 1 + + layer 1 + 1 + + + layer 2 + 1 + + + layer 3 + 1 + + + layer 4 + 1 + + + layer 5 + 1 + + + layer 6 + 1 + + + layer 7 + 1 + + + layer 8 + 1 + + + layer 9 + 1 + + + layer 10 + 1 + + + layer 11 + 1 + + + layer 12 + 1 + + + layer 13 + 1 + + + layer 14 + 1 + + + layer 15 + 1 + + + layer 16 + 1 + + + layer 17 + 1 + + + layer 18 + 1 + + + layer 19 + 1 + + + layer 20 + 1 + + + layer 21 + + + diff --git a/test/fixtures/kml/ok-layers-folders-emptygeometries.kml b/test/fixtures/kml/ok-layers-folders-emptygeometries.kml new file mode 100644 index 0000000..f85a609 --- /dev/null +++ b/test/fixtures/kml/ok-layers-folders-emptygeometries.kml @@ -0,0 +1,368 @@ + + + + my-test.kml + 1 + + + + + + normal + #sn_ylw-pushpin0 + + + highlight + #sh_ylw-pushpin1 + + + + + + + + + normal + #s_ylw-pushpin1 + + + highlight + #s_ylw-pushpin_hl + + + + + normal + #s_ylw-pushpin0 + + + highlight + #s_ylw-pushpin_hl1 + + + + + normal + #s_ylw-pushpin + + + highlight + #s_ylw-pushpin_hl0 + + + + + normal + #sn_ylw-pushpin1 + + + highlight + #sh_ylw-pushpin0 + + + + + + + + + normal + #sn_ylw-pushpin + + + highlight + #sh_ylw-pushpin + + + + + point-01 + desc point-01 + + 16.40966023583767 + 48.1925069364187 + 10368.63837475184 + 9.350800161469291 + 24.50671614749143 + -1.234536521210208 + relativeToSeaFloor + + #m_ylw-pushpin0 + + 1 + 16.41350218768393,48.23785515709087,0 + + + + line-01 + https://www.mapbox.com]]> + #m_ylw-pushpin + + 1 + + 16.43667340118919,48.2137480651988,0 16.43405477350426,48.22364142638448,0 16.43490893897016,48.23095505929151,0 16.4302280634805,48.23577097007643,0 16.42370258660831,48.24160187549051,0 16.41306849787137,48.24664180670233,0 16.4020238900489,48.24966207010675,0 16.39124480989305,48.24998720617013,0 + + + + + polygon-01 + poly 01 desc + #m_ylw-pushpin1 + + 1 + + + + 16.39880116789466,48.24185818661861,0 16.40685069000169,48.23402265838939,0 16.42861766173791,48.21955165225462,0 16.42570050409206,48.23076672152668,0 16.42623122485864,48.23663446297304,0 16.42065446360477,48.24101476128649,0 16.41009581088264,48.2463984284906,0 16.39528063678205,48.24815169056571,0 16.39880116789466,48.24185818661861,0 + + + + + + + folder-02 + 1 + desc folder-02 + 1 + + polygon-03-empty + desc polygon-03-empty + #msn_ylw-pushpin0 + + 1 + + + + + + + + + + line04-empty + desc line04-empty + #msn_ylw-pushpin1 + + 1 + + + + + + point-03-empty + desc point-03-empty + + 16.40966023583767 + 48.1925069364187 + 10368.63837475184 + 9.350800161469291 + 24.50671614749143 + -1.234536521210208 + relativeToSeaFloor + + #m_ylw-pushpin0 + + 1 + 16.42002042755009,48.23441390791601,0 + + + + + folder-01 + 1 + desc folder-01 + + polygon-02 + desc polygon-02 + #msn_ylw-pushpin0 + + 1 + + + + 16.37117328551933,48.2580459823847,0 16.37265821578014,48.24973712333478,0 16.36217236784517,48.23463724442687,0 16.36784902284754,48.22609016751182,0 16.36942113029388,48.22027233512995,0 16.37688332475139,48.21360006593482,0 16.38339377402438,48.21197484254892,0 16.39003747137017,48.21391783901046,0 16.39405868247631,48.21304620343092,0 16.39782482806304,48.20766795435579,0 16.39844282391005,48.20358062827769,0 16.40723315758465,48.20112830060737,0 16.41594063835881,48.19413854002189,0 16.43337009687389,48.20504934949183,0 16.37117328551933,48.2580459823847,0 + + + + + + + line-02 + desc line-02 + #msn_ylw-pushpin + + 1 + + 16.37894775762214,48.21134702941436,0 16.43387685193025,48.24114369357682,0 + + + + + point-02-label + + 16.40966023583767 + 48.1925069364187 + 10368.63837475184 + 9.350800161469291 + 24.50671614749143 + -1.234536521210208 + relativeToSeaFloor + + #m_ylw-pushpin0 + + 1 + 16.38705972732914,48.22482302405027,0 + + + + folder-01-01 + desc folder-01-01: subfolder of folder-01 + + line-03 + desc line-03 + #msn_ylw-pushpin1 + + 1 + + 16.44335342786484,48.20114149643969,0 16.40568329778731,48.23086456817817,0 16.36679489808114,48.27148805960164,0 + + + + + + + diff --git a/test/fixtures/kml/ok-special-character-layer.kml b/test/fixtures/kml/ok-special-character-layer.kml new file mode 100644 index 0000000..fe105ef --- /dev/null +++ b/test/fixtures/kml/ok-special-character-layer.kml @@ -0,0 +1,53 @@ + + + + special-character-layer.kml + 1 + + + + normal + #s_ylw-pushpin + + + highlight + #s_ylw-pushpin_hl + + + + + special (characters; in & this layer) + 1 + + austria + #m_ylw-pushpin + + 1 + + + + 9.948510701496256,47.59254872603884,0 9.672308521268427,47.51887202620658,0 9.611972620508587,47.10144711856087,0 10.11419595290811,46.88714749310341,0 11.09025185164879,46.82641134193905,0 11.2376024800754,46.97309929750735,0 12.13119294670139,46.97849267002405,0 12.51525275763799,46.63142694751628,0 14.50865044058264,46.37350618755779,0 14.99391010229719,46.62667660271019,0 16.02143725382271,46.64721174317448,0 15.96370578945158,46.82053731322057,0 16.4874285966801,46.9927325908894,0 16.37902770093174,47.37192585223085,0 16.65294287220463,47.39888541041359,0 16.73148946028918,47.61863474528875,0 17.07966630883628,47.69436361828313,0 17.24417441158853,48.01782751821557,0 16.83586264286678,48.43898064750138,0 17.0557805956535,48.67694767611934,0 15.03343213066511,48.95727325862828,0 14.7368523875758,48.6029992202009,0 13.91386920367444,48.75468430317257,0 12.79851598707308,48.1284798056311,0 13.04876099811421,47.52574933991964,0 9.916972408310627,47.44292919604388,0 9.948510701496256,47.59254872603884,0 + + + + + + + + diff --git a/test/fixtures/mbtiles-count/make-mbtiles.js b/test/fixtures/mbtiles-count/make-mbtiles.js index 11e3de1..77e15f7 100755 --- a/test/fixtures/mbtiles-count/make-mbtiles.js +++ b/test/fixtures/mbtiles-count/make-mbtiles.js @@ -31,7 +31,14 @@ module.exports = function(filepath, numTiles, callback) { q.awaitAll(function(err) { if (err) return callback(err); - mbtiles.stopWriting(callback); + mbtiles.stopWriting(function(err) { + if (err) return callback(err); + mbtiles.close(function(err) { + if (err) return callback(err); + mbtiles = null; + callback(); + }); + }); }); }); }); diff --git a/test/geojson-bom.test.js b/test/geojson-bom.test.js index 37e3484..49f96a1 100644 --- a/test/geojson-bom.test.js +++ b/test/geojson-bom.test.js @@ -43,6 +43,8 @@ test('[geojson-bom] removes bom', function(assert) { JSON.parse(outData); }, 'outfile is JSON.parse-able'); - assert.end(); + fs.unlink(outfile, function(err) { + assert.end(err); + }); }); }); diff --git a/test/index.test.js b/test/index.test.js index c7def77..e1c307f 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -21,7 +21,8 @@ test('[index] preprocesses', function(assert) { assert.ok(fs.statSync(outfile), 'outfile exists'); assert.ok(!isNaN(parts), 'reported a number of parts'); assert.ok(Array.isArray(descriptions), 'returns array of descriptions'); - fs.unlink(tmpfile); + fs.unlinkSync(outfile); + fs.unlinkSync(tmpfile); assert.end(); }); } @@ -40,7 +41,7 @@ test('[index] preprocesses and creates index last', function(assert) { assert.ifError(err, 'no error'); assert.ok(Array.isArray(descriptions), 'returns array of descriptions'); assert.equal(descriptions[descriptions.length - 1], 'Add a spatial index to GeoJSON or CSV'); - fs.unlink(tmpfile); + fs.unlinkSync(tmpfile); assert.end(); }); } diff --git a/test/mbtiles-byTiles.test.js b/test/mbtiles-byTiles.test.js index 31a9349..6ea97e6 100644 --- a/test/mbtiles-byTiles.test.js +++ b/test/mbtiles-byTiles.test.js @@ -14,7 +14,7 @@ test('[parts mbtiles] correct number of parts for mbtiles without grids table', mbtilesByTiles(fixture, { filetype: 'mbtiles' }, function(err, parts) { assert.ifError(err, 'no error'); assert.equal(parts, 4, 'four parts'); - fs.unlink(fixture); + fs.unlinkSync(fixture); assert.end(); }); }); diff --git a/test/preprocessors.test.js b/test/preprocessors.test.js index a2a4502..b28512d 100644 --- a/test/preprocessors.test.js +++ b/test/preprocessors.test.js @@ -77,7 +77,8 @@ test('[preprocessorcery] preprocessorize', function(assert) { var indir = path.dirname(tmpfile); var outdir = path.dirname(outfile); assert.equal(outdir, indir, 'places output files in same directory as input'); - fs.unlink(tmpfile); + fs.unlinkSync(tmpfile); + fs.unlinkSync(outfile); assert.end(); }); } diff --git a/test/shp-index.test.js b/test/shp-index.test.js index c487b8e..b04af1d 100644 --- a/test/shp-index.test.js +++ b/test/shp-index.test.js @@ -5,6 +5,7 @@ var fs = require('fs'); var crypto = require('crypto'); var index = require('../preprocessors/shp-index.preprocessor'); var mkdirp = require('mkdirp'); +var rimraf = require('rimraf'); function tmpfile(callback) { var dir = path.join(os.tmpdir(), crypto.randomBytes(8).toString('hex')); @@ -46,7 +47,9 @@ test('[shp-index] indexes (input folder output file)', function(assert) { fs.stat(outfile, function(err, stats) { assert.equal(files.length, 1, 'created index file'); assert.equal(stats.size, 428328, 'index created using index-parts'); - assert.end(); + rimraf(path.dirname(outfile), function(err) { + assert.end(err); + }); }); }); }); @@ -65,7 +68,9 @@ test('[shp-index] indexes (input folder output folder)', function(assert) { fs.stat(outfile, function(err, stats) { assert.equal(files.length, 1, 'created index file'); assert.equal(stats.size, 428328, 'index created using index-parts'); - assert.end(); + rimraf(path.dirname(outfile), function(err) { + assert.end(err); + }); }); }); }); @@ -84,7 +89,9 @@ test('[shp-index] indexes (input file output file)', function(assert) { fs.stat(outfile, function(err, stats) { assert.equal(files.length, 1, 'created index file'); assert.equal(stats.size, 428328, 'index created using index-parts'); - assert.end(); + rimraf(path.dirname(outfile), function(err) { + assert.end(err); + }); }); }); }); @@ -103,7 +110,9 @@ test('[shp-index] indexes (input file output folder)', function(assert) { fs.stat(outfile, function(err, stats) { assert.equal(files.length, 1, 'created index file'); assert.equal(stats.size, 428328, 'index created using index-parts'); - assert.end(); + rimraf(path.dirname(outfile), function(err) { + assert.end(err); + }); }); }); }); @@ -120,7 +129,9 @@ test('[shp-index] does not index (input folder output file - no index)', functio }); assert.equal(files.length, 0, 'did not create index file'); - assert.end(); + rimraf(path.dirname(outfile), function(err) { + assert.end(err); + }); }); }); }); @@ -136,7 +147,9 @@ test('[shp-index] does not index (input file output file - no index)', function( }); assert.equal(files.length, 0, 'did not create index file'); - assert.end(); + rimraf(path.dirname(outfile), function(err) { + assert.end(err); + }); }); }); }); @@ -152,7 +165,9 @@ test('[shp-index] does not index (input file output folder - no index)', functio }); assert.equal(files.length, 0, 'did not create index file'); - assert.end(); + rimraf(path.dirname(outfile), function(err) { + assert.end(err); + }); }); }); }); @@ -168,7 +183,9 @@ test('[shp-index] does not contain pre-existing index (input file output folder }); assert.equal(files.length, 0, 'does not contain pre-existing index'); - assert.end(); + rimraf(path.dirname(outfile), function(err) { + assert.end(err); + }); }); }); }); diff --git a/test/spatial-index.test.js b/test/spatial-index.test.js index eaa9db5..3fa51d9 100644 --- a/test/spatial-index.test.js +++ b/test/spatial-index.test.js @@ -6,6 +6,7 @@ var crypto = require('crypto'); var index = require('../preprocessors/spatial-index.preprocessor'); var mkdirp = require('mkdirp'); var checksum = require('checksum'); +var rimraf = require('rimraf'); function tmpdir(callback) { var dir = path.join(os.tmpdir(), crypto.randomBytes(8).toString('hex')); @@ -26,7 +27,7 @@ test('[spatial-index] criteria: not an indexable file', function(assert) { }); test('[spatial-index] exposes index_worthy_size', function(assert) { - assert.equal(index.index_worthy_size, 10485760); + assert.equal(index.index_worthy_size, 10 * 1024 * 1024); //10MB assert.end(); }); @@ -61,7 +62,10 @@ test('[spatial-index] indexes (input folder output file)', function(assert) { assert.equal(original, sum); assert.ifError(err, 'no error'); assert.ok(fs.existsSync(path.join(outdir, 'valid.geojson.index'))); - assert.end(); + rimraf(outdir, function(err) { + if (err) throw err; + assert.end(); + }); }); }); }); @@ -81,7 +85,10 @@ test('[spatial-index] handles error in case of invalid feature', function(assert checksum.file(path.join(outdir, 'index-validate-flag.geojson'), function(error, sum) { assert.equal(original, sum); assert.notOk(fs.existsSync(path.join(outdir, 'index-validate-flag.geojson.index')), 'index file should not exist due to feature error'); - assert.end(); + rimraf(outdir, function(err) { + if (err) throw err; + assert.end(); + }); }); }); }); diff --git a/test/tif-reproject.test.js b/test/tif-reproject.test.js index 20345ed..279ed6a 100644 --- a/test/tif-reproject.test.js +++ b/test/tif-reproject.test.js @@ -1,6 +1,7 @@ var test = require('tape'); var reproject = require('../preprocessors/tif-reproject.preprocessor'); var os = require('os'); +var fs = require('fs'); var path = require('path'); var crypto = require('crypto'); var googleMerc = path.resolve(__dirname, 'fixtures', 'google-merc.tif'); @@ -56,6 +57,9 @@ test('[tif-reproject] reprojection: to epsg:3857', function(assert) { assert.ifError(err, 'no error'); var ds = gdal.open(outfile); assert.ok(ds.srs.isSame(gdal.SpatialReference.fromEPSG(3857)), 'reprojected correctly'); + ds.close(); + ds = null; + fs.unlinkSync(outfile); assert.end(); }); }); diff --git a/test/tif-toBytes.test.js b/test/tif-toBytes.test.js index fcd03e2..e6a04b7 100644 --- a/test/tif-toBytes.test.js +++ b/test/tif-toBytes.test.js @@ -1,6 +1,7 @@ var test = require('tape'); var toBytes = require('../preprocessors/tif-toBytes.preprocessor'); var os = require('os'); +var fs = require('fs'); var path = require('path'); var crypto = require('crypto'); var geojson = path.resolve(__dirname, 'fixtures', 'valid.geojson'); @@ -49,6 +50,10 @@ test('[tif-toBytes] convert to bytes', function(assert) { assert.equal(band.dataType, gdal.GDT_Byte, 'converted band ' + band.id + ' to bytes'); }); - assert.end(); + ds.close(); + ds = null; + fs.unlink(outfile, function(err) { + assert.end(err); + }); }); }); diff --git a/test/togeojson-gpx.test.js b/test/togeojson-gpx.test.js new file mode 100644 index 0000000..b0e2999 --- /dev/null +++ b/test/togeojson-gpx.test.js @@ -0,0 +1,81 @@ +var test = require('tape'); +var path = require('path'); +var os = require('os'); +var fs = require('fs'); +var crypto = require('crypto'); +var mkdirp = require('mkdirp'); +var rimraf = require('rimraf'); +var exec = require('child_process').exec; +var togeojson = require('../preprocessors/togeojson-gpx.preprocessor'); + +// Perform all preprocessorcery in a temporary directory. Otherwise output files +// with random names will litter the fixture directory +function tmpdir(callback) { + var dir = path.join(os.tmpdir(), crypto.randomBytes(8).toString('hex')); + var fixtureDir = path.resolve(__dirname, 'fixtures', 'gpx'); + var copy_cmd = 'cp -r'; + + if (process.platform === 'win32') { + //during tests %PATH% env var is completely stripped. add Windows system dir back to get xcopy + process.env.Path += ';' + path.join(process.env.windir, 'system32'); + copy_cmd = 'xcopy /s /y'; + dir += '\\'; + var cmd = [copy_cmd, fixtureDir, dir].join(' '); + exec(cmd, function(err) { + if (err) return callback(err); + callback(null, dir); + }); + } else { + mkdirp(dir, function(err) { + if (err) return callback(err); + callback(null, dir); + }); + } +} + +test('[GPX togeojson] fails duplicate layer names', function(assert) { + var infile = path.resolve(__dirname, 'fixtures', 'gpx', 'fail-no-features.gpx'); + + tmpdir(function(err, outdir) { + togeojson(infile, outdir, function(err) { + assert.ok(err, 'error properly handled'); + assert.equal(err.message, 'GPX does not contain any valid features.', 'expected error message'); + rimraf(outdir, function(err) { + assert.end(err); + }); + }); + }); +}); + +test('[GPX togeojson] fails empty features only', function(assert) { + var infile = path.resolve(__dirname, 'fixtures', 'gpx', 'fail-corrupted-file.gpx'); + + tmpdir(function(err, outdir) { + togeojson(infile, outdir, function(err) { + assert.ok(err, 'error properly handled'); + assert.equal(err.message, 'Error: Error opening dataset', 'expected error message'); + rimraf(outdir, function(err) { + assert.end(err); + }); + }); + }); +}); + +test('[GPX togeojson] convert and index valid GPX', function(assert) { + var infile = path.resolve(__dirname, 'fixtures', 'gpx', 'ok-valid-file.gpx'); + togeojson.index_worthy_size = 100; // 100 bytes + + tmpdir(function(err, outdir) { + togeojson(infile, outdir, function(err) { + if (err) throw err; + assert.ifError(err, 'no error'); + assert.ok(fs.existsSync(path.join(outdir, 'tracks.geojson')), 'converted layer'); + assert.ok(fs.existsSync(path.join(outdir, 'tracks.geojson.index')), 'created index'); + assert.ok(fs.existsSync(path.join(outdir, 'metadata.json')), 'added metadata of original gpx'); + rimraf(outdir, function(err) { + assert.end(err); + }); + }); + }); +}); + diff --git a/test/togeojson-kml.test.js b/test/togeojson-kml.test.js new file mode 100644 index 0000000..ca8f6de --- /dev/null +++ b/test/togeojson-kml.test.js @@ -0,0 +1,131 @@ +var test = require('tape'); +var path = require('path'); +var os = require('os'); +var fs = require('fs'); +var crypto = require('crypto'); +var mkdirp = require('mkdirp'); +var rimraf = require('rimraf'); +var exec = require('child_process').exec; +var togeojson = require('../preprocessors/togeojson-kml.preprocessor'); + +// Perform all preprocessorcery in a temporary directory. Otherwise output files +// with random names will litter the fixture directory +function tmpdir(callback) { + var dir = path.join(os.tmpdir(), crypto.randomBytes(8).toString('hex')); + var fixtureDir = path.resolve(__dirname, 'fixtures', 'kml'); + var copy_cmd = 'cp -r'; + + if (process.platform === 'win32') { + //during tests %PATH% env var is completely stripped. add Windows system dir back to get xcopy + process.env.Path += ';' + path.join(process.env.windir, 'system32'); + copy_cmd = 'xcopy /s /y'; + dir += '\\'; + var cmd = [copy_cmd, fixtureDir, dir].join(' '); + exec(cmd, function(err) { + if (err) return callback(err); + callback(null, dir); + }); + } + else { + mkdirp(dir, function(err) { + if (err) return callback(err); + callback(null, dir); + }); + } +} + +test('[KML togeojson] fails duplicate layer names', function(assert) { + var infile = path.resolve(__dirname, 'fixtures', 'kml', 'fail-duplicate-layer-names.kml'); + + tmpdir(function(err, outdir) { + togeojson(infile, outdir, function(err) { + assert.ok(err, 'error properly handled'); + assert.equal(err.message, 'Duplicate layer names! \'duplicate layer name\' found 2 times, \'layer 2\' found 2 times', 'expected error message'); + rimraf(outdir, function(err) { + assert.end(err); + }); + }); + }); +}); + +test('[KML togeojson] fails empty features only', function(assert) { + var infile = path.resolve(__dirname, 'fixtures', 'kml', 'fail-empty-features-only.kml'); + + tmpdir(function(err, outdir) { + togeojson(infile, outdir, function(err) { + assert.ok(err, 'error properly handled'); + assert.equal(err.message, 'KML does not contain any valid features', 'expected error message'); + rimraf(outdir, function(err) { + assert.end(err); + }); + }); + }); +}); + +test('[KML togeojson] fails empty features only', function(assert) { + var infile = path.resolve(__dirname, 'fixtures', 'kml', 'fail-invalid.kml'); + + tmpdir(function(err, outdir) { + togeojson(infile, outdir, function(err) { + assert.ok(err, 'error properly handled'); + assert.equal(err.message, 'Error: Error opening dataset', 'expected error message'); + rimraf(outdir, function(err) { + assert.end(err); + }); + }); + }); +}); + +test('[KML togeojson] fails empty features only', function(assert) { + var infile = path.resolve(__dirname, 'fixtures', 'kml', 'fail-more-than-15-layers.kml'); + + tmpdir(function(err, outdir) { + togeojson(infile, outdir, function(err) { + assert.ok(err, 'error properly handled'); + assert.equal(err.message, '22 layers found. Maximum of 15 layers allowed.', 'expected error message'); + rimraf(outdir, function(err) { + assert.end(err); + }); + }); + }); +}); + +test('[KML togeojson] convert and index valid kml', function(assert) { + var infile = path.resolve(__dirname, 'fixtures', 'kml', 'ok-layers-folders-emptygeometries.kml'); + togeojson.index_worthy_size = 100; // 100 bytes + + tmpdir(function(err, outdir) { + togeojson(infile, outdir, function(err) { + assert.ifError(err, 'no error'); + assert.ok(fs.existsSync(path.join(outdir, 'folder-01-01.geojson')), 'converted layer'); + assert.ok(fs.existsSync(path.join(outdir, 'folder-01-01.geojson.index')), 'created index'); + assert.ok(fs.existsSync(path.join(outdir, 'folder-01.geojson')), 'converted layer'); + assert.ok(fs.existsSync(path.join(outdir, 'folder-01.geojson.index')), 'created index'); + assert.ok(fs.existsSync(path.join(outdir, 'folder-02.geojson')), 'converted layer'); + assert.ok(fs.existsSync(path.join(outdir, 'folder-02.geojson.index')), 'created index'); + assert.ok(fs.existsSync(path.join(outdir, 'my-test.geojson')), 'converted layer'); + assert.ok(fs.existsSync(path.join(outdir, 'my-test.geojson.index')), 'created index'); + assert.ok(fs.existsSync(path.join(outdir, 'metadata.json')), 'added metadata of original kml'); + rimraf(outdir, function(err) { + assert.end(err); + }); + }); + }); +}); + +test('[KML togeojson] handle layers with special characters', function(assert) { + var infile = path.resolve(__dirname, 'fixtures', 'kml', 'ok-special-character-layer.kml'); + togeojson.index_worthy_size = 100; // 100 bytes + + tmpdir(function(err, outdir) { + togeojson(infile, outdir, function(err) { + assert.ifError(err, 'no error'); + assert.ok(fs.existsSync(path.join(outdir, 'special_characters_in___this_layer.geojson')), 'converted layer'); + assert.ok(fs.existsSync(path.join(outdir, 'special_characters_in___this_layer.geojson.index')), 'created index'); + assert.ok(fs.existsSync(path.join(outdir, 'metadata.json')), 'added metadata of original kml'); + rimraf(outdir, function(err) { + assert.end(err); + }); + }); + }); +});