[Pkg-javascript-commits] [node-millstone] 09/10: Imported Upstream version 0.6.8
Jérémy Lal
kapouer at alioth.debian.org
Fri Oct 25 20:15:32 UTC 2013
This is an automated email from the git hooks/post-receive script.
kapouer pushed a commit to branch master
in repository node-millstone.
commit aed56fff5eb00284bd74746b9a9e47b8e56105a9
Author: Jérémy Lal <kapouer at melix.org>
Date: Fri Oct 25 22:14:51 2013 +0200
Imported Upstream version 0.6.8
---
.npmignore => .gitignore | 0
.npmignore | 3 +-
.travis.yml | 27 +
CHANGELOG.md | 153 ++++
README.md | 3 +
bin/millstone | 38 +
lib/millstone.js | 933 +++++++++++++++-----
lib/util.js | 159 +++-
package.json | 33 +-
test/UPPERCASE_EXT/project.mml | 13 +
test/UPPERCASE_EXT/style.mss | 1 +
test/UPPERCASE_EXT/test1.CSV | 8 +
test/cache/cache.mml | 12 +-
test/corrupt-zip.test.js | 57 ++
test/corrupt-zip/project.mml | 13 +
test/corrupt-zip/style.mss | 1 +
test/data/9368bdd9-zip_no_ext/.9368bdd9-zip_no_ext | 1 +
test/data/9368bdd9-zip_no_ext/9368bdd9-zip_no_ext | Bin 0 -> 1162 bytes
...e_10m_admin_0_boundary_lines_disputed_areas.zip | Bin 0 -> 49525 bytes
test/data/snow-cover.tif | Bin 0 -> 2876 bytes
test/error.test.js | 64 ++
test/image-noext.test.js | 39 +
test/image-noext/project.mml | 14 +
test/image-noext/style.mss | 1 +
test/invalid-json/broken.json | 8 +
test/invalid-json/project.mml | 13 +
test/invalid-json/style.mss | 1 +
test/macosx-zipped.test.js | 41 +
test/macosx-zipped/project.mml | 13 +
test/macosx-zipped/style.mss | 1 +
test/markers.test.js | 73 ++
test/markers/layers/points.csv | 2 +
test/markers/project.mml | 14 +
test/markers/style.mss | 2 +
test/missing-file-absolute/project.mml | 13 +
test/missing-file-absolute/style.mss | 1 +
test/missing-file-relative/project.mml | 13 +
test/missing-file-relative/style.mss | 1 +
test/multi-shape-zip.test.js | 48 +
test/multi-shape-zip/project.mml | 11 +
test/nosymlink.test.js | 102 +++
test/nosymlink/points.csv | 2 +
test/nosymlink/points.dbf | Bin 0 -> 33 bytes
test/nosymlink/points.prj | 1 +
test/nosymlink/points.shp | Bin 0 -> 100 bytes
test/nosymlink/points.shx | Bin 0 -> 100 bytes
test/nosymlink/points.vrt | 9 +
test/nosymlink/project.mml | 37 +
test/nosymlink/pshape.zip | Bin 0 -> 759 bytes
test/nosymlink/style.mss | 1 +
test/raster-linking.test.js | 48 +
test/raster-linking/project.mml | 12 +
test/support.js | 28 +
test/test.js | 273 ++++--
test/uppercase-ext.test.js | 49 +
test/zip-with-shapefile-and-txt.test.js | 38 +
test/zip-with-shapefile-and-txt/project.mml | 11 +
test/zipped-json/project.mml | 13 +
test/zipped-json/style.mss | 1 +
59 files changed, 2129 insertions(+), 324 deletions(-)
diff --git a/.npmignore b/.gitignore
similarity index 100%
copy from .npmignore
copy to .gitignore
diff --git a/.npmignore b/.npmignore
index 91dfed8..246583f 100644
--- a/.npmignore
+++ b/.npmignore
@@ -1,2 +1,3 @@
.DS_Store
-node_modules
\ No newline at end of file
+node_modules
+./test
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..f06df33
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,27 @@
+language: node_js
+
+node_js:
+ - "0.10"
+ - "0.8"
+ - "0.6"
+
+before_install:
+ - sudo apt-get -qq update
+
+install:
+ - npm install
+
+before_script:
+ - npm test
+
+script:
+ - rm -rf ./node_modules
+ - sudo apt-get -qq install libgdal1-dev
+ - npm install --shared_gdal
+ - npm test
+ - sudo apt-add-repository --yes ppa:ubuntugis/ubuntugis-unstable
+ - sudo apt-get -qq update
+ - sudo apt-get -qq install libgdal1-dev
+ - rm -rf ./node_modules
+ - npm install --shared_gdal
+ - npm test
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..af2b6f3
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,153 @@
+## CHANGELOG
+
+#### 0.6.8
+
+* Fixed clearing of download tracker internal allowing carto to exit cleanly from using millstone from the command line
+
+#### 0.6.7
+
+* Upgraded to latest node-srs at 0.3.6 and node-zipfile at 0.4.2
+
+#### 0.6.6
+
+* Skipped accidentally
+
+#### 0.6.5
+
+* Upgraded to latest node-srs at 0.3.3
+
+#### 0.6.4
+
+* Fixed bug causing zip files to be uncompressed even when they had already been uncompressed
+
+#### 0.6.3
+
+* Added `millstone.drainPool` function to forcefully drain the downloads pool
+
+#### 0.6.2
+
+* Fixed reading for metafiles
+
+#### 0.6.1
+
+* Always honour filepath option in .download #105 (strk)
+
+#### 0.6.0
+
+* Supports node v0.10.x
+* Upgraded various deps: request, underscore, mime, generic-pool, optimist
+* Upgraded to node-srs at 0.3.0
+* Increased the download pool size from 5 to 10 and fixed up release logic
+* Various fixes to zipfile handling and upgrade of node-zipfile to v0.4.0
+
+#### 0.5.15
+
+* Added better error output when millstone is unable to detect the appropriate mapnik datasource
+ for a file based datasources
+* Added support for attempting to re-download zip archives that are cached but cannot be opened
+ (handles partial downloads that may have failed due to network failure)
+
+#### 0.5.14
+
+* Fixed detections and handling of files with upper or mixed case extensions.
+
+#### 0.5.13
+
+* Fixed a bug where millstone would hang if an absolute path to a shapefile was used and that
+ shapefile did not exist at that path.
+
+#### 0.5.12
+
+* Added command line millstone tool
+* Added support for reading any supported file in .zip archives
+* Better error messages if broken symlinks are encountered
+* Support added for downloading images at urls without clear image file extensions
+* Fixed handling of hidden files in zip archives
+* Switched to using console.error for log output
+
+#### 0.5.11
+
+* Will now throw if files do not exist (instead of throwing on missing/unknown srs)
+
+* Fixed support for loading layer datasource files from alternative windows drives
+
+* Moved to no-symlink/no-copy behavior on all windows versions
+
+* Updated node-srs version
+
+* Improved handling of known file extensions to better support guessing extensions via headers
+
+* Fixed handling of sqlite attach with absolute paths
+
+#### 0.5.10
+
+* Fixed missing error handling when localizing Carto URIs
+
+#### 0.5.9
+
+* Improved uri regex methods for carto urls - #68, #69, #70, #72, and #73
+
+* Use copy fallback on Windows platforms supporting symlinks but where the user does not have the symlink 'right' (#71)
+
+* Restored Node v0.4.x support
+
+#### 0.5.8
+
+* Improved uri regex methods for carto urls - amends #63
+
+#### 0.5.7
+
+* Fixed handling of multiple non-unique carto urls in the same stylesheet (#63)
+
+#### 0.5.6
+
+* Fixed extension handling for urls without an extension
+
+* Moved to streaming copy of data when in copy mode to avoid too much memory usage
+
+* Fixed race condition when localizing imag/svg icons in styles like point-file and marker-file.
+
+* Exposed the global downloads object so calling applications can see how many downloads millstone is currently handling
+
+* Removed node v0.8.x deprecation warnings
+
+* Added more agressive re-copying of data when it is out of date and millstone is in copy mode (win XP)
+
+* Moved to processing shapefile parts instead of the directory
+
+#### 0.5.5
+
+* Added a verbose mode that can be trigged by setting NODE_ENV = 'development'
+
+* Switched to request (dropped node-get) for better proxy support
+
+* Support for making relative the paths stored to the download cache
+
+* Support for zipfiles with no extension
+
+* Advertise node v8 support
+
+#### 0.5.4
+
+* Fixes to better support localization of carto resources
+
+#### 0.5.3
+
+* Updated node-get min version in order to fully support proxy auth
+* Improved cross-platform relative path detection
+
+#### 0.5.2
+
+* Improved regex used to detect content-disposition
+
+* Support for localizing uri's in stylesheet
+
+#### 0.5.1
+
+* Moved to mocha for tests
+
+* Made `nosymlink` option optional
+
+#### 0.5.0
+
+* Add `nosymlink` option for not downloading files
diff --git a/README.md b/README.md
index 3f43c8c..534a24a 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,8 @@
# millstone
+[](http://travis-ci.org/mapbox/millstone)
+[](https://david-dm.org/mapbox/millstone)
+
As of [carto 0.2.0](https://github.com/mapbox/carto), the Carto module expects
all datasources and resources to be localized - remote references like
URLs and providers are not downloaded when maps are rendered.
diff --git a/bin/millstone b/bin/millstone
new file mode 100755
index 0000000..aea856e
--- /dev/null
+++ b/bin/millstone
@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+
+var path = require('path');
+var fs = require('fs');
+var millstone = require('millstone');
+var optimist = require('optimist')
+ .usage("Usage: $0 <source MML file>")
+ .options('h', {alias:'help', describe:'Display this help message'})
+ .options('n', {alias:'nosymlink', boolean:true, describe:'Use unmodified paths instead of symlinking files'});
+
+var options = optimist.argv;
+
+if (options.help) {
+ optimist.showHelp();
+ process.exit(0);
+}
+
+var input = options._[0];
+
+if (!input) {
+ console.log('millstone: please provide a path to an mml file');
+ process.exit(1);
+}
+var mml = JSON.parse(fs.readFileSync(input));
+
+var options = {
+ mml: mml,
+ base: path.dirname(input),
+ cache: '/tmp/millstone-test',
+ nosymlink: options.nosymlink
+};
+
+millstone.resolve(options, function(err, resolved) {
+ if (err) throw err;
+ console.log(JSON.stringify(resolved,null,4));
+ // force exit to avoid wait for generic-pool to empty
+ process.exit(0);
+});
\ No newline at end of file
diff --git a/lib/millstone.js b/lib/millstone.js
index fb78d54..a968c23 100644
--- a/lib/millstone.js
+++ b/lib/millstone.js
@@ -4,16 +4,63 @@ var url = require('url');
var crypto = require('crypto');
var EventEmitter = require('events').EventEmitter;
var mime = require('mime');
+var mkdirp = require('mkdirp');
+
+var existsAsync = require('fs').exists || require('path').exists;
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+var env = process.env.NODE_ENV || 'development';
// Third party modules
var _ = require('underscore'),
srs = require('srs'),
- get = require('get'),
+ request = require('request'),
zipfile = require('zipfile'),
Step = require('step'),
utils = require('./util.js');
-var path_sep = process.platform === 'win32' ? '\\' : '/';
+
+// mapping of known file extensions
+// to mapnik datasource plugin name
+var valid_ds_extensions = {
+ '.shp':'shape',
+ '.csv':'csv',
+ '.tsv':'csv',
+ '.txt':'csv',
+ '.geotiff':'gdal',
+ '.geotif':'gdal',
+ '.tif':'gdal',
+ '.tiff':'gdal',
+ '.vrt':'gdal',
+ '.geojson':'ogr',
+ '.json':'ogr',
+ '.gml':'ogr',
+ '.osm':'osm',
+ '.kml':'ogr',
+ '.rss':'ogr',
+ '.gpx':'ogr',
+ '.db':'sqlite',
+ '.sqlite3':'sqlite',
+ '.sqlite':'sqlite',
+ '.spatialite':'sqlite'
+};
+
+// marker types readible by mapnik
+var valid_marker_extensions = {
+ '.svg':'svg',
+ '.png':'image',
+ '.tif':'image',
+ '.tiff':'image',
+ '.jpeg':'image',
+ '.jpg':'image'
+};
+
+var file_linking_method = utils.forcelink;
+
+var never_link = false;
+if (process.platform === 'win32') {
+ never_link = true;
+}
// Known SRS values
var SRS = {
@@ -24,6 +71,11 @@ var SRS = {
// on object of locks for concurrent downloads
var downloads = {};
+// object for tracking logging on downloads
+var download_log_interval = true;
+// last download count, in order to limit logging barf
+var download_last_count = 0;
+
var pool = require('generic-pool').Pool({
create: function(callback) {
callback(null, {});
@@ -31,59 +83,139 @@ var pool = require('generic-pool').Pool({
destroy: function(obj) {
obj = undefined;
},
- max: 5
+ max: 10
});
-function download(url, filepath, callback) {
- var dl = filepath + '.download';
+
+function download(url, options, callback) {
+ if (env == 'development') {
+ download_log_interval = setInterval(function() {
+ var in_use = Object.keys(downloads);
+ if (in_use.length > 0 && (download_last_count !== in_use.length)) {
+ var msg = "[millstone] currently downloading " + in_use.length + ' file';
+ if (in_use.length > 1) {
+ msg += 's';
+ }
+ msg += ': ' + _(in_use).map(function(f) { return path.basename(f); });
+ console.error(msg);
+ }
+ download_last_count = in_use.length;
+ },5000);
+ } else {
+ clearInterval(download_log_interval);
+ }
+ // https://github.com/mapbox/millstone/issues/39
+ url = unescape(url);
+ var dl = options.filepath + '.download';
// If this file is already being downloaded, attach the callback
// to the existing EventEmitter
- if (downloads[url]) {
- return downloads[url].once('done', callback);
+ var dkey = options.filepath + '|' + url;
+ if (downloads[dkey]) {
+ return downloads[dkey].once('done', callback);
} else {
- downloads[url] = new EventEmitter();
- pool.acquire(function(obj) {
- pool.release(obj);
- (new get(url)).toDisk(dl, function(err, file, response, g) {
- if (err) {
- downloads[url].emit('done', err);
- delete downloads[url];
- return callback(err);
+ pool.acquire(function(err, obj) {
+ var make_message = function() {
+ var msg = "Unable to download '" + url + "'";
+ if (options.name)
+ msg += " for '" + options.name + "'";
+ return msg;
+ };
+ var return_on_error = function(err) {
+ downloads[dkey].emit('done', err);
+ delete downloads[dkey];
+ err.message = make_message() + " ("+err.message+")";
+ pool.release(obj);
+ return callback(err);
+ }
+ downloads[dkey] = new EventEmitter();
+ if (err) {
+ return return_on_error(err);
+ } else {
+ if (env == 'development') console.error("[millstone] downloading: '" + url + "'");
+ var req;
+ try {
+ req = request({
+ url: url,
+ proxy: process.env.HTTP_PROXY
+ });
+ } catch (err) {
+ // catch Invalid URI error
+ return return_on_error(err);
}
- fs.rename(dl, filepath, function(err) {
- // We store the headers from the download in a hidden file
- // alongside the data for future reference. Currently, we
- // only use the `content-disposition` header to determine
- // what kind of file we downloaded if it doesn't have an
- // extension.
- fs.writeFile(metapath(filepath), JSON.stringify(response.headers), 'utf-8', function(err) {
- downloads[url].emit('done', err, filepath);
- delete downloads[url];
- return callback(err, filepath);
- });
+ req.on('error', function(err) {
+ return return_on_error(err);
});
- });
+ req.pipe(fs.createWriteStream(dl)).on('error', function(err) {
+ return return_on_error(err);
+ }).on('close', function() {
+ if (!req.response || (req.response && req.response.statusCode >= 400)) {
+ return return_on_error(new Error('server returned ' + req.response.statusCode));
+ } else {
+ pool.release(obj);
+ fs.rename(dl, options.filepath, function(err) {
+ if (err) {
+ downloads[dkey].emit('done', err);
+ delete downloads[dkey];
+ return callback(err);
+ } else {
+ if (env == 'development') console.error("[millstone] finished downloading '" + options.filepath + "'");
+ // We store the headers from the download in a hidden file
+ // alongside the data for future reference. Currently, we
+ // only use the `content-disposition` header to determine
+ // what kind of file we downloaded if it doesn't have an
+ // extension.
+ var req_meta = _(req.req.res.headers).clone();
+ if (req.req.res.request && req.req.res.request.path) {
+ req_meta.path = req.req.res.request.path;
+ }
+ fs.writeFile(metapath(options.filepath), JSON.stringify(req_meta), 'utf-8', function(err) {
+ downloads[dkey].emit('done', err, options.filepath);
+ delete downloads[dkey];
+ return callback(err, options.filepath);
+ });
+ }
+ });
+ }
+ });
+ }
});
}
}
// Retrieve a remote copy of a resource only if we don't already have it.
-function localize(url, filepath, next) {
- path.exists(filepath, function(exists) {
+function localize(url, options, callback) {
+ existsAsync(options.filepath, function(exists) {
if (exists) {
- next(null, filepath);
- } else {
- utils.mkdirP(path.dirname(filepath), 0755, function(err) {
- if (err && err.code !== 'EEXIST') {
- next(err);
- } else {
- download(url, filepath, function(err, filepath) {
- if (err) return next(err);
- next(null, filepath);
- });
+ var re_download = false;
+ // unideal workaround for frequently corrupt/partially downloaded zips
+ // https://github.com/mapbox/millstone/issues/85
+ if (path.extname(options.filepath) == '.zip') {
+ try {
+ var zf = new zipfile.ZipFile(options.filepath);
+ if (zf.names.length < 1) {
+ throw new Error("could not find any valid data in zip archive: '" + options.filepath + "'");
+ }
+ } catch (e) {
+ if (env == 'development') console.error('[millstone] could not open zip archive: "' + options.filepath + '" attempting to re-download from "'+url+"'");
+ re_download = true;
}
- });
+ }
+ if (!re_download) {
+ return callback(null, options.filepath);
+ }
}
+ var dir_path = path.dirname(options.filepath);
+ mkdirp(dir_path, 0755, function(err) {
+ if (err && err.code !== 'EEXIST') {
+ if (env == 'development') console.error('[millstone] could not create directory: ' + dir_path);
+ callback(err);
+ } else {
+ download(url, options, function(err, filepath) {
+ if (err) return callback(err);
+ callback(null, filepath);
+ });
+ }
+ });
});
}
@@ -99,7 +231,7 @@ function cachepath(location) {
.substr(0,8) +
'-' + path.basename(uri.pathname, path.extname(uri.pathname));
var extname = path.extname(uri.pathname);
- return _(['.shp', '.zip']).include(extname.toLowerCase()) ?
+ return _(['.shp', '.zip', '']).include(extname.toLowerCase()) ?
path.join(hash, hash + extname)
: path.join(hash + extname);
}
@@ -110,27 +242,123 @@ function metapath(filepath) {
return path.join(path.dirname(filepath), '.' + path.basename(filepath));
}
+function add_item_to_metafile(metafile,key,item,callback) {
+ existsAsync(metafile, function(exists) {
+ if (exists) {
+ fs.readFile(metafile, 'utf-8', function(err, data) {
+ if (err) return callback(err);
+ var meta;
+ try {
+ meta = JSON.parse(data);
+ } catch (err) {
+ return callback(err);
+ }
+ meta[key] = item;
+ fs.writeFile(metafile, JSON.stringify(meta), 'utf-8', function(err) {
+ if (err) return callback(err);
+ return callback(null);
+ });
+ });
+ } else {
+ var data = {};
+ data[key] = item;
+ fs.writeFile(metafile, JSON.stringify(data), 'utf-8', function(err) {
+ if (err) return callback(err);
+ return callback(null);
+ });
+ }
+ });
+}
+
+function isRelative(loc) {
+ if (process.platform === 'win32') {
+ return loc[0] !== '\\' && loc[0] !== '/' && loc.match(/^[a-zA-Z]:/) === null;
+ } else {
+ return loc[0] !== '/';
+ }
+}
+
+function isValidExt(ext) {
+ if (ext) {
+ var lower_ext = ext.toLowerCase();
+ return lower_ext == '.zip' || valid_marker_extensions[lower_ext] || valid_ds_extensions[lower_ext];
+ }
+ return false;
+}
+
function guessExtension(headers) {
+ if (headers.path) {
+ var ext = path.extname(headers.path);
+ if (isValidExt(ext)) {
+ return ext;
+ }
+ }
if (headers['content-disposition']) {
- // Taken from node-get
- var match = headers['content-disposition'].match(/filename=['"]?([^'";]+)['"]?/);
+ var match = headers['content-disposition'].match(/filename=['"](.*)['"]$/);
+ if (!match) {
+ match = headers['content-disposition'].match(/filename=['"]?([^'";]+)['"]?/);
+ }
if (match) {
var ext = path.extname(match[1]);
- if (ext) {
+ if (isValidExt(ext)) {
return ext;
}
}
- } else if (headers['content-type']) {
- var ext = mime.extension(headers['content-type'].split(';')[0]);
- if (ext) {
- return '.' + ext;
+ }
+ if (headers['content-type']) {
+ if (headers['content-type'].indexOf('subtype=gml') != -1) {
+ return '.gml'; // avoid .xml being detected for gml
+ }
+ var ext_no_dot = mime.extension(headers['content-type'].split(';')[0]);
+ var ext = '.'+ext_no_dot;
+ if (isValidExt(ext)) {
+ return ext;
}
}
- return false;
-};
+ return '';
+}
+
+// Read headers and guess extension
+function readExtension(file, cb) {
+ fs.readFile(metapath(file), 'utf-8', function(err, data) {
+ if (err) {
+ if (err.code === 'ENOENT') return cb(new Error('Metadata file does not exist.'));
+ return cb(err);
+ }
+ try {
+ var ext = guessExtension(JSON.parse(data));
+ if (ext) {
+ if (env == 'development') console.error("[millstone] detected extension of '" + ext + "' for '" + file + "'");
+ }
+ return cb(null, ext);
+ } catch (e) {
+ return cb(e);
+ }
+ });
+}
// Unzip function, geared specifically toward unpacking a shapefile.
function unzip(file, callback) {
+ var metafile = metapath(file);
+ // return cached result of unzipped file
+ // intentionally sync here to avoid race condition unzipping
+ // same file for multiple layers
+ try {
+ var meta_data = fs.readFileSync(metafile);
+ if (meta_data) {
+ var meta = JSON.parse(meta_data);
+ var dest_file = meta['unzipped_file'];
+ if (dest_file && existsSync(dest_file)) {
+ if (env == 'development') console.error('[millstone] found previously unzipped file: ' + dest_file);
+ return callback(null,dest_file);
+ }
+ } else {
+ if (env == 'development') console.error('[millstone] empty meta file for zipfile: ' + metafile);
+ }
+ }
+ catch (err) {
+ if (env == 'development') console.error('[millstone] error opening meta file for zipfile: ' + err.message);
+ }
var zf;
try {
zf = new zipfile.ZipFile(file);
@@ -138,50 +366,99 @@ function unzip(file, callback) {
return callback(err);
}
- var remaining = zf.names.length;
- var shp = _(zf.names).chain()
+ if (zf.names.length < 1) {
+ return callback(new Error("could not find any valid data in zip archive: '" + file + "'"));
+ }
+
+ var remain = zf.names.length;
+ var ds_files = _(zf.names).chain()
+ .reject(function(name) {
+ return (name && (name[0] === '.' || path.basename(name)[0] === '.'));
+ })
.map(function(name) {
- if (path.extname(name).toLowerCase() !== '.shp') return;
- return path.join(
+ if (!valid_ds_extensions[path.extname(name).toLowerCase()]) return;
+ var new_name = path.join(
path.dirname(file),
path.basename(file, path.extname(file)) +
- path.extname(name).toLowerCase()
- );
+ path.extname(name).toLowerCase());
+ return {new_name:new_name,original_name:name};
})
+ .uniq()
.compact()
- .first()
.value();
- if (!shp) return callback(new Error('Shapefile not found in zip ' + file));
+ if (!ds_files || ds_files.length < 1) return callback(new Error("Valid datasource not detected (by extension) in zip: '" + file + "'"));
+
+ var original_name = ds_files[0].original_name;
+ var new_name = ds_files[0].new_name;
+
+ var len = Object.keys(ds_files).length;
+ if (len > 1) {
+ // prefer first .shp
+ for (var i=0;i<len;++i) {
+ var fname = ds_files[i].original_name;
+ if (path.extname(fname) == '.shp') {
+ original_name = fname;
+ new_name = ds_files[i].new_name;
+ break;
+ }
+ }
+ if (env == 'development') {
+ console.warn('[millstone] warning: detected more than one file in zip (by ext) that may be valid: ');
+ for (var i=0;i<len;++i) {
+ console.warn('[millstone] ' + ds_files[i].original_name);
+ }
+ console.warn('[millstone] picked: ' + original_name);
+ }
+ }
+
+ if (!new_name) return callback(new Error("Valid datasource not detected (by extension) in zip: '" + file + "'"));
+
+ if (env == 'development') {
+ console.warn('[millstone] renaming ' + original_name + ' to ' + new_name);
+ }
+ // only unzip files that match our target name
+ // naive '(name.indexOf(search_basename) < 0)' does the trick
+ // yes this is simplistic, but its better than corrupt data: https://github.com/mapbox/millstone/issues/99
+ var search_basename = path.basename(original_name,path.extname(original_name));
zf.names.forEach(function(name) {
// Skip directories, hiddens.
- if (!path.extname(name) || name[0] === '.') {
- remaining--;
- if (!remaining) callback(null, shp);
- }
- // We're brutal in our expectations -- don't support nested
- // directories, and rename any file from `arbitraryName.SHP`
- // to `[hash].shp`.
- var dest = path.join(
- path.dirname(file),
- path.basename(file, path.extname(file)) +
- path.extname(name).toLowerCase()
- );
- zf.readFile(name, function(err, buff) {
- if (err) return callback(err);
- fs.open(dest, 'w', 0644, function(err, fd) {
+ var basename = path.basename(name);
+ if (!path.extname(name) || (name.indexOf(search_basename) < 0) || name[0] === '.' || basename[0] === '.') {
+ remain--;
+ if (!remain) callback(null, new_name);
+ } else {
+ // We're brutal in our expectations -- don't support nested
+ // directories, and rename any file from `arbitraryName.SHP`
+ // to `[hash].shp`.
+ var dest = path.join(
+ path.dirname(file),
+ path.basename(file, path.extname(file)) +
+ path.extname(name).toLowerCase()
+ );
+ zf.readFile(name, function(err, buff) {
if (err) return callback(err);
- fs.write(fd, buff, 0, buff.length, null, function(err) {
+ fs.open(dest, 'w', 0644, function(err, fd) {
if (err) return callback(err);
- fs.close(fd, function(err) {
+ fs.write(fd, buff, 0, buff.length, null, function(err) {
if (err) return callback(err);
- remaining--;
- if (!remaining) callback(null, shp);
+ fs.close(fd, function(err) {
+ if (err) return callback(err);
+ remain--;
+ if (!remain) {
+ add_item_to_metafile(metafile,'unzipped_file',new_name,function(err) {
+ // ignore error from add_item_to_metafile
+ //if (err && env == 'development') console.error('[millstone] ' + err.message);
+ if (err) throw err;
+ callback(null, new_name);
+ });
+ }
+ });
});
});
});
- });
+ }
});
}
@@ -218,7 +495,7 @@ function fixSRS(obj) {
function checkTTL(cache, l) {
var file = l.Datasource.file;
- var ttl = l.Datasource.ttl
+ var ttl = l.Datasource.ttl;
if (!url.parse(file).protocol) return;
@@ -226,9 +503,9 @@ function checkTTL(cache, l) {
fs.stat(filepath, function(err, stats) {
if (err && err.code != 'ENOENT') return console.warn(err);
- var msttl = parseInt(ttl) * 1000;
+ var msttl = parseInt(ttl,10) * 1000;
if (err || Date.now() > (new Date(stats.mtime).getTime() + msttl)) {
- download(file, filepath, function(err, filepath){
+ download(file, {filepath:filepath,name:l.name}, function(err, filepath){
if (err) return console.warn(err);
});
}
@@ -242,14 +519,19 @@ function resolve(options, callback) {
if (!options.mml) return callback(new Error('options.mml is required'));
if (!options.base) return callback(new Error('options.base is required'));
if (!options.cache) return callback(new Error('options.cache is required'));
+ if (typeof options.nosymlink === "undefined") options.nosymlink = false;
+ // respect global no-symlinking preference on windows
+ if (never_link) options.nosymlink = true;
var mml = options.mml,
base = path.resolve(options.base),
cache = path.resolve(options.cache),
- resolved = JSON.parse(JSON.stringify(mml));
+ resolved = JSON.parse(JSON.stringify(mml)),
+ nosymlink = options.nosymlink;
Step(function setup() {
- utils.mkdirP(path.join(base, 'layers'), 0755, this);
+ if (nosymlink) mkdirp(base, 0755, this);
+ else mkdirp(path.join(base, 'layers'), 0755, this);
}, function externals(err) {
if (err && err.code !== 'EEXIST') throw err;
@@ -258,30 +540,36 @@ function resolve(options, callback) {
var next = function(err) {
remaining--;
if (err && err.code !== 'EEXIST') error = err;
- if (!remaining) this(error);
+ if (remaining <= 0) this(error);
}.bind(this);
if (!remaining) return this();
resolved.Stylesheet.forEach(function(s, index) {
- if (typeof s !== 'string') return next();
+ if (typeof s !== 'string') {
+ if (env == 'development') console.error("[millstone] processing style '" + s.id + "'");
+ return localizeCartoURIs(s,next);
+ }
var uri = url.parse(s);
// URL, download.
if (uri.protocol && (uri.protocol == 'http:' || uri.protocol == 'https:')) {
- return (new get(s)).asBuffer(function(err, data) {
+ return request({
+ url: s,
+ proxy: process.env.HTTP_PROXY
+ }, function(err, response, data) {
if (err) return next(err);
resolved.Stylesheet[index] = {
id: path.basename(uri.pathname),
data: data.toString()
};
- next(err);
+ localizeCartoURIs(resolved.Stylesheet[index],next);
});
}
// File, read from disk.
- if (uri.pathname[0] !== path_sep) {
+ if (uri.pathname && isRelative(uri.pathname)) {
uri.pathname = path.join(base, uri.pathname);
}
fs.readFile(uri.pathname, 'utf8', function(err, data) {
@@ -291,79 +579,186 @@ function resolve(options, callback) {
id: s,
data: data
};
- next(err);
+ localizeCartoURIs(resolved.Stylesheet[index],next);
});
});
+ // Handle URIs within the Carto CSS
+ function localizeCartoURIs(s,cb) {
+
+ // Get all unique URIs in stylesheet
+ // TODO - avoid finding non url( uris?
+ var matches = s.data.match(/[-a-zA-Z0-9@:%_\+.~#?&//=]{2,256}\.[a-z]{2,4}\b(\/[-a-zA-Z0-9@:%_\+.~#?&//=]*)?/gi);
+ var URIs = _.uniq(matches || []);
+ var CartoURIs = [];
+ // remove any matched urls that are not clearly
+ // part of a carto style
+ // TODO - consider moving to carto so we can also avoid
+ // downloading commented out code
+ URIs.forEach(function(u) {
+ var idx = s.data.indexOf(u);
+ if (idx > -1) {
+ var pre_url = s.data.slice(idx-5,idx);
+ if (pre_url.indexOf('url(') > -1) {
+ CartoURIs.push(u);
+ // Update number of async calls so that we don't
+ // call this() too soon (before everything is done)
+ remaining += 1;
+ }
+ }
+ });
+
+ CartoURIs.forEach(function(u) {
+ var uri = url.parse(encodeURI(u));
+
+ // URL.
+ if (uri.protocol && (uri.protocol == 'http:' || uri.protocol == 'https:')) {
+ var filepath = path.join(cache, cachepath(u));
+ localize(uri.href, {filepath:filepath,name:s.id}, function(err, file) {
+ if (err) {
+ cb(err);
+ } else {
+ var extname = path.extname(file);
+ if (!extname) {
+ readExtension(file, function(error, ext) {
+ // note - we ignore any readExtension errors
+ if (ext) {
+ var new_filename = file + ext;
+ fs.rename(file, new_filename, function(err) {
+ s.data = s.data.split(u).join(new_filename);
+ cb(err);
+ });
+ } else {
+ s.data = s.data.split(u).join(file);
+ cb(err);
+ }
+ });
+ } else {
+ s.data = s.data.split(u).join(file);
+ cb(err);
+ }
+ }
+ });
+ } else {
+ cb();
+ }
+ });
+ cb();
+ }
+
resolved.Layer.forEach(function(l, index) {
if (!l.Datasource || !l.Datasource.file) return next();
+ if (env == 'development') console.error("[millstone] processing layer '" + l.name + "'");
// TODO find better home for this check.
if (l.Datasource.ttl) checkTTL(cache, l);
- var name = l.name || 'layer-' + index,
- uri = url.parse(encodeURI(l.Datasource.file)),
- pathname = decodeURI(uri.pathname),
- extname = path.extname(pathname);
-
+ var name = l.name || 'layer-' + index;
+ var uri = url.parse(encodeURI(l.Datasource.file));
+ var pathname = decodeURI(uri.pathname);
+ var extname = path.extname(pathname);
+ // unbreak pathname on windows if absolute path is used
+ // to an alternative drive like e:/
+ // https://github.com/mapbox/millstone/issues/81
+ if (process.platform === 'win32') {
+ if (uri.protocol && l.Datasource.file.slice(0,2).match(/^[a-zA-Z]:/)) {
+ pathname = l.Datasource.file;
+ }
+ }
// This function takes (egregious) advantage of scope;
// l, extname, and more is all up-one-level.
//
// `file`: filename to be symlinked in place to l.Datasource.file
- var symlink = function(file, cb) {
+ var processFile = function(file, cb) {
if (!file) return cb();
- switch (extname.toLowerCase()) {
- // Unzip and symlink to directory.
- case '.zip':
- l.Datasource.file =
- path.join(base,
- 'layers',
- name,
- path.basename(file, path.extname(file)) + '.shp');
- path.exists(l.Datasource.file, function(exists) {
- if (exists) return cb();
- unzip(file, function(err, file) {
- if (err) return cb(err);
- utils.forcelink(path.dirname(file),
- path.dirname(l.Datasource.file),
- cb);
- });
- });
- break;
- // Symlink directories
- case '.shp':
- l.Datasource.file =
- path.join(base, 'layers', name, path.basename(file));
- utils.forcelink(
- path.dirname(file),
- path.dirname(l.Datasource.file), cb);
- break;
- // Symlink files
- default:
- l.Datasource.file =
- path.join(base, 'layers', name + extname);
- utils.forcelink( file,
- l.Datasource.file,
- cb);
- break;
- }
+ readExtension(file, function(err, ext) {
+ // ignore errors from extension check
+ //if (err) console.error(err);
+
+ ext = ext || extname;
+
+ switch (ext.toLowerCase()) {
+ // Unzip and symlink to directory.
+ case '.zip':
+ if (nosymlink) {
+ unzip(file, function(err, file) {
+ if (err) return cb(err);
+ l.Datasource.file = file;
+ return cb();
+ });
+ } else {
+ unzip(file, function(err, file_found) {
+ if (err) return cb(err);
+ l.Datasource.file = path.join(base,
+ 'layers',
+ name,
+ path.basename(file_found));
+ return utils.processFiles(file_found,
+ l.Datasource.file,
+ file_linking_method,
+ {cache:cache}, cb);
+ });
+ }
+ break;
+ case '.shp':
+ if (nosymlink) {
+ l.Datasource.file = file;
+ return cb();
+ } else {
+ l.Datasource.file =
+ path.join(base, 'layers', name, path.basename(file));
+ return utils.processFiles(file, l.Datasource.file, file_linking_method, {cache:cache}, cb);
+ }
+ break;
+ default:
+ if (nosymlink) {
+ l.Datasource.file = file;
+ return cb();
+ } else {
+ l.Datasource.file =
+ path.join(base, 'layers', name + ext);
+ return file_linking_method(file, l.Datasource.file, {cache:cache}, cb);
+ }
+ break;
+ }
+ });
};
// URL.
if (uri.protocol && (uri.protocol == 'http:' || uri.protocol == 'https:')) {
var filepath = path.join(cache, cachepath(l.Datasource.file));
- localize(uri.href, filepath, function(err, file) {
+ localize(uri.href, {filepath:filepath,name:l.name}, function(err, file) {
if (err) return next(err);
- symlink(file, next)
+ processFile(file, next);
});
// Absolute path.
- } else if (pathname && pathname[0] === path_sep) {
- symlink(pathname, next);
+ } else if (pathname && !isRelative(pathname)) {
+ existsAsync(pathname, function(exists) {
+ if (!exists && nosymlink) {
+ // throw here before we try to symlink to avoid confusing error message
+ // we only throw here on nosymlink because a tarred/symlink resolved project
+ // may have locally resolved files that exist, see:
+ // https://github.com/mapbox/tilemill/issues/697#issuecomment-6813928
+ return next(new Error("File not found at absolute path: '" + pathname + "'"));
+ } else {
+ processFile(pathname, next);
+ }
+ });
// Local path.
} else {
- l.Datasource.file = path.resolve(path.join(base, pathname));
- next();
+ var local_pathname = path.resolve(path.join(base, pathname));
+ // NOTE : we do not call processFile here to avoid munging the name
+ if (path.extname(local_pathname) === '.zip') {
+ unzip(local_pathname, function(err, file) {
+ if (err) return next(err);
+ l.Datasource.file = file;
+ return next();
+ });
+ } else {
+ l.Datasource.file = local_pathname;
+ return next();
+ }
}
});
}, function processSql(err) {
@@ -389,34 +784,44 @@ function resolve(options, callback) {
var extname = path.extname(pathname);
var alias = dbs[i].split('@').shift();
var name = (l.name || 'layer-' + index) + '-attach-' + alias;
- var index = i;
+ var db_index = i;
- var symlink = function(filepath, cb) {
+ var symlink_db = function(filepath, cb) {
var filename = path.join(base, 'layers', name + extname);
- dbs[index] = alias + '@' + filename;
- utils.forcelink(filepath, filename, cb);
+ dbs[db_index] = alias + '@' + filename;
+ file_linking_method(filepath, filename, {cache:cache}, cb);
};
// URL.
if (file.protocol) {
var filepath = path.join(cache, cachepath(file.href));
- localize(file.href, filepath, function(err) {
+ localize(file.href, {filepath:filepath,name:name}, function(err) {
if (err) return next(err);
- symlink(filepath, next);
+ if (nosymlink) {
+ dbs[db_index] = alias + '@' + filepath;
+ next();
+ } else {
+ symlink_db(filepath, next);
+ }
});
}
// Absolute path.
- else if (pathname[0] === path_sep) {
- symlink(pathname, next);
+ else if (pathname && !isRelative(pathname)) {
+ if (nosymlink) {
+ dbs[db_index] = alias + '@' + pathname;
+ next();
+ } else {
+ symlink_db(pathname, next);
+ }
}
// Local path.
else {
- dbs[index] = alias + '@' + path.join(base, pathname);
+ dbs[db_index] = alias + '@' + path.join(base, pathname);
next();
}
})(group());
}, function(err) {
- if (err) throw err;
+ if (err) return next(err);
d.attachdb = dbs.join(',');
return next(err);
});
@@ -429,102 +834,132 @@ function resolve(options, callback) {
resolved.Layer.forEach(function(l, index) {
var d = l.Datasource;
var next = group();
-
- Step(function() {
- var ext = path.extname(d.file);
- var next = this;
- if (ext) {
- next(null, ext);
+ if (!d.file) return next();
+ existsAsync(d.file, function(exists) {
+ if (!exists) {
+ // https://github.com/mapbox/tilemill/issues/1808
+ // on OS X broken symlinks can be read and resolved but actually
+ // do not "exist" so here we try to resolve in order to avoid
+ // providing a confusing error that says a file does not exist
+ // when it actually does (and is just a broken link)
+ fs.readlink(d.file,function(err,resolvedPath){
+ if (resolvedPath) {
+ return next(new Error("File not found: '" +
+ resolvedPath + "' (broken symlink: '" +
+ d.file + "')"));
+ } else {
+ return next(new Error("File not found: '" + d.file + "'"));
+ }
+ });
} else {
- // This file doesn't have an extension, so we look for a
- // hidden metadata file that will contain headers for the
- // original HTTP request. We looks at the
- // `content-disposition` header to determine the extension.
- fs.readlink(l.Datasource.file, function(err, resolvedPath) {
- var metafile = metapath(resolvedPath);
- path.exists(metafile , function(exists) {
- if (!exists) return next(new Error('Metadata file does not exist.'));
- fs.readFile(metafile, 'utf-8', function(err, data) {
- if (err) return next(err);
+ Step(function() {
+ var ext = path.extname(d.file);
+ var next = this;
+ if (ext) {
+ next(null, ext);
+ } else {
+ // This file doesn't have an extension, so we look for a
+ // hidden metadata file that will contain headers for the
+ // original HTTP request. We look at the
+ // `content-disposition` header to determine the extension.
+ fs.lstat(l.Datasource.file, function(err, stats) {
+ if (err && err.code != 'ENOENT') {
+ next(err);
+ } else {
+ if (!stats.isSymbolicLink()) {
+ readExtension(l.Datasource.file, next);
+ } else {
+ fs.readlink(l.Datasource.file, function(err, resolvedPath) {
+ if (resolvedPath && isRelative(resolvedPath)) {
+ resolvedPath = path.join(path.dirname(l.Datasource.file), resolvedPath);
+ }
+ readExtension(resolvedPath, next);
+ });
+ }
+ }
+ });
+ }
+ }, function(err, ext) {
+ // Ignore errors during extension checks above and let a
+ // missing extension fall through to a missing `type`.
+ var name = l.name || 'layer-' + index;
+ ext = ext || path.extname(d.file);
+ d.type = d.type || valid_ds_extensions[ext.toLowerCase()];
+ switch (ext.toLowerCase()) {
+ case '.vrt':
+ // we default to assuming gdal raster for vrt's
+ // but we need to support OGRVRTLayer as well
+ try {
+ var vrt_file = fs.readFileSync(d.file, 'utf8').toString();
+ if (vrt_file.indexOf('OGRVRTLayer') != -1) {
+ d.type = 'ogr';
+ d.layer_by_index = 0;
+ if (!l.srs) {
+ var match = vrt_file.match(/<LayerSRS>(.+)<\/LayerSRS>/);
+ if (match && match[1]) {
+ var srs_parsed = srs.parse(match[1]);
+ l.srs = srs_parsed.proj4;
+ }
+ }
+ }
+ } catch (e) {
+ if (env == 'development') console.error('failed to open vrt file: ' + e.message);
+ }
+ break;
+ case '.csv':
+ case '.tsv':
+ case '.txt':
+ case '.osm':
+ case '.gpx':
+ l.srs = l.srs || SRS.WGS84;
+ break;
+ case '.geojson':
+ case '.json':
+ d.layer_by_index = 0;
+ try {
+ l.srs = l.srs || srs.parse(d.file).proj4;
+ } catch (e) {
+ next(new Error("Could not parse: '" + d.file + "': error: '" + e.message + "'"));
+ }
+ break;
+ case '.kml':
+ case '.rss':
+ d.layer_by_index = 0;
+ l.srs = SRS.WGS84;
+ break;
+ }
+ // at this point if we do not know the 'type' of mapnik
+ // plugin to dispatch to we are out of luck and there is no
+ // need to check for the projection
+ if (!d.type) {
+ return next(new Error("Could not detect datasource type for: '"+d.file+"'"))
+ }
+ if (l.srs) return next();
+ var error = new Error('Unable to determine SRS for layer "' + name + '" at ' + d.file);
+ if (d.type !== 'shape') {
+ // If we don't have a projection by now, bail out unless we have a shapefile.
+ return next(error);
+ } else {
+ // Special handling that opens .prj files for shapefiles.
+ var prj_path = path.join(
+ path.dirname(d.file),
+ path.basename(d.file, path.extname(d.file)) + '.prj'
+ );
+ fs.readFile(prj_path, 'utf8', function(err, data) {
+ if (err && err.code === 'ENOENT') {
+ return next(error);
+ } else if (err) {
+ return next(err);
+ }
try {
- ext = guessExtension(JSON.parse(data));
- next(null, ext);
+ l.srs = l.srs || srs.parse(data).proj4;
+ l.srs = l.srs || srs.parse('ESRI::' + data).proj4; // See issue #26.
} catch (e) {
next(e);
}
+ next(l.srs ? null : error);
});
- });
- });
- }
- }, function(err, ext) {
- // Ignore errors during extension checks above and let a
- // missing extension fall through to a missing `type`.
-
- var name = l.name || 'layer-' + index;
-
- var ext = ext || path.extname(d.file);
- switch (ext) {
- case '.csv':
- case '.tsv': // google refine uses tsv for tab-delimited
- case '.txt': // resonable assumption that .txt is csv?
- d.quiet = d.quiet || true; // Supress verbose mapnik error reporting by default.
- d.type = d.type || 'csv';
- l.srs = l.srs || SRS.WGS84;
- break;
- case '.shp':
- case '.zip':
- d.type = d.type || 'shape';
- break;
- case '.geotiff':
- case '.geotif':
- case '.vrt':
- case '.tiff':
- case '.tif':
- d.type = d.type || 'gdal';
- break;
- case '.geojson':
- case '.json':
- d.type = d.type || 'ogr';
- d.layer_by_index = 0;
- l.srs = l.srs || srs.parse(d.file).proj4;
- break;
- case '.kml':
- case '.rss':
- d.type = d.type || 'ogr';
- d.layer_by_index = 0;
- // WGS84 is the only valid SRS for KML and RSS so we force
- // it here.
- l.srs = SRS.WGS84;
- break;
- }
-
- if (l.srs) return next();
-
- var error = new Error('Unable to determine SRS for layer "' + name + '" at ' + d.file);
- if (d.type !== 'shape') {
- // If we don't have a projection by now, bail out unless we have a shapefile.
- return next(error);
- } else {
- // Special handling that opens .prj files for shapefiles.
- var prj_path = path.join(
- path.dirname(d.file),
- path.basename(d.file, path.extname(d.file)) + '.prj'
- );
- fs.readFile(prj_path, 'utf8', function(err, data) {
- if (err && err.code === 'ENOENT') {
- return next(error);
- } else if (err) {
- return next(err);
- }
-
- try {
- l.srs = l.srs || srs.parse(data).proj4;
- l.srs = l.srs || srs.parse('ESRI::' + data).proj4; // See issue #26.
- } catch (e) {
- next(e);
}
-
- next(l.srs ? null : error);
});
}
});
@@ -534,8 +969,9 @@ function resolve(options, callback) {
resolved.srs = resolved.srs || SRS['900913'];
fixSRS(resolved);
resolved.Layer.forEach(fixSRS);
-
- callback(err, resolved);
+ if (!err && env == 'development') console.error("[millstone] finished processing '" + options.base + "'");
+ if (Object.keys(downloads).length < 1) clearInterval(download_log_interval);
+ return callback(err, resolved);
});
}
@@ -577,13 +1013,13 @@ function flush(options, callback) {
}
}, function removeCache(err) {
if (err) throw err;
- path.exists(filepath, function(exists) {
+ existsAsync(filepath, function(exists) {
if (!exists) return this();
utils.rm(filepath, this);
}.bind(this));
}, function removeMetafile(err) {
if (err) throw err;
- path.exists(metapath(filepath), function(exists) {
+ existsAsync(metapath(filepath), function(exists) {
if (!exists) return this();
utils.rm(metapath(filepath), this);
}.bind(this));
@@ -592,8 +1028,25 @@ function flush(options, callback) {
});
}
+function drainPool(callback) {
+ clearInterval(download_log_interval);
+ if (pool) {
+ pool.drain(function() {
+ pool.destroyAllNow(function() {
+ return callback();
+ });
+ });
+ }
+}
+
module.exports = {
resolve: resolve,
- flush: flush
+ flush: flush,
+ isRelative: isRelative,
+ guessExtension: guessExtension,
+ downloads: downloads,
+ valid_ds_extensions: valid_ds_extensions,
+ valid_marker_extensions: valid_marker_extensions,
+ drainPool:drainPool
};
diff --git a/lib/util.js b/lib/util.js
index 7732730..3313967 100644
--- a/lib/util.js
+++ b/lib/util.js
@@ -1,8 +1,9 @@
var fs = require('fs'),
path = require('path'),
_ = require('underscore'),
- Step = require('step'),
- mkdirP = require('mkdirp');
+ Step = require('step');
+
+var env = process.env.NODE_ENV || 'development';
// Recursive rm.
function rm(filepath, callback) {
@@ -31,34 +32,160 @@ function rm(filepath, callback) {
// Like fs.symlink except that it will overwrite stale symlinks at the
// given path if it exists.
-function forcelink(linkdata, path, callback) {
- fs.lstat(path, function(err, stat) {
+function forcelink(src, dest, options, callback) {
+ if (!options || !options.cache) throw new Error('options.cache not defined!');
+ if (!callback) throw new Error('callback not defined!');
+ // uses relative path if linking to cache dir
+ if (path.relative) {
+ src = path.relative(options.cache, dest).slice(0, 2) !== '..' ? path.relative(path.dirname(dest), src) : src;
+ }
+ fs.lstat(dest, function(err, stat) {
// Error.
- if (err && err.code !== 'ENOENT')
+ if (err && err.code !== 'ENOENT') {
return callback(err);
+ }
// Path does not exist. Symlink.
- if (err && err.code === 'ENOENT')
- return fs.symlink(linkdata, path, callback);
+ if (err && err.code === 'ENOENT') {
+ if (env == 'development') console.error("[millstone] linking '" + dest + "' -> '" + src + "'");
+ return fs.symlink(src, dest, callback);
+ }
// Path exists and is not a symlink. Do nothing.
- if (!stat.isSymbolicLink()) return callback();
+ if (!stat.isSymbolicLink()) {
+ if (env == 'development') console.error("[millstone] skipping re-linking '" + src + "' because '" + dest + " is already an existing file");
+ return callback();
+ }
- // Path exists and is a symlink. Check existing link path and update
- // if necessary.
- fs.readlink(path, function(err, old) {
+ // Path exists and is a symlink. Check existing link path and update if necessary.
+ // NOTE : broken symlinks will pass through this step
+ fs.readlink(dest, function(err, old) {
if (err) return callback(err);
- if (old === linkdata) return callback();
- fs.unlink(path, function(err) {
+ if (old === src) return callback();
+ fs.unlink(dest, function(err) {
if (err) return callback(err);
- fs.symlink(linkdata, path, callback);
+ if (env == 'development') console.error("[millstone] re-linking '" + dest + "' -> '" + src + "'");
+ fs.symlink(src, dest, callback);
});
});
});
}
+function copyStream(src,dest,callback) {
+ var src_stream = fs.createReadStream(src);
+ src_stream.on('error', function(err) {
+ return callback(err);
+ });
+ var dest_stream = fs.createWriteStream(dest);
+ dest_stream.on('error', function(err) {
+ return callback(err);
+ });
+ src_stream.on('end', function() {
+ if (env == 'development') console.error("[millstone] finished copying '" + src + "' to '" + dest + "'");
+ return callback(null);
+ });
+ src_stream.pipe(dest_stream);
+}
+
+function copy(src, dest, options, callback) {
+ if (!options) throw new Error('options not defined!');
+ if (!callback) throw new Error('callback not defined!');
+ fs.lstat(dest, function(err, dest_stat) {
+ // Error.
+ if (err && err.code !== 'ENOENT'){
+ return callback(err);
+ }
+
+ // Dest path does not exist. Copy.
+ if (err && err.code === 'ENOENT') {
+ if (env == 'development') console.error("[millstone] attempting to copy '" + src + "' to '" + dest + "'");
+ return copyStream(src,dest,callback);
+ }
+
+ // Path exists and is a symlink. Do nothing.
+ if (dest_stat.isSymbolicLink()) {
+ if (env == 'development') console.error("[millstone] skipping copying '" + src + "' because '" + dest + " is an existing symlink");
+ return callback();
+ }
+
+ // Dest path exists and is a file. Check if it needs updating
+ fs.lstat(src, function(err,src_stat) {
+ // Error.
+ if (err) {
+ return callback(err);
+ }
+ // NOTE: we intentially do not compare the STAT exactly
+ // because some windows users will be used to editing files
+ // after they are copied. In future releases we should consider
+ // simply doing if (src_stat != dest_stat)
+ // check size to dodge potentially corrupt data
+ // https://github.com/mapbox/tilemill/issues/1674
+ if ((src_stat.size != dest_stat.size && src_stat.mtime < dest_stat.mtime) || src_stat.mtime > dest_stat.mtime) {
+ if (env == 'development') console.error("[millstone] attempting to re-copy '" + src + "' to '" + dest + "'");
+ return copyStream(src,dest,callback);
+ } else {
+ return callback();
+ }
+ });
+ });
+}
+
+
+function processFiles(src, dest, fn, options, callback) {
+ if (!options) throw new Error('options not defined!');
+ if (!fn) throw new Error('link function not defined!');
+ if (!callback) throw new Error('callback not defined!');
+ var basename = path.basename(src, path.extname(src)),
+ srcdir = path.dirname(src),
+ destdir = path.dirname(dest);
+
+ // Group multiple calls
+ var remaining,
+ err;
+ function done(err) {
+ remaining --;
+ if (!remaining) callback(err);
+ }
+
+ function forcemkdir(dir, callback) {
+ fs.lstat(dir, function(err, stat) {
+ if (err && err.code !== 'ENOENT') {
+ return callback(err);
+ } else if (!err && stat.isSymbolicLink()) {
+ return fs.unlink(dir, function(err) {
+ if (err) return callback(err);
+ fs.mkdir(dir, 0755, callback);
+ });
+
+ } else {
+ return fs.mkdir(dir, 0755, callback);
+ }
+ });
+ }
+
+ function processFiltered(err, files) {
+ if (err) return callback(err);
+
+ files = files.filter(function(f) {
+ return path.basename(f, path.extname(f)) === basename;
+ });
+ if (files.length < 1) return callback(err);
+
+ remaining = files.length;
+
+ files.forEach(function(f) {
+ fn(path.join(srcdir, f), path.join(destdir, f), options, done);
+ });
+ }
+
+ forcemkdir(destdir, function() {
+ fs.readdir(srcdir, processFiltered);
+ });
+}
+
module.exports = {
- mkdirP: mkdirP,
forcelink: forcelink,
- rm: rm
+ rm: rm,
+ copy: copy,
+ processFiles: processFiles
};
diff --git a/package.json b/package.json
index af8367d..b4f3328 100644
--- a/package.json
+++ b/package.json
@@ -1,36 +1,39 @@
{
"name": "millstone",
- "version": "0.4.0",
+ "version": "0.6.8",
"main": "./lib/millstone.js",
"description": "Prepares datasources in an MML file for consumption in Mapnik",
"url": "https://github.com/mapbox/millstone",
"licenses": [{ "type": "BSD" }],
- "repositories": [{
+ "repository": {
"type": "git",
"url": "git://github.com/mapbox/millstone.git"
- }],
+ },
"author": {
"name": "MapBox",
"url": "http://mapbox.com/",
"email": "info at mapbox.com"
},
"dependencies": {
- "underscore" : "1.1.x",
- "step": "0.0.x",
- "generic-pool": "1.0.x",
- "get": "~1.1.3",
- "srs": "~0.2.12",
- "zipfile": "0.x",
+ "underscore": "~1.5.1",
+ "step": "~0.0.5",
+ "generic-pool": "~2.0.3",
+ "request": "~2.26.0",
+ "srs": "~0.3.6",
+ "zipfile": "~0.4.2",
"sqlite3": "2.x",
- "mime": ">= 0.0.1",
- "mkdirp": "~0.3.0"
+ "mime": "~1.2.9",
+ "mkdirp": "~0.3.3",
+ "optimist": "~0.6.0"
},
"devDependencies": {
- "expresso": "0.9.x"
+ "mocha": "*"
+ },
+ "bin": {
+ "millstone": "./bin/millstone"
},
"scripts": {
- "pretest": "which expresso || npm install --dev",
- "test": "which expresso | sh"
+ "test": "mocha -R spec --timeout 10000"
},
- "engines": { "node": "0.4 || 0.6" }
+ "engines": { "node": "0.4 || 0.6 || 0.8 || 0.10" }
}
diff --git a/test/UPPERCASE_EXT/project.mml b/test/UPPERCASE_EXT/project.mml
new file mode 100644
index 0000000..4646e93
--- /dev/null
+++ b/test/UPPERCASE_EXT/project.mml
@@ -0,0 +1,13 @@
+{
+ "Stylesheet": [
+ "style.mss"
+ ],
+ "Layer": [
+ {
+ "name": "uppercase-ext",
+ "Datasource": {
+ "file": "test1.CSV"
+ }
+ }
+ ]
+}
diff --git a/test/UPPERCASE_EXT/style.mss b/test/UPPERCASE_EXT/style.mss
new file mode 100644
index 0000000..230a97c
--- /dev/null
+++ b/test/UPPERCASE_EXT/style.mss
@@ -0,0 +1 @@
+#polygon { }
\ No newline at end of file
diff --git a/test/UPPERCASE_EXT/test1.CSV b/test/UPPERCASE_EXT/test1.CSV
new file mode 100644
index 0000000..86e3501
--- /dev/null
+++ b/test/UPPERCASE_EXT/test1.CSV
@@ -0,0 +1,8 @@
+{ "type": "FeatureCollection",
+ "features": [
+ { "type": "Feature",
+ "geometry": {"type": "Point", "coordinates": [102.0, 0.5]},
+ "properties": {"prop0": "value0"}
+ },
+ ]
+}
\ No newline at end of file
diff --git a/test/cache/cache.mml b/test/cache/cache.mml
index 2801fcd..e143761 100644
--- a/test/cache/cache.mml
+++ b/test/cache/cache.mml
@@ -2,7 +2,7 @@
"Stylesheet": [
{
"id": "cache-inline.mss",
- "data": "Map { backgroound-color:#fff }"
+ "data": "Map { background-color:#fff }"
},
"cache-local.mss",
"http://mapbox.github.com/millstone/test/cache-url.mss"
@@ -52,7 +52,8 @@
"file": "layers/countries.sqlite",
"type": "sqlite",
"table": "countries"
- }
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
},
{
"name": "sqlite-attach",
@@ -61,6 +62,13 @@
"type": "sqlite",
"table": "countries",
"attachdb": "data at layers/data.sqlite"
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
+ },
+ {
+ "name": "zip-no-ext",
+ "Datasource": {
+ "file": "http://fakeurl.com/zip_no_ext"
}
}
]
diff --git a/test/corrupt-zip.test.js b/test/corrupt-zip.test.js
new file mode 100644
index 0000000..ff6b211
--- /dev/null
+++ b/test/corrupt-zip.test.js
@@ -0,0 +1,57 @@
+var fs = require('fs');
+var path = require('path');
+var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+var utils = require('../lib/util.js');
+var millstone = require('../lib/millstone');
+var tests = module.exports = {};
+var rm = require('./support.js').rm;
+
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+beforeEach(function(){
+ rm(path.join(__dirname, '/tmp/millstone-test'));
+})
+
+
+// NOTE: watch out, this zip has both a csv and shape in it and uses
+// non-ascii characters - idea being to be the basis for other tests
+// https://github.com/mapbox/millstone/issues/85
+it('correctly handles re-downloading a zip that is invalid in its cached state', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'corrupt-zip/project.mml')));
+
+ var cache = '/tmp/millstone-test';
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'corrupt-zip'),
+ cache: cache
+ };
+
+ try {
+ fs.mkdirSync(options.cache, 0777);
+ } catch (e) {}
+
+ // write bogus data over the zip archive to simulate a corrupt cache
+ if (!existsSync('/tmp/millstone-test/29f2b277-Cle%CC%81ment/')) fs.mkdirSync('/tmp/millstone-test/29f2b277-Cle%CC%81ment/')
+ fs.writeFileSync('/tmp/millstone-test/29f2b277-Cle%CC%81ment/29f2b277-Cle%CC%81ment.zip','');
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.equal(err,undefined,err);
+ assert.equal(resolved.Stylesheet[0].id, 'style.mss');
+ assert.equal(resolved.Stylesheet[0].data, '#polygon { }');
+ var expected = [
+ {
+ "name": "corrupt-zip",
+ "Datasource": {
+ "file": path.join(__dirname, 'corrupt-zip/layers/corrupt-zip/29f2b277-Cle%CC%81ment.shp'),
+ "type": "shape"
+ },
+ "srs": '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'
+ }
+ ];
+ assert.deepEqual(resolved.Layer, expected);
+ done();
+ });
+});
diff --git a/test/corrupt-zip/project.mml b/test/corrupt-zip/project.mml
new file mode 100644
index 0000000..5418861
--- /dev/null
+++ b/test/corrupt-zip/project.mml
@@ -0,0 +1,13 @@
+{
+ "Stylesheet": [
+ "style.mss"
+ ],
+ "Layer": [
+ {
+ "name": "corrupt-zip",
+ "Datasource": {
+ "file": "https://github.com/mapbox/millstone/raw/gh-pages/test/Cle%CC%81ment.zip"
+ }
+ }
+ ]
+}
diff --git a/test/corrupt-zip/style.mss b/test/corrupt-zip/style.mss
new file mode 100644
index 0000000..230a97c
--- /dev/null
+++ b/test/corrupt-zip/style.mss
@@ -0,0 +1 @@
+#polygon { }
\ No newline at end of file
diff --git a/test/data/9368bdd9-zip_no_ext/.9368bdd9-zip_no_ext b/test/data/9368bdd9-zip_no_ext/.9368bdd9-zip_no_ext
new file mode 100644
index 0000000..693a9ae
--- /dev/null
+++ b/test/data/9368bdd9-zip_no_ext/.9368bdd9-zip_no_ext
@@ -0,0 +1 @@
+{"content-disposition":"attachment; filename=places_by_lon_lat.zip"}
diff --git a/test/data/9368bdd9-zip_no_ext/9368bdd9-zip_no_ext b/test/data/9368bdd9-zip_no_ext/9368bdd9-zip_no_ext
new file mode 100644
index 0000000..a55016d
Binary files /dev/null and b/test/data/9368bdd9-zip_no_ext/9368bdd9-zip_no_ext differ
diff --git a/test/data/ne_10m_admin_0_boundary_lines_disputed_areas.zip b/test/data/ne_10m_admin_0_boundary_lines_disputed_areas.zip
new file mode 100644
index 0000000..4e8705c
Binary files /dev/null and b/test/data/ne_10m_admin_0_boundary_lines_disputed_areas.zip differ
diff --git a/test/data/snow-cover.tif b/test/data/snow-cover.tif
new file mode 100644
index 0000000..239fad9
Binary files /dev/null and b/test/data/snow-cover.tif differ
diff --git a/test/error.test.js b/test/error.test.js
new file mode 100644
index 0000000..8fba15f
--- /dev/null
+++ b/test/error.test.js
@@ -0,0 +1,64 @@
+var fs = require('fs');
+var path = require('path');
+var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+var utils = require('../lib/util.js');
+var millstone = require('../lib/millstone');
+var tests = module.exports = {};
+var rm = require('./support.js').rm;
+
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+beforeEach(function(){
+ rm(path.join(__dirname, '/tmp/millstone-test'));
+})
+
+it('correctly handles invalid json', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'invalid-json/project.mml')));
+
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'invalid-json'),
+ cache: '/tmp/millstone-test'
+ };
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.ok(err.message.search("error: 'Unexpected token ]'") != -1);
+ done();
+ });
+});
+
+it('correctly handles missing shapefile at relative path', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'missing-file-relative/project.mml')));
+
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'missing-file-relative'),
+ cache: '/tmp/millstone-test'
+ };
+
+ millstone.resolve(options, function(err, resolved) {
+ var err_expected = err.message.search("File not found:") != -1 || err.message.search("Can't open") != -1;
+ assert.ok(err_expected);
+ done();
+ });
+});
+
+
+it('correctly handles missing shapefile at absolute path', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'missing-file-absolute/project.mml')));
+
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'missing-file-absolute'),
+ cache: '/tmp/millstone-test'
+ };
+
+ millstone.resolve(options, function(err, resolved) {
+ var err_expected = err.message.search("File not found:") != -1 || err.message.search("Can't open") != -1;
+ assert.ok(err_expected);
+ done();
+ });
+});
\ No newline at end of file
diff --git a/test/image-noext.test.js b/test/image-noext.test.js
new file mode 100644
index 0000000..0fe4416
--- /dev/null
+++ b/test/image-noext.test.js
@@ -0,0 +1,39 @@
+var fs = require('fs');
+var path = require('path');
+var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+var utils = require('../lib/util.js');
+var millstone = require('../lib/millstone');
+var tests = module.exports = {};
+var rm = require('./support.js').rm;
+
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+beforeEach(function(){
+ rm(path.join(__dirname, '/tmp/millstone-test'));
+})
+
+it('correctly handles images with no extension', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'image-noext/project.mml')));
+
+ var cache = '/tmp/millstone-test';
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'image-noext'),
+ cache: cache
+ };
+
+ try {
+ fs.mkdirSync(options.cache, 0777);
+ } catch (e) {}
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.equal(err,undefined,err);
+ assert.equal(resolved.Stylesheet[0].id, 'style.mss');
+ assert.equal(resolved.Stylesheet[0].data, "Map {background-image: url('/tmp/millstone-test/2b2cf79a-images/2b2cf79a-images.jpeg');}");
+ assert.ok(existsSync('/tmp/millstone-test/2b2cf79a-images/2b2cf79a-images.jpeg'));
+ done();
+ });
+});
diff --git a/test/image-noext/project.mml b/test/image-noext/project.mml
new file mode 100644
index 0000000..ca6f060
--- /dev/null
+++ b/test/image-noext/project.mml
@@ -0,0 +1,14 @@
+{
+ "Stylesheet": [
+ "style.mss"
+ ],
+ "Layer": [
+ {
+ "name": "background-image-noext",
+ "Datasource": {
+ "type": "csv",
+ "inline": "x,y\n0,0"
+ }
+ }
+ ]
+}
diff --git a/test/image-noext/style.mss b/test/image-noext/style.mss
new file mode 100644
index 0000000..21b83db
--- /dev/null
+++ b/test/image-noext/style.mss
@@ -0,0 +1 @@
+Map {background-image: url('http://t2.gstatic.com/images?q=tbn:ANd9GcQu_OWjgK-0EgHE2kBp9aaGbPMuQ77tuyhbMOybMZ_FFY6TrUAL3wzh0xw7Yw');}
\ No newline at end of file
diff --git a/test/invalid-json/broken.json b/test/invalid-json/broken.json
new file mode 100644
index 0000000..86e3501
--- /dev/null
+++ b/test/invalid-json/broken.json
@@ -0,0 +1,8 @@
+{ "type": "FeatureCollection",
+ "features": [
+ { "type": "Feature",
+ "geometry": {"type": "Point", "coordinates": [102.0, 0.5]},
+ "properties": {"prop0": "value0"}
+ },
+ ]
+}
\ No newline at end of file
diff --git a/test/invalid-json/project.mml b/test/invalid-json/project.mml
new file mode 100644
index 0000000..f9b6585
--- /dev/null
+++ b/test/invalid-json/project.mml
@@ -0,0 +1,13 @@
+{
+ "Stylesheet": [
+ "style.mss"
+ ],
+ "Layer": [
+ {
+ "name": "polygons-zipped",
+ "Datasource": {
+ "file": "broken.json"
+ }
+ }
+ ]
+}
diff --git a/test/invalid-json/style.mss b/test/invalid-json/style.mss
new file mode 100644
index 0000000..230a97c
--- /dev/null
+++ b/test/invalid-json/style.mss
@@ -0,0 +1 @@
+#polygon { }
\ No newline at end of file
diff --git a/test/macosx-zipped.test.js b/test/macosx-zipped.test.js
new file mode 100644
index 0000000..231eff9
--- /dev/null
+++ b/test/macosx-zipped.test.js
@@ -0,0 +1,41 @@
+var fs = require('fs');
+var path = require('path');
+var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+var utils = require('../lib/util.js');
+var millstone = require('../lib/millstone');
+var tests = module.exports = {};
+var rm = require('./support.js').rm;
+
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+before(function(){
+ rm('/tmp/millstone-test');
+});
+
+it('correctly handles mac os x zipped archives with the lame __MACOSX/ subfolder', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'macosx-zipped/project.mml')));
+
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'macosx-zipped'),
+ cache: '/tmp/millstone-test'
+ };
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.equal(err,undefined,err);
+ assert.deepEqual(resolved.Layer, [
+ {
+ "name": "points",
+ "Datasource": {
+ "file": path.join(__dirname, 'macosx-zipped/layers/points/9afe4795-crashes_2007_2009.shp'),
+ "type": "shape"
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
+ }
+ ]);
+ done();
+ });
+});
\ No newline at end of file
diff --git a/test/macosx-zipped/project.mml b/test/macosx-zipped/project.mml
new file mode 100644
index 0000000..4464d23
--- /dev/null
+++ b/test/macosx-zipped/project.mml
@@ -0,0 +1,13 @@
+{
+ "Stylesheet": [
+ "style.mss"
+ ],
+ "Layer": [
+ {
+ "name": "points",
+ "Datasource": {
+ "file": "http://cartodb.s3.amazonaws.com/static/crashes_2007_2009.zip"
+ }
+ }
+ ]
+}
diff --git a/test/macosx-zipped/style.mss b/test/macosx-zipped/style.mss
new file mode 100644
index 0000000..c713d24
--- /dev/null
+++ b/test/macosx-zipped/style.mss
@@ -0,0 +1 @@
+#points { }
\ No newline at end of file
diff --git a/test/markers.test.js b/test/markers.test.js
new file mode 100644
index 0000000..db66c91
--- /dev/null
+++ b/test/markers.test.js
@@ -0,0 +1,73 @@
+var fs = require('fs');
+var path = require('path');
+var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+var utils = require('../lib/util.js');
+var millstone = require('../lib/millstone');
+var tests = module.exports = {};
+var rm = require('./support.js').rm;
+
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+before(function(){
+ rm('/tmp/millstone-test');
+});
+
+
+it('correctly localizes remote image/svg files', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'markers/project.mml')));
+
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'markers'),
+ cache: '/tmp/millstone-test'
+ };
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.equal(err,undefined,err);
+ assert.equal(resolved.Stylesheet[0].id, 'style.mss');
+ assert.equal(resolved.Stylesheet[0].data, '// a url like https:example.com in the comments\n#points { one/marker-file: url(\'/tmp/millstone-test/e33af80e-Cup_of_coffee.svg\'); two/marker-file: url(\'/tmp/millstone-test/e33af80e-Cup_of_coffee.svg\'); four/marker-file: url("/tmp/millstone-test/c953e0d1-pin-m-fast-food+AA0000.png"); five/marker-file:url("/tmp/millstone-test/7b9b9979-fff&text=x/7b9b9979-fff&text=x.png"); }\n');
+ assert.deepEqual(resolved.Layer, [
+ {
+ "name": "points",
+ "Datasource": {
+ "file": path.join(__dirname, 'markers/layers/points.csv'),
+ "type": "csv"
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
+ }
+ ]);
+ done();
+ });
+});
+
+it('correctly localizes zipped json', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'zipped-json/project.mml')));
+
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'zipped-json'),
+ cache: '/tmp/millstone-test'
+ };
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.equal(err,undefined,err);
+ assert.equal(resolved.Stylesheet[0].id, 'style.mss');
+ assert.equal(resolved.Stylesheet[0].data, '#polygon { }');
+ var expected = [
+ {
+ "name": "polygons-zipped",
+ "Datasource": {
+ "file": path.join(__dirname, 'zipped-json/layers/polygons-zipped/7e482cc8-polygons.json.json'),
+ "type": "ogr",
+ "layer_by_index": 0
+ },
+ "srs": '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'
+ }
+ ];
+ assert.deepEqual(resolved.Layer, expected);
+ done();
+ });
+});
diff --git a/test/markers/layers/points.csv b/test/markers/layers/points.csv
new file mode 100644
index 0000000..51cb66f
--- /dev/null
+++ b/test/markers/layers/points.csv
@@ -0,0 +1,2 @@
+x,y,name
+0,0,"null island"
\ No newline at end of file
diff --git a/test/markers/project.mml b/test/markers/project.mml
new file mode 100644
index 0000000..ea5f18c
--- /dev/null
+++ b/test/markers/project.mml
@@ -0,0 +1,14 @@
+{
+ "Stylesheet": [
+ "style.mss"
+ ],
+ "Layer": [
+ {
+ "name": "points",
+ "Datasource": {
+ "type": "csv",
+ "file": "layers/points.csv"
+ }
+ }
+ ]
+}
diff --git a/test/markers/style.mss b/test/markers/style.mss
new file mode 100644
index 0000000..4e1e67d
--- /dev/null
+++ b/test/markers/style.mss
@@ -0,0 +1,2 @@
+// a url like https:example.com in the comments
+#points { one/marker-file: url('http://upload.wikimedia.org/wikipedia/commons/7/72/Cup_of_coffee.svg'); two/marker-file: url('http://upload.wikimedia.org/wikipedia/commons/7/72/Cup_of_coffee.svg'); four/marker-file: url("http://a.tiles.mapbox.com/v3/marker/pin-m-fast-food+AA0000.png"); five/marker-file:url("http://dummyimage.com/16x16/000/fff&text=x"); }
diff --git a/test/missing-file-absolute/project.mml b/test/missing-file-absolute/project.mml
new file mode 100644
index 0000000..1ddf11b
--- /dev/null
+++ b/test/missing-file-absolute/project.mml
@@ -0,0 +1,13 @@
+{
+ "Stylesheet": [
+ "style.mss"
+ ],
+ "Layer": [
+ {
+ "name": "missing-file-relative",
+ "Datasource": {
+ "file": "/missing.shp"
+ }
+ }
+ ]
+}
diff --git a/test/missing-file-absolute/style.mss b/test/missing-file-absolute/style.mss
new file mode 100644
index 0000000..230a97c
--- /dev/null
+++ b/test/missing-file-absolute/style.mss
@@ -0,0 +1 @@
+#polygon { }
\ No newline at end of file
diff --git a/test/missing-file-relative/project.mml b/test/missing-file-relative/project.mml
new file mode 100644
index 0000000..7d49cc8
--- /dev/null
+++ b/test/missing-file-relative/project.mml
@@ -0,0 +1,13 @@
+{
+ "Stylesheet": [
+ "style.mss"
+ ],
+ "Layer": [
+ {
+ "name": "missing-file-relative",
+ "Datasource": {
+ "file": "missing.shp"
+ }
+ }
+ ]
+}
diff --git a/test/missing-file-relative/style.mss b/test/missing-file-relative/style.mss
new file mode 100644
index 0000000..230a97c
--- /dev/null
+++ b/test/missing-file-relative/style.mss
@@ -0,0 +1 @@
+#polygon { }
\ No newline at end of file
diff --git a/test/multi-shape-zip.test.js b/test/multi-shape-zip.test.js
new file mode 100644
index 0000000..20d3789
--- /dev/null
+++ b/test/multi-shape-zip.test.js
@@ -0,0 +1,48 @@
+var fs = require('fs');
+var path = require('path');
+var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+var utils = require('../lib/util.js');
+var millstone = require('../lib/millstone');
+var tests = module.exports = {};
+var rm = require('./support.js').rm;
+
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+beforeEach(function(){
+ rm(path.join(__dirname, '/tmp/millstone-test'));
+})
+
+// https://github.com/mapbox/millstone/issues/99
+it('correctly handles a zipfile containing multiple shapefiles without corrupting data', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'multi-shape-zip/project.mml')));
+
+ var cache = '/tmp/millstone-test';
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'multi-shape-zip'),
+ cache: cache
+ };
+
+ try {
+ fs.mkdirSync(options.cache, 0777);
+ } catch (e) {}
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.equal(err,undefined,err);
+ var expected = [
+ {
+ "name": "multi-shape-zip",
+ "Datasource": {
+ "file": path.join(__dirname, 'multi-shape-zip/layers/multi-shape-zip/134ecf39-PLATES_PlateBoundary_ArcGIS.shp'),
+ "type": "shape"
+ },
+ "srs": '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'
+ }
+ ];
+ assert.deepEqual(resolved.Layer, expected);
+ done();
+ });
+});
diff --git a/test/multi-shape-zip/project.mml b/test/multi-shape-zip/project.mml
new file mode 100644
index 0000000..889b579
--- /dev/null
+++ b/test/multi-shape-zip/project.mml
@@ -0,0 +1,11 @@
+{
+ "Stylesheet": [],
+ "Layer": [
+ {
+ "name": "multi-shape-zip",
+ "Datasource": {
+ "file": "https://github.com/mapbox/millstone/raw/gh-pages/test/PLATES_PlateBoundary_ArcGIS.zip"
+ }
+ }
+ ]
+}
diff --git a/test/nosymlink.test.js b/test/nosymlink.test.js
new file mode 100644
index 0000000..e57b5cf
--- /dev/null
+++ b/test/nosymlink.test.js
@@ -0,0 +1,102 @@
+var fs = require('fs');
+var path = require('path');
+var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+var utils = require('../lib/util.js');
+var millstone = require('../lib/millstone');
+var tests = module.exports = {};
+var rm = require('./support.js').rm;
+
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+beforeEach(function(){
+ rm('/tmp/millstone-test');
+});
+
+
+it('correctly handles files without symlinking', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'nosymlink/project.mml')));
+
+ var cache = '/tmp/millstone-test';
+ mml.Layer[4].Datasource.file = path.join(cache, "pshape.zip");
+
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'nosymlink'),
+ cache: cache,
+ nosymlink:true
+ };
+
+ try {
+ fs.mkdirSync(options.cache, 0777);
+ } catch (e) {}
+
+ try {
+ var newFile = fs.createWriteStream(path.join(options.cache, 'pshape.zip'));
+ var oldFile = fs.createReadStream(path.join(__dirname, 'nosymlink/pshape.zip'));
+ oldFile.pipe(newFile);
+ } catch (e) {console.log(e)}
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.equal(err,undefined,err);
+ assert.equal(resolved.Stylesheet[0].id, 'style.mss');
+ var expected = [
+ {
+ "name": "one",
+ "Datasource": {
+ "file": path.join(__dirname, 'nosymlink/points.csv'),
+ "type": "csv"
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
+ },
+ {
+ "name": "two",
+ "Datasource": {
+ "file": path.join(__dirname, "nosymlink/pshape.shp"),
+ "type": "shape"
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
+ },
+ {
+ "name": "three",
+ "Datasource": {
+ "file": path.join(__dirname, "nosymlink/pshape.shp"),
+ "type": "shape"
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
+ },
+ {
+ "name": "four",
+ "Datasource": {
+ "file": path.join(__dirname, "nosymlink/points.vrt"),
+ "type": "ogr",
+ "layer_by_index": 0
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
+ },
+ {
+ "name": "five",
+ "Datasource": {
+ "file": path.join(options.cache, "pshape.shp"),
+ "type": "shape"
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
+ }
+ ];
+ for (var i=0;i<=expected.length;i++) {
+ assert.deepEqual(resolved.Layer[i], expected[i]);
+ }
+ done();
+ });
+});
+
+after(function() {
+ // cleanup
+ rm(path.join(__dirname, 'nosymlink','pshape.shp'));
+ rm(path.join(__dirname, 'nosymlink','pshape.dbf'));
+ rm(path.join(__dirname, 'nosymlink','pshape.prj'));
+ rm(path.join(__dirname, 'nosymlink','pshape.shx'));
+ rm(path.join(__dirname, 'nosymlink','.pshape.zip'));
+})
diff --git a/test/nosymlink/points.csv b/test/nosymlink/points.csv
new file mode 100644
index 0000000..51cb66f
--- /dev/null
+++ b/test/nosymlink/points.csv
@@ -0,0 +1,2 @@
+x,y,name
+0,0,"null island"
\ No newline at end of file
diff --git a/test/nosymlink/points.dbf b/test/nosymlink/points.dbf
new file mode 100644
index 0000000..b07eb81
Binary files /dev/null and b/test/nosymlink/points.dbf differ
diff --git a/test/nosymlink/points.prj b/test/nosymlink/points.prj
new file mode 100644
index 0000000..a30c00a
--- /dev/null
+++ b/test/nosymlink/points.prj
@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]
\ No newline at end of file
diff --git a/test/nosymlink/points.shp b/test/nosymlink/points.shp
new file mode 100644
index 0000000..b3346b7
Binary files /dev/null and b/test/nosymlink/points.shp differ
diff --git a/test/nosymlink/points.shx b/test/nosymlink/points.shx
new file mode 100644
index 0000000..b3346b7
Binary files /dev/null and b/test/nosymlink/points.shx differ
diff --git a/test/nosymlink/points.vrt b/test/nosymlink/points.vrt
new file mode 100644
index 0000000..3df8850
--- /dev/null
+++ b/test/nosymlink/points.vrt
@@ -0,0 +1,9 @@
+<OGRVRTDataSource>
+ <OGRVRTLayer name="points">
+ <relativeToVRT>1</relativeToVRT>
+ <SrcDataSource>points.csv</SrcDataSource>
+ <GeometryType>wkbPoint</GeometryType>
+ <LayerSRS>WGS84</LayerSRS>
+ <GeometryField encoding="PointFromColumns" x="x" y="y"/>
+ </OGRVRTLayer>
+</OGRVRTDataSource>
\ No newline at end of file
diff --git a/test/nosymlink/project.mml b/test/nosymlink/project.mml
new file mode 100644
index 0000000..8cfd84f
--- /dev/null
+++ b/test/nosymlink/project.mml
@@ -0,0 +1,37 @@
+{
+ "Stylesheet": [
+ "style.mss"
+ ],
+ "Layer": [
+ {
+ "name": "one",
+ "Datasource": {
+ "file": "points.csv"
+ }
+ },
+ {
+ "name": "two",
+ "Datasource": {
+ "file": "pshape.shp"
+ }
+ },
+ {
+ "name": "three",
+ "Datasource": {
+ "file": "pshape.zip"
+ }
+ },
+ {
+ "name": "four",
+ "Datasource": {
+ "file": "points.vrt"
+ }
+ },
+ {
+ "name": "five",
+ "Datasource": {
+ "file": "DYNAMICALLY FILED IN"
+ }
+ }
+ ]
+}
diff --git a/test/nosymlink/pshape.zip b/test/nosymlink/pshape.zip
new file mode 100644
index 0000000..6cc3f9a
Binary files /dev/null and b/test/nosymlink/pshape.zip differ
diff --git a/test/nosymlink/style.mss b/test/nosymlink/style.mss
new file mode 100644
index 0000000..a71aeca
--- /dev/null
+++ b/test/nosymlink/style.mss
@@ -0,0 +1 @@
+#points { marker-width:6; }
\ No newline at end of file
diff --git a/test/raster-linking.test.js b/test/raster-linking.test.js
new file mode 100644
index 0000000..a7271bc
--- /dev/null
+++ b/test/raster-linking.test.js
@@ -0,0 +1,48 @@
+var fs = require('fs');
+var path = require('path');
+var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+var utils = require('../lib/util.js');
+var millstone = require('../lib/millstone');
+var tests = module.exports = {};
+var rm = require('./support.js').rm;
+
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+beforeEach(function(){
+ rm(path.join(__dirname, '/tmp/millstone-test'));
+})
+
+// https://github.com/mapbox/millstone/issues/99
+it('correctly handles a zipfile containing multiple shapefiles without corrupting data', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'raster-linking/project.mml')));
+
+ var cache = '/tmp/millstone-test';
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'raster-linking'),
+ cache: cache
+ };
+
+ try {
+ fs.mkdirSync(options.cache, 0777);
+ } catch (e) {}
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.equal(err,undefined,err);
+ var expected = [
+ {
+ "name": "raster-linking",
+ "Datasource": {
+ "file": path.join(__dirname,"/data/snow-cover.tif"),
+ "type": "gdal"
+ },
+ "srs": "+init=epsg:3857"
+ }
+ ];
+ assert.deepEqual(resolved.Layer, expected);
+ done();
+ });
+});
diff --git a/test/raster-linking/project.mml b/test/raster-linking/project.mml
new file mode 100644
index 0000000..7ad1ce0
--- /dev/null
+++ b/test/raster-linking/project.mml
@@ -0,0 +1,12 @@
+{
+ "Stylesheet": [],
+ "Layer": [
+ {
+ "name": "raster-linking",
+ "Datasource": {
+ "file": "../data/snow-cover.tif"
+ },
+ "srs":"+init=epsg:3857"
+ }
+ ]
+}
diff --git a/test/support.js b/test/support.js
new file mode 100644
index 0000000..542e03f
--- /dev/null
+++ b/test/support.js
@@ -0,0 +1,28 @@
+var fs = require('fs');
+var path = require('path');
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+// Recursive, synchronous rm.
+exports.rm = rm = function(filepath) {
+ if (existsSync(filepath)) {
+ var stat;
+ var files;
+
+ try { stat = fs.lstatSync(filepath); } catch(e) { throw e; }
+
+ // File.
+ if (stat.isFile() || stat.isSymbolicLink()) {
+ return fs.unlinkSync(filepath);
+ // Directory.
+ } else if (stat.isDirectory()) {
+ try { files = fs.readdirSync(filepath); } catch(e) { throw e; }
+ files.forEach(function(file) {
+ try { rm(path.join(filepath, file)); } catch(e) { throw e; }
+ });
+ try { fs.rmdirSync(filepath); } catch(e) { throw e; }
+ // Other?
+ } else {
+ throw new Error('Unrecognized file.');
+ }
+ }
+}
diff --git a/test/test.js b/test/test.js
index c3b7f17..ba6c9c9 100644
--- a/test/test.js
+++ b/test/test.js
@@ -1,33 +1,138 @@
var fs = require('fs');
var path = require('path');
var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+
+var utils = require('../lib/util.js');
var millstone = require('../lib/millstone');
var tests = module.exports = {};
+var rm = require('./support.js').rm;
-// Recursive, synchronous rm.
-function rm(filepath) {
- var stat;
- var files;
-
- try { stat = fs.lstatSync(filepath); } catch(e) { throw e };
-
- // File.
- if (stat.isFile() || stat.isSymbolicLink()) {
- return fs.unlinkSync(filepath);
- // Directory.
- } else if (stat.isDirectory()) {
- try { files = fs.readdirSync(filepath); } catch(e) { throw e };
- files.forEach(function(file) {
- try { rm(path.join(filepath, file)); } catch(e) { throw e };
- });
- try { fs.rmdirSync(filepath); } catch(e) { throw e };
- // Other?
- } else {
- throw new Error('Unrecognized file.')
- }
-};
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+
+beforeEach(function(){
+ rm(path.join(__dirname, 'tmp'));
+})
+
+it('correctly detects content-disposition from kml', function() {
+ // https://github.com/mapbox/millstone/issues/37
+ var header = {
+ 'content-disposition':'attachment; filename="New York City\'s Solidarity Economy.kml"'
+ };
+ var res = millstone.guessExtension(header)
+ assert.equal(res,'.kml');
+});
+
+it('correctly detects content-disposition from google docs csv', function() {
+ // google docs
+ var header = {
+ 'content-disposition':'attachment; filename="Untitledspreadsheet.csv"'
+ };
+ var res = millstone.guessExtension(header)
+ assert.equal(res,'.csv');
+});
+
+it('correctly detects content-disposition from geoserver', function() {
+ // https://github.com/mapbox/millstone/issues/27
+ // geoserver
+ var header = {
+ 'content-disposition':"attachment; filename=foo.csv"
+ };
+ var res = millstone.guessExtension(header)
+ assert.equal(res,'.csv');
+});
+
+it('correctly detects content-disposition from cartodb', function() {
+ // cartodb
+ var header = {
+ 'content-disposition':'inline; filename=cartodb-query.geojson; modification-date="Thu, 10 Nov 2011 19:53:40 GMT";'
+ };
+ var res = millstone.guessExtension(header)
+ assert.equal(res,'.geojson');
+});
+
+it('correctly detects content-type bin', function() {
+ var header = {
+ 'content-type':'application/octet-stream'
+ };
+ var res = millstone.guessExtension(header)
+ assert.equal(res,'');
+});
+
+it('correctly detects datacouch csv content-type', function() {
+ var header = {
+ 'content-type':'text/csv; charset=UTF-8'
+ };
+ var res = millstone.guessExtension(header)
+ assert.equal(res,'.csv');
+});
+
+// http://horn.rcmrd.org/data/geonode:Eth_Region_Boundary
+it('correctly detects geonode content-types', function() {
+ assert.equal('.gml',millstone.guessExtension({'content-type':'text/xml; subtype=gml/2.1.2'}));
+ assert.equal('.zip',millstone.guessExtension({'content-type':'application/zip'}));
+ assert.equal('.kml',millstone.guessExtension({'content-type':'application/vnd.google-earth.kml+xml'}));
+ assert.equal('.json',millstone.guessExtension({'content-type':'application/json'}));
+});
+
+describe('isRelative', function() {
+
+ var realPlatform = process.platform; // Store real platform
+
+ afterEach(function() {
+ process.platform = realPlatform;
+ });
+
+
+ it('detects C:\\ as an absolute path on Windows', function() {
+ process.platform = 'win32';
+ var path = 'C:\\some\\path';
+ var res = millstone.isRelative(path);
+ assert.equal(res, false);
+ });
+
+ it('detects C:\\ as relative path on non-Windows', function() {
+ process.platform = 'linux';
+ var path = 'C:\\some\\path';
+ var res = millstone.isRelative(path);
+ assert.equal(res, true);
+ });
+
+ it('detects paths starting with \\ as absolute on Windows', function() {
+ process.platform = 'win32';
+ var path = '\\some\\path';
+ var res = millstone.isRelative(path);
+ assert.equal(res, false);
+ });
+
+ it('detects paths starting with \\ as relative on non-Windows', function() {
+ process.platform = 'linux';
+ var path = '\\some\\path';
+ var res = millstone.isRelative(path);
+ assert.equal(res, true);
+ });
+
+ it('detects paths starting with / as absolute on non-Windows', function() {
+ process.platform = 'linux';
+ var path = '/some/path';
+ var res = millstone.isRelative(path);
+ assert.equal(res, false);
+ });
+
+ it('detects paths starting with / as absolute on Windows', function() {
+ process.platform = 'win32';
+ var path = '/some/path';
+ var res = millstone.isRelative(path);
+ assert.equal(res, false);
+ });
-tests['cache'] = function() {
+});
+
+
+it('correctly caches remote files', function(done) {
var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'cache/cache.mml')));
// Set absolute paths dynamically at test time.
@@ -39,14 +144,36 @@ tests['cache'] = function() {
base: path.join(__dirname, 'cache'),
cache: path.join(__dirname, 'tmp')
};
+
+ // Cleanup from old test runs
+ try {
+ fs.unlinkSync(path.join(__dirname, 'cache/layers/absolute-json.json'));
+ rm(path.join(__dirname, 'cache/layers/absolute-shp'));
+ fs.unlinkSync(path.join(__dirname, 'cache/layers/polygons.json'));
+ fs.unlinkSync(path.join(__dirname, 'cache/layers/csv.csv'));
+ rm(path.join(__dirname, 'cache/layers/zip-no-ext'));
+ } catch (e) {}
+
+ // Copy "cached" files to mock request headers
+ try {
+ fs.mkdirSync(options.cache, 0777);
+ fs.mkdirSync(path.join(options.cache, '9368bdd9-zip_no_ext'),0777);
+ } catch(e) { console.log("mkdirSync failed with: " + e ); }
+ var files = ['9368bdd9-zip_no_ext/9368bdd9-zip_no_ext', '9368bdd9-zip_no_ext/.9368bdd9-zip_no_ext'];
+ for (var i = 0; i < files.length; i++) {
+ var newFile = fs.createWriteStream(path.join(options.cache, files[i]));
+ var oldFile = fs.createReadStream(path.join(__dirname, 'data', files[i]));
+ oldFile.pipe(newFile);
+ }
+
millstone.resolve(options, function(err, resolved) {
- assert.equal(err.message, "Unable to determine SRS for layer \"sqlite-attach\" at " + path.join(__dirname, "cache/layers/countries.sqlite"));
+ if (err) throw err;
assert.deepEqual(resolved.Stylesheet, [
- { id:'cache-inline.mss', data:'Map { backgroound-color:#fff }' },
+ { id:'cache-inline.mss', data:'Map { background-color:#fff }' },
{ id:'cache-local.mss', data: '#world { polygon-fill: #fff }\n' },
{ id:'cache-url.mss', data:'#world { line-width:1; }\n' }
]);
- assert.deepEqual(resolved.Layer, [
+ var expected = [
{
"name": "local-json",
"Datasource": {
@@ -54,7 +181,7 @@ tests['cache'] = function() {
"type": "ogr",
"layer_by_index": 0
},
- "srs": "+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs"
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
},
{
"name": "local-shp",
@@ -71,7 +198,7 @@ tests['cache'] = function() {
"type": "ogr",
"layer_by_index": 0
},
- "srs": "+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs"
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
},
{
"name": "absolute-shp",
@@ -88,7 +215,7 @@ tests['cache'] = function() {
"type": "ogr",
"layer_by_index": 0
},
- "srs": "+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs"
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
},
{
"name": "stations",
@@ -101,8 +228,7 @@ tests['cache'] = function() {
{
"name": "csv",
"Datasource": {
- "file": path.join(__dirname, 'cache/layers/csv'),
- "quiet": true,
+ "file": path.join(__dirname, 'cache/layers/csv.csv'),
"type": "csv"
},
"srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
@@ -113,7 +239,8 @@ tests['cache'] = function() {
"file": path.join(__dirname, 'cache/layers/countries.sqlite'),
"type": 'sqlite',
"table": 'countries',
- }
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
},
{
"name": 'sqlite-attach',
@@ -122,15 +249,27 @@ tests['cache'] = function() {
"type": 'sqlite',
"table": 'countries',
"attachdb": 'data@' + path.join(__dirname, 'cache/layers/data.sqlite'),
- }
+ },
+ "srs": "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
+ },
+ {
+ "name": 'zip-no-ext',
+ "Datasource": {
+ "file": path.join(__dirname, 'cache/layers/zip-no-ext/9368bdd9-zip_no_ext.shp'),
+ "type": 'shape'
+ },
+ "srs": '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'
}
- ]);
+ ];
+ for (var i=0;i<=10;i++) {
+ assert.deepEqual(resolved.Layer[i], expected[i]);
+ }
// Check that URLs are downloaded and symlinked.
- assert.ok(path.existsSync(path.join(__dirname, 'tmp/5c505ff4-polygons.json')));
- assert.ok(path.existsSync(path.join(__dirname, 'tmp/87c0c757-stations/87c0c757-stations.shp')));
+ assert.ok(existsSync(path.join(__dirname, 'tmp/5c505ff4-polygons.json')));
+ assert.ok(existsSync(path.join(__dirname, 'tmp/87c0c757-stations/87c0c757-stations.shp')));
assert.ok(fs.lstatSync(path.join(__dirname, 'cache/layers/polygons.json')).isSymbolicLink());
- assert.ok(fs.lstatSync(path.join(__dirname, 'cache/layers/stations')).isSymbolicLink());
+ assert.ok(fs.lstatSync(path.join(__dirname, 'cache/layers/stations')).isDirectory());
assert.equal(
fs.readFileSync(path.join(__dirname, 'tmp/5c505ff4-polygons.json'), 'utf8'),
fs.readFileSync(path.join(__dirname, 'cache/layers/polygons.json'), 'utf8')
@@ -142,7 +281,7 @@ tests['cache'] = function() {
// Check that absolute paths are symlinked correctly.
assert.ok(fs.lstatSync(path.join(__dirname, 'cache/layers/absolute-json.json')).isSymbolicLink());
- assert.ok(fs.lstatSync(path.join(__dirname, 'cache/layers/absolute-shp')).isSymbolicLink());
+ assert.ok(fs.lstatSync(path.join(__dirname, 'cache/layers/absolute-shp')).isDirectory());
assert.equal(
fs.readFileSync(path.join(__dirname, 'cache/layers/absolute-json.json'), 'utf8'),
fs.readFileSync(path.join(__dirname, 'data/absolute.json'), 'utf8')
@@ -161,20 +300,54 @@ tests['cache'] = function() {
assert.equal(err, undefined);
// Polygons layer and cache should still exist.
- assert.ok(path.existsSync(path.join(__dirname, 'cache/layers/polygons.json')));
- assert.ok(path.existsSync(path.join(__dirname, 'tmp/5c505ff4-polygons.json')));
+ assert.ok(existsSync(path.join(__dirname, 'cache/layers/polygons.json')));
+ assert.ok(existsSync(path.join(__dirname, 'tmp/5c505ff4-polygons.json')));
// Stations layer and cache should be gone.
- assert.ok(!path.existsSync(path.join(__dirname, 'layers/stations')));
- assert.ok(!path.existsSync(path.join(__dirname, 'tmp/87c0c757-stations')));
-
- // Cleanup.
- rm(path.join(__dirname, 'tmp'));
- fs.unlinkSync(path.join(__dirname, 'cache/layers/absolute-json.json'));
- fs.unlinkSync(path.join(__dirname, 'cache/layers/absolute-shp'));
- fs.unlinkSync(path.join(__dirname, 'cache/layers/polygons.json'));
- fs.unlinkSync(path.join(__dirname, 'cache/layers/csv'));
+ assert.ok(!existsSync(path.join(__dirname, 'layers/stations')));
+ assert.ok(!existsSync(path.join(__dirname, 'tmp/87c0c757-stations')));
+
+ done();
+ });
+ });
+});
+
+describe('util', function() {
+
+ var copypath = path.join(__dirname, 'copypath');
+ var cache = path.join(__dirname, 'tmp');
+
+ beforeEach(function() {
+ if (!existsSync(copypath)) fs.mkdirSync(copypath,0777);
+ });
+
+ afterEach(function() {
+ rm(copypath);
+ });
+
+ it('copies all files from shapefiles (and no extras)', function(done) {
+
+ utils.processFiles(path.join(__dirname, 'data/absolute/absolute.shp'), path.join(copypath, 'absolute/absolute.shp'), utils.copy, {cache:cache}, function(err) {
+ assert.ok(!err);
+
+ assert.ok(existsSync(path.join(copypath, 'absolute/absolute.shp')));
+ assert.ok(existsSync(path.join(copypath, 'absolute/absolute.dbf')));
+ assert.ok(existsSync(path.join(copypath, 'absolute/absolute.shx')));
+ assert.ok(existsSync(path.join(copypath, 'absolute/absolute.prj')));
+ assert.ok(existsSync(path.join(copypath, 'absolute/absolute.index')));
+ assert.ok(!existsSync(path.join(copypath, 'absolute/othername.shp')));
+
+ done();
+ });
+
+ });
+
+ it('copies single files correctly', function(done) {
+ utils.copy(path.join(__dirname, 'data/absolute.json'), path.join(copypath, 'absolute.json'), {cache:cache}, function(err) {
+ assert.equal(err, undefined);
+ assert.ok(existsSync(path.join(copypath, 'absolute.json')));
+ done();
});
});
-};
+});
diff --git a/test/uppercase-ext.test.js b/test/uppercase-ext.test.js
new file mode 100644
index 0000000..bb3f601
--- /dev/null
+++ b/test/uppercase-ext.test.js
@@ -0,0 +1,49 @@
+var fs = require('fs');
+var path = require('path');
+var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+var utils = require('../lib/util.js');
+var millstone = require('../lib/millstone');
+var tests = module.exports = {};
+var rm = require('./support.js').rm;
+
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+beforeEach(function(){
+ rm(path.join(__dirname, '/tmp/millstone-test'));
+})
+
+it('correctly handles datasources with uppercase extensions', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'UPPERCASE_EXT/project.mml')));
+
+ var cache = '/tmp/millstone-test';
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'UPPERCASE_EXT'),
+ cache: cache
+ };
+
+ try {
+ fs.mkdirSync(options.cache, 0777);
+ } catch (e) {}
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.equal(err,undefined,err);
+ assert.equal(resolved.Stylesheet[0].id, 'style.mss');
+ assert.equal(resolved.Stylesheet[0].data, '#polygon { }');
+ var expected = [
+ {
+ "name": "uppercase-ext",
+ "Datasource": {
+ "file": path.join(__dirname, 'UPPERCASE_EXT/test1.CSV'),
+ "type": "csv"
+ },
+ "srs": '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs'
+ }
+ ];
+ assert.deepEqual(resolved.Layer, expected);
+ done();
+ });
+});
diff --git a/test/zip-with-shapefile-and-txt.test.js b/test/zip-with-shapefile-and-txt.test.js
new file mode 100644
index 0000000..60f5a19
--- /dev/null
+++ b/test/zip-with-shapefile-and-txt.test.js
@@ -0,0 +1,38 @@
+var fs = require('fs');
+var path = require('path');
+var assert = require('assert');
+
+// switch to 'development' for more verbose logging
+process.env.NODE_ENV = 'production'
+var utils = require('../lib/util.js');
+var millstone = require('../lib/millstone');
+var tests = module.exports = {};
+var rm = require('./support.js').rm;
+
+var existsSync = require('fs').existsSync || require('path').existsSync;
+
+beforeEach(function(){
+ rm(path.join(__dirname, '/tmp/millstone-test'));
+})
+
+// https://github.com/mapbox/millstone/issues/101
+it('correctly prefers (for back compatibility) shapefiles over .txt files in zip archive', function(done) {
+ var mml = JSON.parse(fs.readFileSync(path.join(__dirname, 'zip-with-shapefile-and-txt/project.mml')));
+
+ var cache = '/tmp/millstone-test';
+ var options = {
+ mml: mml,
+ base: path.join(__dirname, 'zip-with-shapefile-and-txt'),
+ cache: cache
+ };
+
+ try {
+ fs.mkdirSync(options.cache, 0777);
+ } catch (e) {}
+
+ millstone.resolve(options, function(err, resolved) {
+ assert.equal(err,undefined,err);
+ assert.deepEqual(resolved.Layer[0].Datasource.type, 'shape');
+ done();
+ });
+});
diff --git a/test/zip-with-shapefile-and-txt/project.mml b/test/zip-with-shapefile-and-txt/project.mml
new file mode 100644
index 0000000..0517d5f
--- /dev/null
+++ b/test/zip-with-shapefile-and-txt/project.mml
@@ -0,0 +1,11 @@
+{
+ "Stylesheet": [ ],
+ "Layer": [
+ {
+ "name": "zip-with-shapefile-and-txt",
+ "Datasource": {
+ "file": "../data/ne_10m_admin_0_boundary_lines_disputed_areas.zip"
+ }
+ }
+ ]
+}
diff --git a/test/zipped-json/project.mml b/test/zipped-json/project.mml
new file mode 100644
index 0000000..636e559
--- /dev/null
+++ b/test/zipped-json/project.mml
@@ -0,0 +1,13 @@
+{
+ "Stylesheet": [
+ "style.mss"
+ ],
+ "Layer": [
+ {
+ "name": "polygons-zipped",
+ "Datasource": {
+ "file": "http://mapbox.github.com/millstone/test/polygons.json.zip"
+ }
+ }
+ ]
+}
diff --git a/test/zipped-json/style.mss b/test/zipped-json/style.mss
new file mode 100644
index 0000000..230a97c
--- /dev/null
+++ b/test/zipped-json/style.mss
@@ -0,0 +1 @@
+#polygon { }
\ No newline at end of file
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/collab-maint/node-millstone.git
More information about the Pkg-javascript-commits
mailing list