From 05895ec36d2ae590f7de9febc9a76ad0cec2783f Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Fri, 27 May 2011 14:45:02 -0400 Subject: [PATCH] First pass, server-side bones stuff only. --- server/bootstrap.js => commands/global.bones | 64 +- index.js | 4 + {server => lib}/api.js | 0 {server => lib}/export-worker.js | 0 {server => lib}/library-directory.js | 0 {server => lib}/library-s3.js | 0 models/Asset.bones | 25 + models/Assets.bones | 50 + models/AssetsS3.bones | 46 + models/Datasource.bones | 118 +++ models/Export.bones | 82 ++ models/Export.server.bones | 82 ++ models/Exports.bones | 11 + models/Layer.bones | 78 ++ models/Layers.bones | 24 + models/Libraries.bones | 11 + models/Library.bones | 78 ++ models/Project.bones | 197 ++++ .../Project.server.bones | 92 +- models/Projects.bones | 11 + models/Projects.server.bones | 1 + models/Stylesheet.bones | 27 + models/Stylesheets.bones | 24 + models/models.js | 89 ++ package.json | 4 +- server/tiles.js | 66 -- servers/App.bones | 34 + servers/Core.bones | 9 + servers/Route.bones | 26 + servers/Tile.bones | 51 + settings.js | 17 - shared/models.js | 883 ------------------ tests.sh | 3 - tilemill.js | 87 -- 34 files changed, 1118 insertions(+), 1176 deletions(-) rename server/bootstrap.js => commands/global.bones (62%) create mode 100755 index.js rename {server => lib}/api.js (100%) rename {server => lib}/export-worker.js (100%) rename {server => lib}/library-directory.js (100%) rename {server => lib}/library-s3.js (100%) create mode 100644 models/Asset.bones create mode 100644 models/Assets.bones create mode 100644 models/AssetsS3.bones create mode 100644 models/Datasource.bones create mode 100644 models/Export.bones create mode 100644 models/Export.server.bones create mode 100644 models/Exports.bones create mode 100644 models/Layer.bones create mode 100644 models/Layers.bones create mode 100644 models/Libraries.bones create mode 100644 models/Library.bones create mode 100644 models/Project.bones rename server/models-server.js => models/Project.server.bones (79%) create mode 100644 models/Projects.bones create mode 100644 models/Projects.server.bones create mode 100644 models/Stylesheet.bones create mode 100644 models/Stylesheets.bones create mode 100644 models/models.js delete mode 100644 server/tiles.js create mode 100644 servers/App.bones create mode 100644 servers/Core.bones create mode 100644 servers/Route.bones create mode 100644 servers/Tile.bones delete mode 100644 settings.js delete mode 100644 shared/models.js delete mode 100755 tests.sh delete mode 100755 tilemill.js diff --git a/server/bootstrap.js b/commands/global.bones similarity index 62% rename from server/bootstrap.js rename to commands/global.bones index 229d97a02..9ba6717ee 100644 --- a/server/bootstrap.js +++ b/commands/global.bones @@ -1,39 +1,49 @@ -// Application bootstrap. Ensures that files directories exist at server start. var fs = require('fs'), path = require('path'), - Step = require('step'); + Step = require('step'), + settings = Bones.plugin.config; -module.exports = function(app, settings) { - try { - fs.statSync(settings.files); - } catch (Exception) { - console.log('Creating files dir %s', settings.files); - fs.mkdirSync(settings.files, 0777); - } +Bones.Command.options['port'] = { + 'title': 'port=[port]', + 'description': 'Server port.', + 'default': 8889 +}; - try { - fs.statSync(settings.mapfile_dir); - } catch (Exception) { - console.log('Creating mapfile dir %s', settings.mapfile_dir); - fs.mkdirSync(settings.mapfile_dir, 0777); +Bones.Command.options['files'] = { + 'title': 'files=[path]', + 'description': 'Path to files directory.', + 'default': path.join(process.cwd(), 'files') +}; + +Bones.Command.options['export'] = { + 'title': 'export=[path]', + 'description': 'Path to export directory.', + 'default': path.join(process.cwd(), 'files', 'export') +}; + +Bones.Command.augment({ + bootstrap: function(parent, plugin, callback) { + parent.call(this, plugin, function() { + bootstrap(callback); + }); } +}); +var bootstrap = function(callback) { try { - fs.statSync(settings.data_dir); + fs.statSync(settings.files); } catch (Exception) { - console.log('Creating data dir %s', settings.data_dir); - fs.mkdirSync(settings.data_dir, 0777); + console.log('Creating files dir %s', settings.files); + fs.mkdirSync(settings.files, 0777); } - try { - fs.statSync(settings.export_dir); + fs.statSync(settings.export); } catch (Exception) { - console.log('Creating export dir %s', settings.export_dir); - fs.mkdirSync(settings.export_dir, 0777); + console.log('Creating export dir %s', settings.export); + fs.mkdirSync(settings.export, 0777); } // @TODO: Better infrastructure for handling updates. - // Update 1: Migrate to new backbone-dirty key format. try { var db = fs.readFileSync(settings.files + '/app.db', 'utf8'); @@ -47,13 +57,10 @@ module.exports = function(app, settings) { } catch (Exception) {} // Apply server-side mixins/overrides. - var Backbone = require('backbone'); var sync = require('backbone-dirty')(settings.files + '/app.db').sync; Backbone.sync = sync; - require('models-server'); // Create a default library for the local data directory. - var models = require('models'); var data = new models.Library({ id: 'data', name: 'Local data', @@ -66,16 +73,17 @@ module.exports = function(app, settings) { function() { if (!data.get('directory_path')) { data.save({ - 'directory_path': path.join(__dirname, '..', 'files', 'data') + directory_path: path.join(settings.files, 'data') }); } } ); // Process any waiting exports. - (new models.ExportList).fetch({success: function(collection) { + (new models.Exports).fetch({success: function(collection) { collection.each(function(model) { model.get('status') === 'waiting' && model.process(); }); }}); -} + callback(); +}; diff --git a/index.js b/index.js new file mode 100755 index 000000000..e777f1976 --- /dev/null +++ b/index.js @@ -0,0 +1,4 @@ +#!/usr/bin/env node +process.title = 'tilemill'; +require('bones').load(__dirname); +!module.parent && require('bones').start(); diff --git a/server/api.js b/lib/api.js similarity index 100% rename from server/api.js rename to lib/api.js diff --git a/server/export-worker.js b/lib/export-worker.js similarity index 100% rename from server/export-worker.js rename to lib/export-worker.js diff --git a/server/library-directory.js b/lib/library-directory.js similarity index 100% rename from server/library-directory.js rename to lib/library-directory.js diff --git a/server/library-s3.js b/lib/library-s3.js similarity index 100% rename from server/library-s3.js rename to lib/library-s3.js diff --git a/models/Asset.bones b/models/Asset.bones new file mode 100644 index 000000000..13f34a875 --- /dev/null +++ b/models/Asset.bones @@ -0,0 +1,25 @@ +// Asset +// ----- +// Model. Single external asset, e.g. a shapefile, image, etc. +model = Backbone.Model.extend({ + schema: { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + 'required': true + }, + 'url': { + 'type': 'string', + 'required': true + }, + 'bytes': { + 'type': 'string' + } + } + }, + extension: function() { + return this.id.split('.').pop(); + } +}); + diff --git a/models/Assets.bones b/models/Assets.bones new file mode 100644 index 000000000..04f7c1f4d --- /dev/null +++ b/models/Assets.bones @@ -0,0 +1,50 @@ +// AssetList +// --------- +// Collection. List of all assets for a given Library. Must be given a +// Library model at `options.library` in order to determine its URL endpoint. +// The REST endpoint for a LibraryList collection must return an array of asset +// models, or may optionally return a more complex object suited for handling +// pagination: +// +// { +// page: 0, // The current page number +// pageTotal: 10, // The total number of pages +// models: [] // An array of asset models +// } +model = Backbone.Collection.extend({ + model: models.Asset, + url: function() { + return 'api/Library/' + this.library.id + '/assets/' + this.page; + }, + initialize: function(options) { + this.page = 0; + this.pageTotal = 1; + this.library = options.library; + }, + parse: function(response) { + if (_.isArray(response)) { + return response; + } else { + this.page = response.page; + this.pageTotal = response.pageTotal; + return response.models; + } + }, + hasNext: function() { + return this.page < (this.pageTotal - 1); + }, + hasPrev: function() { + return this.page > 0; + }, + nextPage: function(options) { + if (!this.hasNext()) return; + this.page++; + this.fetch(options); + }, + prevPage: function(options) { + if (!this.hasPrev()) return; + this.page--; + this.fetch(options); + } +}); + diff --git a/models/AssetsS3.bones b/models/AssetsS3.bones new file mode 100644 index 000000000..8b7ddbaf8 --- /dev/null +++ b/models/AssetsS3.bones @@ -0,0 +1,46 @@ +// AssetListS3 +// ----------- +// Collection. Override of AssetList for S3 library. S3 uses a marker key +// system for pagination instead of a page # system. +model = models.Assets.extend({ + url: function() { + var url = 'api/Library/' + this.library.id + '/assets'; + if (this.marker()) { + url += '/' + Base64.encodeURI(this.marker()); + } + return url; + }, + initialize: function(options) { + this.markers = []; + this.library = options.library; + }, + marker: function() { + if (this.markers.length) { + return this.markers[this.markers.length - 1]; + } + return false; + }, + parse: function(response) { + if (this.marker() != response.marker) { + this.markers.push(response.marker); + } + return response.models; + }, + hasNext: function() { + return this.marker(); + }, + hasPrev: function() { + return this.markers.length > 1; + }, + nextPage: function(options) { + if (!this.hasNext()) return; + this.fetch(options); + }, + prevPage: function(options) { + if (!this.hasPrev()) return; + this.markers.pop(); + this.markers.pop(); + this.fetch(options); + } +}); + diff --git a/models/Datasource.bones b/models/Datasource.bones new file mode 100644 index 000000000..2cffe037b --- /dev/null +++ b/models/Datasource.bones @@ -0,0 +1,118 @@ +// Datasource (read-only) +// ---------------------- +// Model. Inspection metadata about a map layer. Use `fetchFeatures()` to do +// a datasource fetch that includes layer feature objects. +model = Backbone.Model.extend({ + // @TODO either as a feature or a bug, object attributes are not set + // automatically when passed to the constructor. We set it manually here. + initialize: function(attributes) { + this.set({'fields': attributes.fields}); + }, + url: function() { + var url = 'api/Datasource'; + this.getFeatures && (url += '/features'); + if (typeof module === 'undefined') { + url += '?' + $.param(this.attributes); + } + return url; + }, + fetchFeatures: function(options) { + this.getFeatures = true; + this.fetch(options); + } +}); + +/* +// FileDatasource (read-only) +// -------------------------- +// Model. Inspection metadata about a map layer. Use `fetchFeatures()` to do +// a datasource fetch that includes layer feature objects. +var FileDatasource = Datasource.extend({ + schema: { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + }, + 'project': { + 'required': true, + 'type': 'string', + 'description': 'Project to which this datasource belongs.' + }, + 'url': { + 'type': 'string', + 'required': true, + 'minLength': 1, + 'title': 'URL', + 'description': 'URL of the datasource.' + }, + 'fields': { + 'type': 'object' + }, + 'features': { + 'type': 'array' + }, + 'ds_options': { + 'type': 'object' + }, + 'ds_type': { + 'type': 'string' + }, + 'geometry_type': { + 'type': 'string', + 'enum': ['polygon', 'point', 'linestring', 'raster', 'unknown'] + } + } + } +}); + +// PostgisDatasource (read-only) +// ----------------------------- +// Model. Inspection metadata about a map layer. Use `fetchFeatures()` to do +// a datasource fetch that includes layer feature objects. +var PostgisDatasource = Datasource.extend({ + schema: { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + }, + 'project': { + 'required': true, + 'type': 'string', + 'description': 'Project to which this datasource belongs.' + }, + 'dbname': { + 'type': 'string', + 'required': true, + 'minLength': 1, + 'title': 'Database', + 'description': 'Invalid PostGIS database.' + }, + 'table': { + 'type': 'string', + 'required': true, + 'minLength': 1, + 'title': 'Query', + 'description': 'Invalid PostGIS query.' + }, + 'fields': { + 'type': 'object' + }, + 'features': { + 'type': 'array' + }, + 'ds_options': { + 'type': 'object' + }, + 'ds_type': { + 'type': 'string' + }, + 'geometry_type': { + 'type': 'string', + 'enum': ['polygon', 'point', 'linestring', 'raster', 'unknown'] + } + } + } +}); +*/ diff --git a/models/Export.bones b/models/Export.bones new file mode 100644 index 000000000..1726c06e0 --- /dev/null +++ b/models/Export.bones @@ -0,0 +1,82 @@ +// Export +// ------ +// Model. Describes a single export task, e.g. rendering a map to a PDF. +model = Backbone.Model.extend({ + schema: { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + 'required': true + }, + 'project': { + 'type': 'string', + 'required': true + }, + 'format': { + 'type': 'string', + 'required': true, + 'enum': ['png', 'pdf', 'mbtiles'] + }, + 'status': { + 'type': 'string', + 'required': true, + 'enum': ['waiting', 'processing', 'complete', 'error'] + }, + 'progress': { + 'type': 'number', + 'minimum': 0, + 'maximum': 1 + }, + 'filename': { + 'type': 'string', + 'pattern': '^[A-Za-z0-9\-_.]+$' + }, + 'created': { + 'type': 'integer' + }, + 'updated': { + 'type': 'integer' + }, + 'error': { + 'type': 'string' + } + } + }, + initialize: function() { + if (this.isNew()){ + this.set({ + created: +new Date, + id: (+new Date) + '' + }, {silent: true}); + } + }, + url: function() { + return 'api/Export/' + this.id; + }, + defaults: { + progress: 0, + status: 'waiting' + }, + // Generate a download URL for an Export. + downloadURL: function() { + return (this.get('status') === 'complete') && 'export/download/' + this.get('filename'); + }, + // Get the duration of the current export job. + time: function() { + if (this.get('updated')) { + var seconds = parseInt((this.get('updated') - this.get('created')) * .001, 10); + var minutes = parseInt(seconds / 60, 10); + var remainder = seconds - (minutes * 60); + if (minutes && remainder) { + return minutes + ' min ' + remainder + ' sec'; + } else if (minutes) { + return minutes + ' min'; + } else { + return seconds + ' sec'; + } + } + return '0 sec'; + } +}); + diff --git a/models/Export.server.bones b/models/Export.server.bones new file mode 100644 index 000000000..5d5730024 --- /dev/null +++ b/models/Export.server.bones @@ -0,0 +1,82 @@ +var Step = require('step'), + Pool = require('generic-pool').Pool, + Worker = require('worker').Worker, + fs = require('fs'), + path = require('path'); + +// Export +// ------ +// Implement custom sync method for Export model. Removes any files associated +// with the export model at `filename` when a model is destroyed. +var workers = []; +var pool = Pool({ + create: function(callback) { + callback(null, new Worker(require.resolve('./export-worker.js'))); + }, + destroy: function(worker) { + worker.terminate(); + }, + max: 3, + idleTimeoutMillis: 5000 +}); + +models.Export.prototype.sync = function(method, model, success, error) { + switch (method) { + case 'delete': + Step(function() { + Backbone.sync('read', model, this, this); + }, + function(data) { + if (data && data.filename) { + var filepath = path.join(settings.export_dir, data.filename); + path.exists(filepath, function(exists) { + exists && fs.unlink(filepath, this) || this(); + }.bind(this)); + } else { + this(false); + } + }, + function() { + Backbone.sync(method, model, success, error); + }); + break; + case 'read': + Backbone.sync('read', model, function(data) { + if (data.status === 'processing' && !workers[model.id]) { + data.status = 'error'; + data.error = 'Export did not complete.'; + } + success(data); + }, error); + break; + case 'create': + case 'update': + model.get('status') === 'waiting' && model.process(); + Backbone.sync(method, model, success, error); + break; + } +}; + +models.Export.prototype.process = function() { + var model = this; + pool.acquire(function(err, worker) { + if (err) return callback(err); + workers[model.id] = worker; + worker.on('message', function(data) { + if (data.event === 'complete') { + worker.removeAllListeners('message'); + pool.release(worker); + } else if (data.event === 'update') { + model.save(data.attributes); + } + }); + worker.postMessage(_(model.toJSON()).extend({ + datasource: path.join( + settings.files, + 'project', + model.get('project'), + model.get('project') + '.mml' + ) + })); + }); +}; diff --git a/models/Exports.bones b/models/Exports.bones new file mode 100644 index 000000000..4561972f0 --- /dev/null +++ b/models/Exports.bones @@ -0,0 +1,11 @@ +// ExportList +// ---------- +// Collection. List of all Exports. +model = Backbone.Collection.extend({ + model: models.Export, + url: 'api/Export', + comparator: function(job) { + return job.get('created'); + } +}); + diff --git a/models/Layer.bones b/models/Layer.bones new file mode 100644 index 000000000..e3389f1e1 --- /dev/null +++ b/models/Layer.bones @@ -0,0 +1,78 @@ +// Layer +// ----- +// Model. Represents a single map layer. This model is a child of +// the Project model and is saved serialized as part of the parent. +// **This model is not backed directly by the server.** +model = Backbone.Model.extend({ + schema: { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + 'required': true, + 'pattern': '^[A-Za-z0-9\-_]+$', + 'title': 'ID', + 'description': 'ID may include alphanumeric characters, dashes and underscores.' + }, + 'class': { + 'type': 'string', + 'pattern': '^[A-Za-z0-9\-_ ]*$', + 'title': 'Class', + 'description': 'Class may include alphanumeric characters, spaces, dashes and underscores.' + }, + 'srs': { + 'type': 'string' + }, + 'geometry': { + 'type': 'string', + 'enum': ['polygon', 'point', 'linestring', 'raster', 'unknown'] + }, + 'Datasource': { + 'type': 'object', + 'required': true + } + } + }, + // @TODO either as a feature or a bug, object attributes are not set + // automatically when passed to the constructor. We set it manually here. + initialize: function(attributes) { + this.set({'Datasource': attributes.Datasource}); + }, + // Constant. Hash of simple names to known SRS strings. + SRS: { + // note: 900913 should be the same as EPSG 3857 + '900913': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over', + 'WGS84': '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs' + }, + // Get the name of a model's SRS string if known, otherwise reteurn + // 'custom' or 'autodetect' if empty. + srsName: function() { + for (name in this.SRS) { + if (this.SRS[name] === this.get('srs')) { + return name; + } + } + return this.get('srs') ? 'custom' : 'autodetect'; + }, + // Implementation of `Model.set()` that allows a datasource model to be + // passed in as `options.datasource`. If provided, the datasource will be + // used to enforce key attributes. + set: function(attributes, options) { + if (options && options.datasource) { + if (options.datasource.get('ds_type') === 'gdal') { + attributes.srs = this.SRS['900913']; + } + if (options.datasource.get('geometry_type')) { + attributes.geometry = options.datasource.get('geometry_type'); + } + if (options.datasource.get('ds_options')) { + attributes.Datasource = _.extend( + attributes.Datasource, + options.datasource.get('ds_options') + ); + } + } + return Backbone.Model.prototype.set.call(this, attributes, options); + } +}); + diff --git a/models/Layers.bones b/models/Layers.bones new file mode 100644 index 000000000..f7b1f0bd2 --- /dev/null +++ b/models/Layers.bones @@ -0,0 +1,24 @@ +// LayerList +// --------- +// Collection. List of Layer models. This collection is a child of the +// Project model and updates its parent on update events. +// **This collection is not backed directly by the server.** +model = Backbone.Collection.extend({ + model: models.Layer, + initialize: function(models, options) { + var self = this; + this.parent = options.parent; + this.bind('change', function() { + this.parent.set({ 'Layer': self }); + this.parent.change(); + }); + this.bind('add', function() { + this.parent.set({ 'Layer': self }); + this.parent.change(); + }); + this.bind('remove', function() { + this.parent.set({ 'Layer': self }); + this.parent.change(); + }); + } +}); diff --git a/models/Libraries.bones b/models/Libraries.bones new file mode 100644 index 000000000..397350163 --- /dev/null +++ b/models/Libraries.bones @@ -0,0 +1,11 @@ +// LibraryList +// ------------ +// Collection. All librarys. +model = Backbone.Collection.extend({ + model: models.Library, + url: 'api/Library', + comparator: function(model) { + return model.get('name'); + } +}); + diff --git a/models/Library.bones b/models/Library.bones new file mode 100644 index 000000000..62212fe78 --- /dev/null +++ b/models/Library.bones @@ -0,0 +1,78 @@ +// Library +// -------- +// Model. Stores settings for a given asset library type, e.g. a local file +// directory or an Amazon S3 bucket. +model = Backbone.Model.extend({ + schema: { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + 'required': true, + 'minLength': 1 + }, + 'type': { + 'type': 'string', + 'required': true, + 'enum': ['s3', 'directory'] + }, + 'name': { + 'type': 'string', + 'required': true, + 'minLength': 1 + }, + 's3_bucket': { + 'type': 'string', + 'required': true, + 'minLength': 1 + }, + 'directory_path': { + 'type': 'string', + 'required': true, + 'minLength': 1 + } + }, + 'dependencies': { + 's3_bucket': { + 'properties': { + 'type': { 'enum': ['s3'], 'required': true } + } + }, + 's3_key': { + 'properties': { + 'type': { 'enum': ['s3'], 'required': true } + } + }, + 's3_secret': { + 'properties': { + 'type': { 'enum': ['s3'], 'required': true } + } + }, + 'directory_path': { + 'properties': { + 'type': { 'enum': ['directory'], 'required': true } + } + } + } + }, + url: function() { + return 'api/Library/' + this.id; + }, + defaults: { + type: 'directory' + }, + initialize: function(options) { + if (this.isNew()){ + this.set({id: (+new Date) + ''}, {silent: true}); + } + switch (this.get('type')) { + case 's3': + this.assets = new models.AssetsS3({ library: this }); + break; + default: + this.assets = new models.Assets({ library: this }); + break; + } + } +}); + diff --git a/models/Project.bones b/models/Project.bones new file mode 100644 index 000000000..f4a71f806 --- /dev/null +++ b/models/Project.bones @@ -0,0 +1,197 @@ +// Project +// ------- +// Model. A single TileMill map project. Describes an MML JSON map object that +// can be used by `carto` to render a map. +model = Backbone.Model.extend({ + schema: { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + 'required': true, + 'pattern': '^[A-Za-z0-9\-_]+$', + 'title': 'Name', + 'description': 'Name may include alphanumeric characters, dashes and underscores.' + }, + 'srs': { + 'type': 'string', + 'required': true + }, + 'Stylesheet': { + 'type': ['object', 'array'], + 'required': true + }, + 'Layer': { + 'type': ['object', 'array'], + 'required': true + }, + '_format': { + 'type': 'string', + 'enum': ['png', 'png24', 'png8', 'jpeg80', 'jpeg85', 'jpeg90', 'jpeg95'] + }, + '_center': { + 'type': 'object' + }, + '_interactivity': { + 'type': ['object', 'boolean'] + }, + '_updated': { + 'type': 'integer' + } + } + }, + STYLESHEET_DEFAULT: [{ + id: 'style.mss', + data: 'Map {\n' + + ' background-color: #fff;\n' + + '}\n\n' + + '#world {\n' + + ' polygon-fill: #eee;\n' + + ' line-color: #ccc;\n' + + ' line-width: 0.5;\n' + + '}' + }], + LAYER_DEFAULT: [{ + id: 'world', + name: 'world', + srs: '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 ' + + '+lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over', + geometry: 'polygon', + Datasource: { + file: 'http://tilemill-data.s3.amazonaws.com/world_borders_merc.zip', + type: 'shape' + } + }], + defaults: { + '_center': { lat:0, lon:0, zoom:2 }, + '_format': 'png', + '_interactivity': false, + 'srs': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 ' + + '+lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over', + 'Stylesheet': [], + 'Layer': [] + }, + // Custom setDefaults() method for creating a project with default layers, + // stylesheets, etc. Note that we do not use Backbone native initialize() + // or defaults(), both of which make default values far pervasive than the + // expected use here. + setDefaults: function() { + var template = {}; + !this.get('Stylesheet').length && (template.Stylesheet = this.STYLESHEET_DEFAULT); + !this.get('Layer').length && (template.Layer = this.LAYER_DEFAULT); + this.set(template, { silent: true }); + }, + // Instantiate collections from arrays. + parse: function(resp) { + resp.Stylesheet && (resp.Stylesheet = new models.Stylesheets( + resp.Stylesheet, + {parent: this} + )); + resp.Layer && (resp.Layer = new models.Layers( + resp.Layer, + {parent: this} + )); + return resp; + }, + url: function() { + return 'api/Project/' + this.id; + }, + // Custom validation method that allows for asynchronous processing. + // Expects options.success and options.error callbacks to be consistent + // with other Backbone methods. + validateAsync: function(attributes, options) { + // If client-side, pass-through. + if (typeof require === 'undefined') { + return options.success(this, null); + } + + var carto = require('tilelive-mapnik/node_modules/carto'), + mapnik = require('tilelive-mapnik/node_modules/mapnik'), + that = this, + stylesheets = this.get('Stylesheet'), + env = { + returnErrors: true, + errors: [], + validation_data: { + fonts: mapnik.fonts() + }, + deferred_externals: [], + only_validate: true, + effects: [] + }; + + // Hard clone the model JSON before rendering as rendering will change + // properties (e.g. localize a datasource URL to the filesystem). + var data = JSON.parse(JSON.stringify(attributes)); + new carto.Renderer(env) + .render(data, function(err, output) { + if (err) { + options.error(that, err); + } else { + options.success(that, null); + } + }); + }, + // Interactivity: Convert teaser/full template markup into formatter js. + // Replaces tokens like `[NAME]` with string concatentations of `data.NAME` + // removes line breaks and escapes single quotes. + // @TODO properly handle other possible #fail. Maybe use underscore + // templating? + formatterJS: function() { + if (_.isEmpty(this.get('_interactivity'))) return; + + var full = this.get('_interactivity').template_full || ''; + var teaser = this.get('_interactivity').template_teaser || ''; + var location = this.get('_interactivity').template_location || ''; + full = full.replace(/\'/g, '\\\'').replace(/\[([\w\d]+)\]/g, "' + data.$1 + '").replace(/\n/g, ' '); + teaser = teaser.replace(/\'/g, '\\\'').replace(/\[([\w\d]+)\]/g, "' + data.$1 + '").replace(/\n/g, ' '); + location = location.replace(/\'/g, '\\\'').replace(/\[([\w\d]+)\]/g, "' + data.$1 + '").replace(/\n/g, ' '); + return "function(options, data) { " + + " switch (options.format) {" + + " case 'full': " + + " return '" + full + "'; " + + " break; " + + " case 'location': " + + " return '" + location + "'; " + + " break; " + + " case 'teaser': " + + " default: " + + " return '" + teaser + "'; " + + " break; " + + " }" + + "}"; + }, + // Interactivity: Retrieve array of field names to be included in + // interactive tiles by parsing `[field]` tokens. + formatterFields: function() { + if (_.isEmpty(this.get('_interactivity'))) return; + var fields = []; + var full = this.get('_interactivity').template_full || ''; + var teaser = this.get('_interactivity').template_teaser || ''; + fields = fields + .concat(full.match(/\[([\w\d]+)\]/g)) + .concat(teaser.match(/\[([\w\d]+)\]/g)); + fields = _(fields).chain() + .filter(_.isString) + .map(function(field) { return field.replace(/[\[|\]]/g, ''); }) + .uniq() + .value(); + return fields; + }, + // Single tile thumbnail URL generation. From [OSM wiki][1]. + // [1]: http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#lon.2Flat_to_tile_numbers_2 + thumb: function() { + var lat = this.get('_center').lat * -1; // TMS + var lon = this.get('_center').lon; + var z = this.get('_center').zoom; + var lat_rad = lat * Math.PI / 180; + var x = parseInt((lon + 180.0) / 360.0 * Math.pow(2, z)); + var y = parseInt( + (1.0 - + Math.log(Math.tan(lat_rad) + (1 / Math.cos(lat_rad))) / + Math.PI) / + 2.0 * Math.pow(2, z)); + return '/' + ['1.0.0', this.id, z, x, y].join('/') + '.png?updated=' + this.get('_updated'); + }, +}); + diff --git a/server/models-server.js b/models/Project.server.bones similarity index 79% rename from server/models-server.js rename to models/Project.server.bones index 5ffdf9457..8ffd50a1b 100644 --- a/server/models-server.js +++ b/models/Project.server.bones @@ -1,24 +1,14 @@ -// Server-side overrides for the Backbone models defined in `shared/models.js`. -// Provides model-specific storage overrides. -var _ = require('underscore')._, - Backbone = require('backbone'), - settings = require('settings'), - fs = require('fs'), +var fs = require('fs'), Step = require('step'), - Pool = require('generic-pool').Pool, - Worker = require('worker').Worker, path = require('path'), - models = require('models'), - constants = (!process.EEXIST >= 1) ? - require('constants') : - { EEXIST: process.EEXIST }; + constants = require('constants'), + settings = Bones.plugin.config; // Project // ------- // Implement custom sync method for Project model. Writes projects to // individual directories and splits out Stylesheets from the main project // MML JSON file. -models.ProjectList.prototype.sync = models.Project.prototype.sync = function(method, model, success, error) { switch (method) { case 'read': @@ -328,79 +318,3 @@ function saveProject(model, callback) { }); } -// Export -// ------ -// Implement custom sync method for Export model. Removes any files associated -// with the export model at `filename` when a model is destroyed. -var workers = []; -var pool = Pool({ - create: function(callback) { - callback(null, new Worker(require.resolve('./export-worker.js'))); - }, - destroy: function(worker) { - worker.terminate(); - }, - max: 3, - idleTimeoutMillis: 5000 -}); - -models.Export.prototype.sync = function(method, model, success, error) { - switch (method) { - case 'delete': - Step(function() { - Backbone.sync('read', model, this, this); - }, - function(data) { - if (data && data.filename) { - var filepath = path.join(settings.export_dir, data.filename); - path.exists(filepath, function(exists) { - exists && fs.unlink(filepath, this) || this(); - }.bind(this)); - } else { - this(false); - } - }, - function() { - Backbone.sync(method, model, success, error); - }); - break; - case 'read': - Backbone.sync('read', model, function(data) { - if (data.status === 'processing' && !workers[model.id]) { - data.status = 'error'; - data.error = 'Export did not complete.'; - } - success(data); - }, error); - break; - case 'create': - case 'update': - model.get('status') === 'waiting' && model.process(); - Backbone.sync(method, model, success, error); - break; - } -}; - -models.Export.prototype.process = function() { - var model = this; - pool.acquire(function(err, worker) { - if (err) return callback(err); - workers[model.id] = worker; - worker.on('message', function(data) { - if (data.event === 'complete') { - worker.removeAllListeners('message'); - pool.release(worker); - } else if (data.event === 'update') { - model.save(data.attributes); - } - }); - worker.postMessage(_(model.toJSON()).extend({ - datasource: path.join( - settings.files, - 'project', - model.get('project'), - model.get('project') + '.mml' - ) - })); - }); -}; diff --git a/models/Projects.bones b/models/Projects.bones new file mode 100644 index 000000000..9988aa0bc --- /dev/null +++ b/models/Projects.bones @@ -0,0 +1,11 @@ +// ProjectList +// ----------- +// Collection. All project models. +model = Backbone.Collection.extend({ + model: models.Project, + url: 'api/Project', + comparator: function(project) { + return project.get('id'); + } +}); + diff --git a/models/Projects.server.bones b/models/Projects.server.bones new file mode 100644 index 000000000..9cfadf3cc --- /dev/null +++ b/models/Projects.server.bones @@ -0,0 +1 @@ +models.Projects.prototype.sync = models.Project.prototype.sync; diff --git a/models/Stylesheet.bones b/models/Stylesheet.bones new file mode 100644 index 000000000..5d5e301c8 --- /dev/null +++ b/models/Stylesheet.bones @@ -0,0 +1,27 @@ +// Stylesheet +// ---------- +// Model. Represents a single map MSS stylesheet. This model is a child of +// the Project model and is saved serialized as part of the parent. +// **This model is not backed directly by the server.** +model = Backbone.Model.extend({ + schema: { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + 'required': true, + 'pattern': '^[A-Za-z0-9\-_.]+$', + 'title': 'Name', + 'description': 'Name may include alphanumeric characters, dots, dashes and underscores.' + }, + 'data': { + 'type': 'string', + 'required': true + } + } + }, + defaults: { + 'data': '' + } +}); + diff --git a/models/Stylesheets.bones b/models/Stylesheets.bones new file mode 100644 index 000000000..0f21531bd --- /dev/null +++ b/models/Stylesheets.bones @@ -0,0 +1,24 @@ +// StylesheetList +// -------------- +// Collection. List of Stylesheet models. This collection is a child of the +// Project model and updates its parent on update events. +// **This collection is not backed directly by the server.** +model = Backbone.Collection.extend({ + model: models.Stylesheet, + initialize: function(models, options) { + var that = this; + this.parent = options.parent; + this.bind('change', function() { + this.parent.set({ 'Stylesheet': that }); + this.parent.change(); + }); + this.bind('add', function() { + this.parent.set({ 'Stylesheet': that }); + this.parent.change(); + }); + this.bind('remove', function() { + this.parent.set({ 'Stylesheet': that }); + this.parent.change(); + }); + } +}); diff --git a/models/models.js b/models/models.js new file mode 100644 index 000000000..fc2da32f3 --- /dev/null +++ b/models/models.js @@ -0,0 +1,89 @@ +//// JSON schema validation +//// ---------------------- +//// Provide a default `validate()` method for all models. If a `schema` property +//// is defined on a model, use JSON-schema validation by default. +//Backbone.Model.prototype.validate = function(attributes) { +// if (!this.schema || !this.schema.properties) return; +// var env = JSV.createEnvironment(); +// for (var key in attributes) { +// if (this.schema.properties[key]) { +// var property = this.schema.properties[key], +// value = attributes[key]; +// // Do a custom check for required properties, (e.g. do not allow +// // an empty string to validate against a required property.) +// if (!value && property.required) { +// return (property.title || key) + ' is required.'; +// } + +// var errors = env.validate(value, property).errors; +// if (errors.length) { +// var error = errors.pop(); +// if (property.description) { +// return property.description; +// } else { +// return (property.title || key) + ': ' + error.message; +// } +// } +// } +// } +//}; + +//// Abilities (read-only) +//// --------------------- +//// Model. Describes server API abilities. +//var Abilities = Backbone.Model.extend({ +// schema: { +// 'type': 'object', +// 'properties': { +// 'fonts': { +// 'type': 'array', +// 'title': 'Fonts', +// 'description': 'Fonts available to Mapnik.' +// }, +// 'datasources': { +// 'type': 'array', +// 'title': 'Datasources', +// 'description': 'Datasource types available to Mapnik.' +// }, +// 'exports': { +// 'type': 'object', +// 'title': 'Exports', +// 'description': 'Export types available to Mapnik.' +// } +// } +// }, +// url: 'api/Abilities' +//}); + +//// Reference (read-only) +//// --------------------- +//// Model. MSS syntax reference. +//var Reference = Backbone.Model.extend({ +// url: 'api/Reference' +//}); + +//// Settings +//// -------- +//// Model. Stores any user-specific configuration related to the app. +//var Settings = Backbone.Model.extend({ +// schema: { +// 'type': 'object', +// 'properties': { +// 'id': { +// 'type': 'string', +// 'required': true, +// 'enum': ['settings'] +// }, +// 'mode': { +// 'type': 'string', +// 'enum': ['normal', 'minimal'], +// 'title': 'Editing mode', +// 'description': 'Editing mode may be \'normal\' or \'minimal\' to allow use of external editors.' +// } +// } +// }, +// url: function() { +// return 'api/Settings/' + this.id; +// } +//}); + diff --git a/package.json b/package.json index a40f23cc1..80acd7a16 100644 --- a/package.json +++ b/package.json @@ -21,13 +21,11 @@ ], "licenses": [{ "type": "BSD" }], "dependencies": { - "backbone": "= 0.3.3", "backbone-dirty": "1.1.x", - "express": "2.2.x", + "bones": "1.3.x", "generic-pool": "1.0.x", "JSV": "3.5.x", "mbtiles": "0.0.x", - "mirror": "0.2.x", "sax": "0.1.x", "step": "0.0.x", "tilelive": "3.0.x", diff --git a/server/tiles.js b/server/tiles.js deleted file mode 100644 index 2c0cad852..000000000 --- a/server/tiles.js +++ /dev/null @@ -1,66 +0,0 @@ -var _ = require('underscore'), - path = require('path'), - Project = require('../shared/models').Project, - tilelive = new (require('tilelive').Server)(require('tilelive-mapnik')); - -module.exports = function(app, settings) { - app.enable('jsonp callback'); - - // Route middleware. Load a project model. - var loadProject = function(req, res, next) { - res.project = new Project({id: req.param('id')}); - res.project.fetch({ - success: function(model, resp) { - next(); - }, - error: function(model, resp) { - next(new Error('Invalid model')); - } - }); - }; - - // GET endpoint for TMS tile image requests. Uses `tilelive.js` Tile API. - // - // - `:id` String, project model id. - // - `:z` Number, zoom level of the tile requested. - // - `:x` Number, x coordinate of the tile requested. - // - `:y` Number, y coordinate of the tile requested. - // - `*` String, file format of the tile requested, e.g. `png`, `jpeg`. - app.get('/1.0.0/:id/:z/:x/:y.(png8|png|jpeg[\\d]+|jpeg|grid.json)', loadProject, function(req, res, next) { - req.params.datasource = path.join( - settings.files, - 'project', - res.project.id, - res.project.id + '.mml' - ); - req.params.format = req.params[0]; - if (req.params.format === 'grid.json') { - var interactivity = res.project.get('_interactivity'); - req.params.layer = interactivity.layer; - req.params.fields = res.project.formatterFields(); - } - tilelive.serve(req.params, function(err, data) { - if (!err) { - // Using `apply()` here allows the tile rendering function to - // send custom headers without access to the request object. - data[1] = _.extend(settings.header_defaults, data[1]); - res.send.apply(res, data); - } else { - next(err); - } - }); - }); - - // Interaction layer.json endpoint. - app.get('/1.0.0/:id/layer.json', loadProject, function(req, res, next) { - if (!res.project.get('_interactivity') && !res.project.get('_legend')) { - res.send('Formatter not found', 404); - } else { - var json = { - formatter: res.project.formatterJS(), - legend: res.project.get('_legend') - }; - res.send(json); - } - }); -}; diff --git a/servers/App.bones b/servers/App.bones new file mode 100644 index 000000000..df3ddfb60 --- /dev/null +++ b/servers/App.bones @@ -0,0 +1,34 @@ +var mapnik = require('tilelive-mapnik/node_modules/mapnik'), + reference = require('tilelive-mapnik/node_modules/carto').tree.Reference.data; + +server = Bones.Server.extend({}); + +server.prototype.initialize = function() { + _.bindAll(this, 'index', 'support'); + this.get('/', this.index); + this.get('/assets/tilemill/js/support.js', this.support); +}; + +server.prototype.index = function(req, res, next) { + res.send(templates['App']()); +}; + +server.prototype.support = function(req, res, next) { + var template = _.template( + 'var tilemill = tilemill || {};\n\n' + + 'tilemill.ABILITIES = <%= JSON.stringify(abilities) %>;\n\n' + + 'tilemill.REFERENCE = <%= JSON.stringify(reference) %>;\n\n' + ); + res.send(template({ + abilities: { + 'fonts': mapnik.fonts(), + 'datasources': mapnik.datasources(), + 'exports': { + mbtiles: true, + png: true, + pdf: mapnik.supports.cairo + } + }, + reference: reference + }), {'Content-Type': 'text/javascript'}); +}; diff --git a/servers/Core.bones b/servers/Core.bones new file mode 100644 index 000000000..b0b4e6642 --- /dev/null +++ b/servers/Core.bones @@ -0,0 +1,9 @@ +servers['Core'].prototype.port = 8889; +servers['Core'].prototype.initialize = function(app) { + this.port = app.config.port || this.port; + this.use(new servers['Middleware'](app)); + this.use(new servers['Tile'](app)); + this.use(new servers['App'](app)); + this.use(new servers['Route'](app)); + this.use(new servers['Asset'](app)); +}; diff --git a/servers/Route.bones b/servers/Route.bones new file mode 100644 index 000000000..974444437 --- /dev/null +++ b/servers/Route.bones @@ -0,0 +1,26 @@ +servers['Route'].augment({ + client: { + styles: [ + require.resolve('../build/vendor.css'), + require.resolve('../client/css/reset.css'), + require.resolve('../client/css/tilemill.css'), + require.resolve('../client/css/code.css') + ], + scripts: [ + require.resolve('../client/js/libraries/jquery-ui.js'), + require.resolve('../client/js/libraries/colorpicker/js/colorpicker.js'), + require.resolve('../build/vendor.js'), + require.resolve('wax/build/wax.mm.min.js'), + require.resolve('JSV/lib/uri/uri.js'), + require.resolve('JSV/lib/jsv.js'), + require.resolve('JSV/lib/json-schema-draft-03.js') + ] + }, + initializeclient: function(parent, app) { + parent.call(this, app); + this.get('/client/tilemill/css/vendor.css', + mirror.client(this.client.styles, { type: '.css' })); + this.get('/client/tilemill/js/vendor.js', + mirror.client(this.client.scripts, { type: '.js' })); + } +}); diff --git a/servers/Tile.bones b/servers/Tile.bones new file mode 100644 index 000000000..e6b721787 --- /dev/null +++ b/servers/Tile.bones @@ -0,0 +1,51 @@ +var path = require('path'), + tilelive = new (require('tilelive').Server)(require('tilelive-mapnik')), + settings = Bones.plugin.config; + +server = Bones.Server.extend({}); + +server.prototype.initialize = function() { + _.bindAll(this, 'load', 'layer', 'tile'); + this.get('/1.0.0/:id/:z/:x/:y.(png8|png|jpeg[\\d]+|jpeg)', this.tile); + this.get('/1.0.0/:id/:z/:x/:y.grid.json', this.load, this.tile); + this.get('/1.0.0/:id/layer.json', this.load, this.layer); +}; + +server.prototype.load = function(req, res, next) { + res.project = new models.Project({id: req.param('id')}); + res.project.fetch({ + success: function(model, resp) { next(); }, + error: function(model, resp) { next(resp); } + }); +}; + +server.prototype.tile = function(req, res, next) { + req.params.datasource = path.join( + settings.files, + 'project', + req.param('id'), + req.param('id') + '.mml' + ); + req.params.format = req.params[0]; + if (req.params.format === 'grid.json' && res.project) { + var interactivity = res.project.get('_interactivity'); + req.params.layer = interactivity.layer; + req.params.fields = res.project.formatterFields(); + } + tilelive.serve(req.params, function(err, data) { + if (err) return next(err); + data[1]['max-age'] = 3600; + res.send.apply(res, data); + }); +}; + +server.prototype.layer = function(req, res, next) { + if (!res.project.get('_interactivity') && !res.project.get('_legend')) { + next(new Error('Formatter not found.')); + } else { + res.send({ + formatter: res.project.formatterJS(), + legend: res.project.get('_legend') + }); + } +}; diff --git a/settings.js b/settings.js deleted file mode 100644 index b0471df86..000000000 --- a/settings.js +++ /dev/null @@ -1,17 +0,0 @@ -module.exports = { - 'port': 8889, - 'files': __dirname + '/files', - 'mapfile_dir': __dirname + '/files/.cache', - 'data_dir': __dirname + '/files/.cache', - 'export_dir': __dirname + '/files/export', - // TODO: request-specific overrides - 'header_defaults': { - 'Expires': new Date(Date.now() + - 1000 // second - * 60 // minute - * 60 // hour - * 24 // day - * 365 // year - ) - } -} diff --git a/shared/models.js b/shared/models.js deleted file mode 100644 index 2363cd90e..000000000 --- a/shared/models.js +++ /dev/null @@ -1,883 +0,0 @@ -// Backbone models and collections for use on both the client and server. -// @TODO if we use the keyword 'var' in front of Backbone, _, IE will wipe the -// globally defined Backbone and underscore leaving us with broken objects. -// This is obviously not ideal. -if (typeof require !== 'undefined' && typeof window === 'undefined') { - _ = require('underscore')._; - Backbone = require('backbone'); - JSV = require('JSV').JSV; -} - -// JSON schema validation -// ---------------------- -// Provide a default `validate()` method for all models. If a `schema` property -// is defined on a model, use JSON-schema validation by default. -Backbone.Model.prototype.validate = function(attributes) { - if (!this.schema || !this.schema.properties) return; - var env = JSV.createEnvironment(); - for (var key in attributes) { - if (this.schema.properties[key]) { - var property = this.schema.properties[key], - value = attributes[key]; - // Do a custom check for required properties, (e.g. do not allow - // an empty string to validate against a required property.) - if (!value && property.required) { - return (property.title || key) + ' is required.'; - } - - var errors = env.validate(value, property).errors; - if (errors.length) { - var error = errors.pop(); - if (property.description) { - return property.description; - } else { - return (property.title || key) + ': ' + error.message; - } - } - } - } -}; - -// Abilities (read-only) -// --------------------- -// Model. Describes server API abilities. -var Abilities = Backbone.Model.extend({ - schema: { - 'type': 'object', - 'properties': { - 'fonts': { - 'type': 'array', - 'title': 'Fonts', - 'description': 'Fonts available to Mapnik.' - }, - 'datasources': { - 'type': 'array', - 'title': 'Datasources', - 'description': 'Datasource types available to Mapnik.' - }, - 'exports': { - 'type': 'object', - 'title': 'Exports', - 'description': 'Export types available to Mapnik.' - } - } - }, - url: 'api/Abilities' -}); - -// Reference (read-only) -// --------------------- -// Model. MSS syntax reference. -var Reference = Backbone.Model.extend({ - url: 'api/Reference' -}); - -// Datasource (read-only) -// ---------------------- -// Model. Inspection metadata about a map layer. Use `fetchFeatures()` to do -// a datasource fetch that includes layer feature objects. -var Datasource = Backbone.Model.extend({ - // @TODO either as a feature or a bug, object attributes are not set - // automatically when passed to the constructor. We set it manually here. - initialize: function(attributes) { - this.set({'fields': attributes.fields}); - }, - url: function() { - var url = 'api/Datasource'; - this.getFeatures && (url += '/features'); - if (typeof module === 'undefined') { - url += '?' + $.param(this.attributes); - } - return url; - }, - fetchFeatures: function(options) { - this.getFeatures = true; - this.fetch(options); - } -}); - -// FileDatasource (read-only) -// -------------------------- -// Model. Inspection metadata about a map layer. Use `fetchFeatures()` to do -// a datasource fetch that includes layer feature objects. -var FileDatasource = Datasource.extend({ - schema: { - 'type': 'object', - 'properties': { - 'id': { - 'type': 'string', - }, - 'project': { - 'required': true, - 'type': 'string', - 'description': 'Project to which this datasource belongs.' - }, - 'url': { - 'type': 'string', - 'required': true, - 'minLength': 1, - 'title': 'URL', - 'description': 'URL of the datasource.' - }, - 'fields': { - 'type': 'object' - }, - 'features': { - 'type': 'array' - }, - 'ds_options': { - 'type': 'object' - }, - 'ds_type': { - 'type': 'string' - }, - 'geometry_type': { - 'type': 'string', - 'enum': ['polygon', 'point', 'linestring', 'raster', 'unknown'] - } - } - } -}); - -// PostgisDatasource (read-only) -// ----------------------------- -// Model. Inspection metadata about a map layer. Use `fetchFeatures()` to do -// a datasource fetch that includes layer feature objects. -var PostgisDatasource = Datasource.extend({ - schema: { - 'type': 'object', - 'properties': { - 'id': { - 'type': 'string', - }, - 'project': { - 'required': true, - 'type': 'string', - 'description': 'Project to which this datasource belongs.' - }, - 'dbname': { - 'type': 'string', - 'required': true, - 'minLength': 1, - 'title': 'Database', - 'description': 'Invalid PostGIS database.' - }, - 'table': { - 'type': 'string', - 'required': true, - 'minLength': 1, - 'title': 'Query', - 'description': 'Invalid PostGIS query.' - }, - 'fields': { - 'type': 'object' - }, - 'features': { - 'type': 'array' - }, - 'ds_options': { - 'type': 'object' - }, - 'ds_type': { - 'type': 'string' - }, - 'geometry_type': { - 'type': 'string', - 'enum': ['polygon', 'point', 'linestring', 'raster', 'unknown'] - } - } - } -}); - -// Settings -// -------- -// Model. Stores any user-specific configuration related to the app. -var Settings = Backbone.Model.extend({ - schema: { - 'type': 'object', - 'properties': { - 'id': { - 'type': 'string', - 'required': true, - 'enum': ['settings'] - }, - 'mode': { - 'type': 'string', - 'enum': ['normal', 'minimal'], - 'title': 'Editing mode', - 'description': 'Editing mode may be \'normal\' or \'minimal\' to allow use of external editors.' - } - } - }, - url: function() { - return 'api/Settings/' + this.id; - } -}); - -// Stylesheet -// ---------- -// Model. Represents a single map MSS stylesheet. This model is a child of -// the Project model and is saved serialized as part of the parent. -// **This model is not backed directly by the server.** -var Stylesheet = Backbone.Model.extend({ - schema: { - 'type': 'object', - 'properties': { - 'id': { - 'type': 'string', - 'required': true, - 'pattern': '^[A-Za-z0-9\-_.]+$', - 'title': 'Name', - 'description': 'Name may include alphanumeric characters, dots, dashes and underscores.' - }, - 'data': { - 'type': 'string', - 'required': true - } - } - }, - defaults: { - 'data': '' - } -}); - -// StylesheetList -// -------------- -// Collection. List of Stylesheet models. This collection is a child of the -// Project model and updates its parent on update events. -// **This collection is not backed directly by the server.** -var StylesheetList = Backbone.Collection.extend({ - model: Stylesheet, - initialize: function(models, options) { - var that = this; - this.parent = options.parent; - this.bind('change', function() { - this.parent.set({ 'Stylesheet': that }); - this.parent.change(); - }); - this.bind('add', function() { - this.parent.set({ 'Stylesheet': that }); - this.parent.change(); - }); - this.bind('remove', function() { - this.parent.set({ 'Stylesheet': that }); - this.parent.change(); - }); - } -}); - -// Layer -// ----- -// Model. Represents a single map layer. This model is a child of -// the Project model and is saved serialized as part of the parent. -// **This model is not backed directly by the server.** -var Layer = Backbone.Model.extend({ - schema: { - 'type': 'object', - 'properties': { - 'id': { - 'type': 'string', - 'required': true, - 'pattern': '^[A-Za-z0-9\-_]+$', - 'title': 'ID', - 'description': 'ID may include alphanumeric characters, dashes and underscores.' - }, - 'class': { - 'type': 'string', - 'pattern': '^[A-Za-z0-9\-_ ]*$', - 'title': 'Class', - 'description': 'Class may include alphanumeric characters, spaces, dashes and underscores.' - }, - 'srs': { - 'type': 'string' - }, - 'geometry': { - 'type': 'string', - 'enum': ['polygon', 'point', 'linestring', 'raster', 'unknown'] - }, - 'Datasource': { - 'type': 'object', - 'required': true - } - } - }, - // @TODO either as a feature or a bug, object attributes are not set - // automatically when passed to the constructor. We set it manually here. - initialize: function(attributes) { - this.set({'Datasource': attributes.Datasource}); - }, - // Constant. Hash of simple names to known SRS strings. - SRS: { - // note: 900913 should be the same as EPSG 3857 - '900913': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over', - 'WGS84': '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs' - }, - // Get the name of a model's SRS string if known, otherwise reteurn - // 'custom' or 'autodetect' if empty. - srsName: function() { - for (name in this.SRS) { - if (this.SRS[name] === this.get('srs')) { - return name; - } - } - return this.get('srs') ? 'custom' : 'autodetect'; - }, - // Implementation of `Model.set()` that allows a datasource model to be - // passed in as `options.datasource`. If provided, the datasource will be - // used to enforce key attributes. - set: function(attributes, options) { - if (options && options.datasource) { - if (options.datasource.get('ds_type') === 'gdal') { - attributes.srs = this.SRS['900913']; - } - if (options.datasource.get('geometry_type')) { - attributes.geometry = options.datasource.get('geometry_type'); - } - if (options.datasource.get('ds_options')) { - attributes.Datasource = _.extend( - attributes.Datasource, - options.datasource.get('ds_options') - ); - } - } - return Backbone.Model.prototype.set.call(this, attributes, options); - } -}); - -// LayerList -// --------- -// Collection. List of Layer models. This collection is a child of the -// Project model and updates its parent on update events. -// **This collection is not backed directly by the server.** -var LayerList = Backbone.Collection.extend({ - model: Layer, - initialize: function(models, options) { - var self = this; - this.parent = options.parent; - this.bind('change', function() { - this.parent.set({ 'Layer': self }); - this.parent.change(); - }); - this.bind('add', function() { - this.parent.set({ 'Layer': self }); - this.parent.change(); - }); - this.bind('remove', function() { - this.parent.set({ 'Layer': self }); - this.parent.change(); - }); - } -}); - -// Project -// ------- -// Model. A single TileMill map project. Describes an MML JSON map object that -// can be used by `carto` to render a map. -var Project = Backbone.Model.extend({ - schema: { - 'type': 'object', - 'properties': { - 'id': { - 'type': 'string', - 'required': true, - 'pattern': '^[A-Za-z0-9\-_]+$', - 'title': 'Name', - 'description': 'Name may include alphanumeric characters, dashes and underscores.' - }, - 'srs': { - 'type': 'string', - 'required': true - }, - 'Stylesheet': { - 'type': ['object', 'array'], - 'required': true - }, - 'Layer': { - 'type': ['object', 'array'], - 'required': true - }, - '_format': { - 'type': 'string', - 'enum': ['png', 'png24', 'png8', 'jpeg80', 'jpeg85', 'jpeg90', 'jpeg95'] - }, - '_center': { - 'type': 'object' - }, - '_interactivity': { - 'type': ['object', 'boolean'] - }, - '_updated': { - 'type': 'integer' - } - } - }, - STYLESHEET_DEFAULT: [{ - id: 'style.mss', - data: 'Map {\n' - + ' background-color: #fff;\n' - + '}\n\n' - + '#world {\n' - + ' polygon-fill: #eee;\n' - + ' line-color: #ccc;\n' - + ' line-width: 0.5;\n' - + '}' - }], - LAYER_DEFAULT: [{ - id: 'world', - name: 'world', - srs: '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 ' - + '+lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over', - geometry: 'polygon', - Datasource: { - file: 'http://tilemill-data.s3.amazonaws.com/world_borders_merc.zip', - type: 'shape' - } - }], - defaults: { - '_center': { lat:0, lon:0, zoom:2 }, - '_format': 'png', - '_interactivity': false, - 'srs': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 ' - + '+lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over', - 'Stylesheet': [], - 'Layer': [] - }, - // Custom setDefaults() method for creating a project with default layers, - // stylesheets, etc. Note that we do not use Backbone native initialize() - // or defaults(), both of which make default values far pervasive than the - // expected use here. - setDefaults: function() { - var template = {}; - !this.get('Stylesheet').length && (template.Stylesheet = this.STYLESHEET_DEFAULT); - !this.get('Layer').length && (template.Layer = this.LAYER_DEFAULT); - this.set(template, { silent: true }); - }, - // Instantiate StylesheetList and LayerList collections from JSON lists - // of plain JSON objects. - parse: function(resp) { - resp.Stylesheet && (resp.Stylesheet = new StylesheetList( - resp.Stylesheet, - {parent: this} - )); - resp.Layer && (resp.Layer = new LayerList( - resp.Layer, - {parent: this} - )); - return resp; - }, - url: function() { - return 'api/Project/' + this.id; - }, - // Custom validation method that allows for asynchronous processing. - // Expects options.success and options.error callbacks to be consistent - // with other Backbone methods. - validateAsync: function(attributes, options) { - // If client-side, pass-through. - if (typeof require === 'undefined') { - return options.success(this, null); - } - - var carto = require('tilelive-mapnik/node_modules/carto'), - mapnik = require('tilelive-mapnik/node_modules/mapnik'), - that = this, - stylesheets = this.get('Stylesheet'), - env = { - returnErrors: true, - errors: [], - validation_data: { - fonts: mapnik.fonts() - }, - deferred_externals: [], - only_validate: true, - effects: [] - }; - - // Hard clone the model JSON before rendering as rendering will change - // properties (e.g. localize a datasource URL to the filesystem). - var data = JSON.parse(JSON.stringify(attributes)); - new carto.Renderer(env) - .render(data, function(err, output) { - if (err) { - options.error(that, err); - } else { - options.success(that, null); - } - }); - }, - // Interactivity: Convert teaser/full template markup into formatter js. - // Replaces tokens like `[NAME]` with string concatentations of `data.NAME` - // removes line breaks and escapes single quotes. - // @TODO properly handle other possible #fail. Maybe use underscore - // templating? - formatterJS: function() { - if (_.isEmpty(this.get('_interactivity'))) return; - - var full = this.get('_interactivity').template_full || ''; - var teaser = this.get('_interactivity').template_teaser || ''; - var location = this.get('_interactivity').template_location || ''; - full = full.replace(/\'/g, '\\\'').replace(/\[([\w\d]+)\]/g, "' + data.$1 + '").replace(/\n/g, ' '); - teaser = teaser.replace(/\'/g, '\\\'').replace(/\[([\w\d]+)\]/g, "' + data.$1 + '").replace(/\n/g, ' '); - location = location.replace(/\'/g, '\\\'').replace(/\[([\w\d]+)\]/g, "' + data.$1 + '").replace(/\n/g, ' '); - return "function(options, data) { " - + " switch (options.format) {" - + " case 'full': " - + " return '" + full + "'; " - + " break; " - + " case 'location': " - + " return '" + location + "'; " - + " break; " - + " case 'teaser': " - + " default: " - + " return '" + teaser + "'; " - + " break; " - + " }" - + "}"; - }, - // Interactivity: Retrieve array of field names to be included in - // interactive tiles by parsing `[field]` tokens. - formatterFields: function() { - if (_.isEmpty(this.get('_interactivity'))) return; - var fields = []; - var full = this.get('_interactivity').template_full || ''; - var teaser = this.get('_interactivity').template_teaser || ''; - fields = fields - .concat(full.match(/\[([\w\d]+)\]/g)) - .concat(teaser.match(/\[([\w\d]+)\]/g)); - fields = _(fields).chain() - .filter(_.isString) - .map(function(field) { return field.replace(/[\[|\]]/g, ''); }) - .uniq() - .value(); - return fields; - } -}); - -// ProjectList -// ----------- -// Collection. All project models. -var ProjectList = Backbone.Collection.extend({ - model: Project, - url: 'api/Project', - comparator: function(project) { - return project.get('id'); - } -}); - -// Export -// ------ -// Model. Describes a single export task, e.g. rendering a map to a PDF. -var Export = Backbone.Model.extend({ - schema: { - 'type': 'object', - 'properties': { - 'id': { - 'type': 'string', - 'required': true - }, - 'project': { - 'type': 'string', - 'required': true - }, - 'format': { - 'type': 'string', - 'required': true, - 'enum': ['png', 'pdf', 'mbtiles'] - }, - 'status': { - 'type': 'string', - 'required': true, - 'enum': ['waiting', 'processing', 'complete', 'error'] - }, - 'progress': { - 'type': 'number', - 'minimum': 0, - 'maximum': 1 - }, - 'filename': { - 'type': 'string', - 'pattern': '^[A-Za-z0-9\-_.]+$' - }, - 'created': { - 'type': 'integer' - }, - 'updated': { - 'type': 'integer' - }, - 'error': { - 'type': 'string' - } - } - }, - initialize: function() { - if (this.isNew()){ - this.set({ - created: +new Date, - id: (+new Date) + '' - }, {silent: true}); - } - }, - url: function() { - return 'api/Export/' + this.id; - }, - defaults: { - progress: 0, - status: 'waiting' - }, - // Generate a download URL for an Export. - downloadURL: function() { - return (this.get('status') === 'complete') && 'export/download/' + this.get('filename'); - }, - // Get the duration of the current export job. - time: function() { - if (this.get('updated')) { - var seconds = parseInt((this.get('updated') - this.get('created')) * .001, 10); - var minutes = parseInt(seconds / 60, 10); - var remainder = seconds - (minutes * 60); - if (minutes && remainder) { - return minutes + ' min ' + remainder + ' sec'; - } else if (minutes) { - return minutes + ' min'; - } else { - return seconds + ' sec'; - } - } - return '0 sec'; - } -}); - -// ExportList -// ---------- -// Collection. List of all Exports. -var ExportList = Backbone.Collection.extend({ - model: Export, - url: 'api/Export', - comparator: function(job) { - return job.get('created'); - } -}); - -// Asset -// ----- -// Model. Single external asset, e.g. a shapefile, image, etc. -var Asset = Backbone.Model.extend({ - schema: { - 'type': 'object', - 'properties': { - 'id': { - 'type': 'string', - 'required': true - }, - 'url': { - 'type': 'string', - 'required': true - }, - 'bytes': { - 'type': 'string' - } - } - }, - extension: function() { - return this.id.split('.').pop(); - } -}); - -// AssetList -// --------- -// Collection. List of all assets for a given Library. Must be given a -// Library model at `options.library` in order to determine its URL endpoint. -// The REST endpoint for a LibraryList collection must return an array of asset -// models, or may optionally return a more complex object suited for handling -// pagination: -// -// { -// page: 0, // The current page number -// pageTotal: 10, // The total number of pages -// models: [] // An array of asset models -// } -var AssetList = Backbone.Collection.extend({ - model: Asset, - url: function() { - return 'api/Library/' + this.library.id + '/assets/' + this.page; - }, - initialize: function(options) { - this.page = 0; - this.pageTotal = 1; - this.library = options.library; - }, - parse: function(response) { - if (_.isArray(response)) { - return response; - } else { - this.page = response.page; - this.pageTotal = response.pageTotal; - return response.models; - } - }, - hasNext: function() { - return this.page < (this.pageTotal - 1); - }, - hasPrev: function() { - return this.page > 0; - }, - nextPage: function(options) { - if (!this.hasNext()) return; - this.page++; - this.fetch(options); - }, - prevPage: function(options) { - if (!this.hasPrev()) return; - this.page--; - this.fetch(options); - } -}); - -// AssetListS3 -// ----------- -// Collection. Override of AssetList for S3 library. S3 uses a marker key -// system for pagination instead of a page # system. -var AssetListS3 = AssetList.extend({ - url: function() { - var url = 'api/Library/' + this.library.id + '/assets'; - if (this.marker()) { - url += '/' + Base64.encodeURI(this.marker()); - } - return url; - }, - initialize: function(options) { - this.markers = []; - this.library = options.library; - }, - marker: function() { - if (this.markers.length) { - return this.markers[this.markers.length - 1]; - } - return false; - }, - parse: function(response) { - if (this.marker() != response.marker) { - this.markers.push(response.marker); - } - return response.models; - }, - hasNext: function() { - return this.marker(); - }, - hasPrev: function() { - return this.markers.length > 1; - }, - nextPage: function(options) { - if (!this.hasNext()) return; - this.fetch(options); - }, - prevPage: function(options) { - if (!this.hasPrev()) return; - this.markers.pop(); - this.markers.pop(); - this.fetch(options); - } -}); - -// Library -// -------- -// Model. Stores settings for a given asset library type, e.g. a local file -// directory or an Amazon S3 bucket. -var Library = Backbone.Model.extend({ - schema: { - 'type': 'object', - 'properties': { - 'id': { - 'type': 'string', - 'required': true, - 'minLength': 1 - }, - 'type': { - 'type': 'string', - 'required': true, - 'enum': ['s3', 'directory'] - }, - 'name': { - 'type': 'string', - 'required': true, - 'minLength': 1 - }, - 's3_bucket': { - 'type': 'string', - 'required': true, - 'minLength': 1 - }, - 'directory_path': { - 'type': 'string', - 'required': true, - 'minLength': 1 - } - }, - 'dependencies': { - 's3_bucket': { - 'properties': { - 'type': { 'enum': ['s3'], 'required': true } - } - }, - 's3_key': { - 'properties': { - 'type': { 'enum': ['s3'], 'required': true } - } - }, - 's3_secret': { - 'properties': { - 'type': { 'enum': ['s3'], 'required': true } - } - }, - 'directory_path': { - 'properties': { - 'type': { 'enum': ['directory'], 'required': true } - } - } - } - }, - url: function() { - return 'api/Library/' + this.id; - }, - defaults: { - type: 'directory' - }, - initialize: function(options) { - if (this.isNew()){ - this.set({id: (+new Date) + ''}, {silent: true}); - } - switch (this.get('type')) { - case 's3': - this.assets = new AssetListS3({ library: this }); - break; - default: - this.assets = new AssetList({ library: this }); - break; - } - } -}); - -// LibraryList -// ------------ -// Collection. All librarys. -var LibraryList = Backbone.Collection.extend({ - model: Library, - url: 'api/Library', - comparator: function(model) { - return model.get('name'); - } -}); - -(typeof module !== 'undefined') && (module.exports = { - Asset: Asset, - AssetList: AssetList, - AssetListS3: AssetListS3, - Library: Library, - LibraryList: LibraryList, - Project: Project, - ProjectList: ProjectList, - Export: Export, - ExportList: ExportList, - FileDatasource: FileDatasource, - PostgisDatasource: PostgisDatasource, - Settings: Settings -}); - diff --git a/tests.sh b/tests.sh deleted file mode 100755 index 540095b4f..000000000 --- a/tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -# Help script for running tests -test -d test/files && rm -r test/files -NODE_ENV=test ./bin/expresso --serial --port 8889 $* diff --git a/tilemill.js b/tilemill.js deleted file mode 100755 index 71f8911bf..000000000 --- a/tilemill.js +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env node -process.title = 'tilemill'; -// The TileMill application consists of: -// -// +-----------------+ -// | tilemill client | -// +-----------------+ -// | +----------------+ -// +-----------------+ |+----------------+ -// | express | -------> ||+----------------+ -// | tilemill server | -------> +|| node-worker | -// +-----------------+ +| export process | -// +----------------+ -// -// ### /client -// -// The TileMill client which consists of a single static HTML page and -// client-side javascript. -// -// ### /server -// -// The TileMill server which communicates to the client using JSON over HTTP -// requests. -// -// ### /server/export-worker.js -// -// `node-worker` export process created whenever a map export is requested by -// the user. Each of these export jobs run in a separate node process. -// -// ### /modules -// -// `backbone`, `underscore`, and `JSV` are libraries used by both the -// client and the server. The `/modules` directory is exposed to the client -// via `express.staticProvider` and contains both server-side and client-side -// modules. -// -// ### /shared -// -// The `/shared/models.js` file contains Backbone models and collections -// that are used on both the client and server. -// -// This file is the main Express server. -require.paths.splice(0, require.paths.length); -require.paths.unshift( - __dirname + '/node_modules', - __dirname + '/server', - __dirname + '/shared', - __dirname -); - -var express = require('express'), - mirror = require('mirror'), - settings = require('settings'); - -var app = module.exports = express.createServer(); - -app.use(express.bodyParser()); -app.use(express.static('client')); -app.use(express.static('shared')); - -var scripts = [ - './client/js/libraries/jquery.js', - './client/js/libraries/jquery-ui.js', - './client/js/libraries/colorpicker/js/colorpicker.js', - './build/vendor.js', - './client/js/parsecarto.js', - require.resolve('wax/build/wax.mm.min.js'), - require.resolve('underscore/underscore.js'), - require.resolve('backbone/backbone.js'), - require.resolve('JSV/lib/uri/uri.js'), - require.resolve('JSV/lib/jsv.js'), - require.resolve('JSV/lib/json-schema-draft-03.js'), - './shared/models.js' -]; -app.get('/vendor.js', mirror.assets(scripts)); - -var stylesheets = ['./build/vendor.css']; -app.get('/vendor.css', mirror.assets(stylesheets)); - -require('bootstrap')(app, settings); -require('api')(app, settings); -require('tiles')(app, settings); - -if (app.settings.env !== 'test') { - app.listen(settings.port); - console.log('Started TileMill on port %d.', settings.port); -}