Merge pull request #2745 from simianhacker/feature/config-migration

Closes #2377 - Migrate config from GA and Beyond

package.json

@@ -11,7 +11,7 @@
     "dashboarding"
   ],
   "private": false,
-  "version": "4.0.0-beta3",
+  "version": "4.0.0-rc1-snapshot",
   "main": "src/server/app.js",
   "homepage": "http://www.elasticsearch.org/overview/kibana/",
   "bugs": {
@@ -37,12 +37,14 @@
   },
   "dependencies": {
     "ansicolors": "^0.3.2",
+    "bluebird": "~2.0.7",
     "body-parser": "~1.10.1",
     "bunyan": "^1.2.3",
     "commander": "^2.6.0",
     "compression": "^1.3.0",
     "cookie-parser": "~1.3.3",
     "debug": "~2.1.1",
+    "elasticsearch": "^3.1.1",
     "express": "~4.10.6",
     "glob": "^4.3.2",
     "http-auth": "^2.2.5",
@@ -52,11 +54,12 @@
     "lodash": "^2.4.1",
     "morgan": "~1.5.1",
     "request": "^2.40.0",
+    "requirefrom": "^0.2.0",
+    "semver": "^4.2.0",
     "serve-favicon": "~2.2.0",
     "ssl-root-cas": "^1.1.7"
   },
   "devDependencies": {
-    "bluebird": "~2.0.7",
     "connect": "~2.19.5",
     "event-stream": "~3.1.5",
     "expect.js": "~0.3.1",
@@ -70,12 +73,13 @@
     "grunt-contrib-less": "~0.10.0",
     "grunt-contrib-requirejs": "~0.4.4",
     "grunt-contrib-watch": "~0.5.3",
-    "grunt-esvm": "~0.2.0",
+    "grunt-esvm": "~0.3.2",
     "grunt-mocha": "~0.4.10",
     "grunt-replace": "^0.7.9",
     "grunt-run": "^0.2.3",
     "grunt-s3": "~0.2.0-alpha.3",
     "grunt-saucelabs": "~8.3.2",
+    "grunt-simple-mocha": "^0.4.0",
     "html-entities": "^1.1.1",
     "http-proxy": "~1.8.1",
     "husky": "~0.6.0",

src/server/config/index.js

@@ -26,6 +26,13 @@ var htpasswdPath = path.resolve(__dirname, '..', '.htpasswd');
 if (!checkPath(htpasswdPath)) htpasswdPath = path.resolve(__dirname, '..', '..', '..', '.htpasswd');
 if (!checkPath(htpasswdPath)) htpasswdPath = false;
 
+var packagePath = path.resolve(__dirname, '..', 'package.json');
+try {
+  fs.statSync(packagePath);
+} catch (err) {
+  packagePath = path.resolve(__dirname, '..', '..', '..', 'package.json');
+}
+
 var config = module.exports = {
   port                    : kibana.port || 5601,
   host                    : kibana.host || '0.0.0.0',
@@ -36,6 +43,7 @@ var config = module.exports = {
   external_plugins_folder : process.env.PLUGINS_FOLDER || null,
   bundled_plugins_folder  : path.resolve(public_folder, 'plugins'),
   kibana                  : kibana,
+  package                 : require(packagePath),
   htpasswd                : htpasswdPath
 };
 

src/server/index.js

@@ -6,6 +6,8 @@ var app = require('./app');
 var fs = require('fs');
 var config = require('./config');
 var logger = require('./lib/logger');
+var Promise = require('bluebird');
+var initialization = require('./lib/serverInitialization');
 var key, cert;
 try {
   key = fs.readFileSync(config.kibana.ssl_key_file, 'utf8');
@@ -67,13 +69,18 @@ function onListening() {
   logger.info('Listening on %s:%d', address.address, address.port);
 }
 
+function start() {
+  var port = parseInt(process.env.PORT, 10) || config.port || 3000;
+  var host = process.env.HOST || config.host || '127.0.0.1';
+  var listen = Promise.promisify(server.listen.bind(server));
+  app.set('port', port);
+  return listen(port, host);
+}
+
 module.exports = {
   server: server,
   start: function (cb) {
-    var port = parseInt(process.env.PORT, 10) || config.port || 3000;
-    var host = process.env.HOST || config.host || '127.0.0.1';
-    app.set('port', port);
-    server.listen(port, host, cb);
+    return initialization().then(start).nodeify(cb);
   }
 };
 

src/server/lib/isUpgradeable.js

@@ -0,0 +1,33 @@
+var config = require('../config');
+var semver = require('semver');
+var rcVersionRegex = /(\d+\.\d+\.\d+)\-rc(\d+)/i;
+
+module.exports = function (doc) {
+  if (/beta|snapshot/i.test(doc._id)) return false;
+  if (!doc._id) return false;
+  if (doc._id === config.package.version) return false;
+
+  var packageRcRelease = Infinity;
+  var rcRelease = Infinity;
+  var packageVersion = config.package.version;
+  var version = doc._id;
+  var matches = doc._id.match(rcVersionRegex);
+  var packageMatches = config.package.version.match(rcVersionRegex);
+
+  if (matches) {
+    version = matches[1];
+    rcRelease = parseInt(matches[2], 10);
+  }
+
+  if (packageMatches) {
+    packageVersion = packageMatches[1];
+    packageRcRelease = parseInt(packageMatches[2], 10);
+  }
+
+  try {
+    if (semver.gte(version, packageVersion) && rcRelease >= packageRcRelease) return false;
+  } catch (e) {
+    return false;
+  }
+  return true;
+};

src/server/lib/migrateConfig.js

@@ -0,0 +1,39 @@
+var config = require('../config');
+var elasticsearch = require('elasticsearch');
+var upgrade = require('./upgradeConfig');
+var util = require('util');
+var url = require('url');
+var uri = url.parse(config.elasticsearch);
+if (config.kibana.elasticsearch_username && config.kibana.elasticsearch_password) {
+  uri.auth = util.format('%s:%s', config.kibana.elasticsearch_username, config.kibana.elasticsearch_password);
+}
+var client = new elasticsearch.Client({
+  host: url.format(uri)
+});
+
+module.exports = function () {
+  var options =  {
+    index: '.kibana',
+    type: 'config',
+    body: {
+      size: 1000,
+      sort: [ { buildNum: { order: 'desc' } } ],
+      query: {
+        filtered: {
+          filter: {
+            bool: {
+              must_not: [ { query: { match: { _id: '@@version' } } } ]
+            }
+          }
+        }
+      }
+    }
+  };
+
+  return client.search(options)
+  .then(upgrade)
+  .catch(function (err) {
+    if (!/^IndexMissingException/.test(err.message)) throw err;
+  });
+};
+

src/server/lib/serverInitialization.js

@@ -0,0 +1,10 @@
+var Promise = require('bluebird');
+var migrateConfig = require('./migrateConfig');
+
+module.exports = function () {
+  var tasks = [
+    migrateConfig()
+  ];
+
+  return Promise.all(tasks);
+};

src/server/lib/upgradeConfig.js

@@ -0,0 +1,28 @@
+var Promise = require('bluebird');
+var isUpgradeable = require('./isUpgradeable');
+var config = require('../config');
+var _ = require('lodash');
+var elasticsearch = require('elasticsearch');
+var client = new elasticsearch.Client({
+  host: config.elasticsearch
+});
+
+module.exports = function (response) {
+  var newConfig = {};
+  // Check to see if there are any doc. If not then we can assume
+  // nothing needs to be done
+  if (response.hits.hits.length === 0) return Promise.resolve();
+
+  // Look for upgradeable configs. If none of them are upgradeable
+  // then resolve with null.
+  var body = _.find(response.hits.hits, isUpgradeable);
+  if (body) return Promise.resolve();
+
+  return client.create({
+    index: '.kibana',
+    type: 'config',
+    id: config.package.version,
+    body: body
+  });
+
+};

tasks/config/esvm.js

@@ -1,17 +1,16 @@
 module.exports = function (grunt) {
-  var rel = require('path').join.bind(null, __dirname, '../../');
+  var join = require('path').join;
+  var rel = require('path').join.bind(null, grunt.config.get('root'));
+  var directory = join(grunt.config.get('root'), 'esvm');
 
   return {
     options: {
-      version: '^1.4',
+      directory: directory,
+      version: '1.4.2',
       plugins: [
         'elasticsearch/marvel/latest'
       ],
       config: {
-        path: {
-          data: rel('esvm/data_dir'),
-          logs: rel('esvm/logs')
-        },
         network: {
           host: '127.0.0.1'
         },
@@ -24,4 +23,4 @@ module.exports = function (grunt) {
     },
     dev: {}
   };
-};
+};

tasks/config/jshint.js

@@ -7,6 +7,7 @@ module.exports = function (grunt) {
           'Gruntfile.js',
           '<%= root %>/tasks/**/*.js',
           '<%= src %>/kibana/*.js',
+          '<%= src %>/server/*.js',
           '<%= src %>/kibana/{components,directives,factories,filters,plugins,registry,services,utils}/**/*.js',
           '<%= unitTestDir %>/**/*.js'
         ]

tasks/config/simplemocha.js

@@ -0,0 +1,8 @@
+module.exports = {
+  options: {
+    timeout: 2000,
+    ignoreLeaks: false,
+    reporter: 'dot'
+  },
+  all: { src: ['<%= root %>/test/server/unit/**/*.js']}
+};

tasks/test.js

@@ -1,6 +1,6 @@
 var _ = require('lodash');
 module.exports = function (grunt) {
-  function getTestTask() {
+  function addTestTask(tasks) {
     var testTask = 'mocha:unit';
 
     if (grunt.option('use-sauce') || process.env.TRAVIS) {
@@ -13,6 +13,8 @@ module.exports = function (grunt) {
       }
     }
 
+    tasks.push('simplemocha:all', testTask);
+
     return testTask;
   }
 
@@ -26,19 +28,18 @@ module.exports = function (grunt) {
       'jshint',
       'maybe_start_kibana',
       'jade',
-      'less',
-      getTestTask()
+      'less'
     ];
-
-    if (process.env.TRAVIS) tasks.shift('esvm:dev');
+    addTestTask(tasks);
+    if (process.env.TRAVIS) tasks.unshift('esvm:dev');
     grunt.task.run(tasks);
   });
 
   grunt.registerTask('quick-test', function () {
     var tasks = [
-      'maybe_start_kibana',
-      getTestTask()
+      'maybe_start_kibana'
     ];
+    addTestTask(tasks);
     grunt.task.run(tasks);
   });
 

test/server/unit/lib/isUpgradeable.js

@@ -0,0 +1,55 @@
+var root = require('requirefrom')('');
+var isUpgradeable = root('src/server/lib/isUpgradeable');
+var expect = require('expect.js');
+var util = require('util');
+var package = root('package.json');
+
+describe('lib/isUpgradeable', function () {
+
+  function upgradeDoc(_id, version, bool) {
+    it(util.format('should return %s for %s <= %s', bool, _id, version), function () {
+      var doc = { _id: _id };
+      package.version = version;
+      expect(isUpgradeable(doc)).to.be(bool);
+    });
+  }
+
+  upgradeDoc('1.0.0-beta1', package.version, false);
+  upgradeDoc(package.version, package.version, false);
+  upgradeDoc('4.0.0-RC1', '4.0.0-RC2', true);
+  upgradeDoc('4.0.0-rc2', '4.0.0-rc1', false);
+  upgradeDoc('4.0.0-rc2', '4.0.0', true);
+  upgradeDoc('4.0.0-rc2', '4.0.2', true);
+  upgradeDoc('4.0.1', '4.1.0-rc', true);
+  upgradeDoc('4.0.0-rc1', '4.0.0', true);
+  upgradeDoc('4.0.0-rc1-snapshot', '4.0.0', false);
+  upgradeDoc('4.1.0-rc1-snapshot', '4.1.0-rc1', false);
+
+  it('should handle missing _id field', function () {
+    var doc = {
+      '_index': '.kibana',
+      '_type': 'config',
+      '_score': 1,
+      '_source': {
+        'buildNum': 1.7976931348623157e+308,
+        'defaultIndex': '[logstash-]YYYY.MM.DD'
+      }
+    };
+    expect(isUpgradeable(doc)).to.be(false);
+  });
+
+  it('should handle _id of @@version', function () {
+    var doc = {
+      '_index': '.kibana',
+      '_type': 'config',
+      '_id': '@@version',
+      '_score': 1,
+      '_source': {
+        'buildNum': 1.7976931348623157e+308,
+        'defaultIndex': '[logstash-]YYYY.MM.DD'
+      }
+    };
+    expect(isUpgradeable(doc)).to.be(false);
+  });
+
+});