feat: gui mode support loading many databases from databaseUrls.json

.babelignore

@@ -0,0 +1 @@
+./lib/gui

lib/common-utils.js

@@ -2,6 +2,8 @@
 
 const {SUCCESS, FAIL, ERROR, SKIPPED, UPDATED, IDLE, RUNNING, QUEUED} = require('./constants/test-statuses');
 const saveFormats = require('./constants/save-formats');
+const {DB_MAX_AVAILABLE_PAGE_SIZE, DB_TABLE_NAME, DB_TYPES} = require('./constants/database');
+
 const statusPriority = [
     // non-final
     RUNNING, QUEUED,
@@ -17,6 +19,29 @@ exports.isErroredStatus = (status) => status === ERROR;
 exports.isSkippedStatus = (status) => status === SKIPPED;
 exports.isUpdatedStatus = (status) => status === UPDATED;
 
+exports.selectAllQuery = `SELECT * FROM ${DB_TABLE_NAME}`;
+
+exports.dbColumns = [
+    {name: 'suitePath', type: DB_TYPES.text},
+    {name: 'suiteName', type: DB_TYPES.text},
+    {name: 'name', type: DB_TYPES.text},
+    {name: 'suiteUrl', type: DB_TYPES.text},
+    {name: 'metaInfo', type: DB_TYPES.text},
+    {name: 'description', type: DB_TYPES.text},
+    {name: 'error', type: DB_TYPES.text},
+    {name: 'skipReason', type: DB_TYPES.text},
+    {name: 'imagesInfo', type: DB_TYPES.text},
+    {name: 'screenshot', type: DB_TYPES.int}, //boolean - 0 or 1
+    {name: 'multipleTabs', type: DB_TYPES.int}, //boolean - 0 or 1
+    {name: 'status', type: DB_TYPES.text},
+    {name: 'timestamp', type: DB_TYPES.int}
+];
+
+exports.dbColumnIndexes = exports.dbColumns.reduce((acc, {name}, index) => {
+    acc[name] = index;
+    return acc;
+}, {});
+
 exports.determineStatus = (statuses) => {
     if (!statuses.length) {
         return SUCCESS;
@@ -33,23 +58,30 @@ exports.determineStatus = (statuses) => {
 };
 
 exports.createTableQuery = () => {
-    const columns = [
-        'suitePath TEXT',
-        'suiteName TEXT',
-        'name TEXT',
-        'suiteUrl TEXT',
-        'metaInfo TEXT',
-        'description TEXT',
-        'error TEXT',
-        'skipReason TEXT',
-        'imagesInfo TEXT',
-        'screenshot INT', //boolean - 0 or 1
-        'multipleTabs INT', //boolean - 0 or 1
-        'status TEXT',
-        'timestamp INT'
-    ].join(', ');
-
-    return `CREATE TABLE suites (${columns})`;
+    const columns = exports.dbColumns.map(({name, type}) => {
+        return `${name} ${type}`;
+    }).join(', ');
+
+    return `CREATE TABLE IF NOT EXISTS ${DB_TABLE_NAME} (${columns})`;
+};
+
+exports.mergeTablesQueries = (dbPaths) => {
+    const queries = [
+        `PRAGMA page_size = ${DB_MAX_AVAILABLE_PAGE_SIZE}`,
+        exports.createTableQuery()
+    ];
+
+    for (const dbPath of dbPaths) {
+        queries.push(`ATTACH '${dbPath}' AS attached`);
+        queries.push(`INSERT OR IGNORE INTO ${DB_TABLE_NAME} SELECT * FROM attached.${DB_TABLE_NAME}`);
+        queries.push('DETACH attached');
+    }
+
+    return queries;
+};
+
+exports.compareDatabaseRowsByTimestamp = (row1, row2) => {
+    return row1[exports.dbColumnIndexes.timestamp] - row2[exports.dbColumnIndexes.timestamp];
 };
 
 exports.isSqlite = saveFormat => saveFormat === saveFormats.SQLITE;

lib/constants/database.js

@@ -0,0 +1,10 @@
+'use strict';
+
+module.exports = {
+    DB_MAX_AVAILABLE_PAGE_SIZE: 65536, // helps to speed up queries
+    DB_TABLE_NAME: 'suites',
+    DB_TYPES: {
+        int: 'INT',
+        text: 'TEXT'
+    }
+};

lib/constants/file-names.js

@@ -2,5 +2,5 @@
 
 module.exports = {
     LOCAL_DATABASE_NAME: 'sqlite.db',
-    DATABASE_TO_REUSE: 'old_sqlite.db'
+    DATABASE_URLS_JSON_NAME: 'databaseUrls.json'
 };

lib/gui/api/facade.js

@@ -1,9 +1,9 @@
 'use strict';
 
-const {EventEmitter} = require('events');
+const {AsyncEmitter} = require('gemini-core').events;
 const guiEvents = require('../constants/gui-events');
 
-module.exports = class ApiFacade extends EventEmitter {
+module.exports = class ApiFacade extends AsyncEmitter {
     static create() {
         return new ApiFacade();
     }

lib/gui/api/index.js

@@ -11,11 +11,11 @@ module.exports = class Api {
         this._gui = hermione.gui = ApiFacade.create();
     }
 
-    initServer(server) {
-        this._gui.emit(this._gui.events.SERVER_INIT, server);
+    async initServer(server) {
+        await this._gui.emitAndWait(this._gui.events.SERVER_INIT, server);
     }
 
-    serverReady(data) {
-        this._gui.emit(this._gui.events.SERVER_READY, data);
+    async serverReady(data) {
+        await this._gui.emitAndWait(this._gui.events.SERVER_READY, data);
     }
 };

lib/gui/server.js

@@ -12,14 +12,14 @@ const {MAX_REQUEST_SIZE, KEEP_ALIVE_TIMEOUT, HEADERS_TIMEOUT} = require('./const
 const {IMAGES_PATH, ERROR_DETAILS_PATH} = require('../constants/paths');
 const {logger, initializeCustomGui, runCustomGuiAction} = require('../server-utils');
 
-exports.start = ({paths, hermione, guiApi, configs}) => {
+exports.start = async ({paths, hermione, guiApi, configs}) => {
     const {options, pluginConfig} = configs;
     const app = App.create(paths, hermione, configs);
     const server = express();
 
     server.use(bodyParser.json({limit: MAX_REQUEST_SIZE}));
 
-    guiApi.initServer(server);
+    await guiApi.initServer(server);
 
     server.use(express.static(path.join(__dirname, '../static'), {index: 'gui.html'}));
     server.use(express.static(process.cwd()));
@@ -94,17 +94,18 @@ exports.start = ({paths, hermione, guiApi, configs}) => {
         logger.log('server shutting down');
     });
 
-    return app.initialize()
-        .then(() => {
-            return Promise.fromCallback((callback) => {
-                const httpServer = server.listen(options.port, options.hostname, callback);
-                httpServer.keepAliveTimeout = KEEP_ALIVE_TIMEOUT;
-                httpServer.headersTimeout = HEADERS_TIMEOUT;
-            });
-        })
-        .then(() => {
-            const data = {url: `http://${options.hostname}:${options.port}`};
-            guiApi.serverReady(data);
-            return data;
-        });
+    await app.initialize();
+
+    const {port, hostname} = options;
+    await Promise.fromCallback((callback) => {
+        const httpServer = server.listen(port, hostname, callback);
+        httpServer.keepAliveTimeout = KEEP_ALIVE_TIMEOUT;
+        httpServer.headersTimeout = HEADERS_TIMEOUT;
+    });
+
+    const data = {url: `http://${hostname}:${port}`};
+
+    await guiApi.serverReady(data);
+
+    return data;
 };

lib/gui/tool-runner/index.js

@@ -22,7 +22,7 @@ const {
     formatTests,
     formatId, getShortMD5,
     mkFullTitle,
-    renameDatabaseForReuse,
+    mergeDatabasesForReuse,
     getDataFromDatabase,
     findTestResult
 } = require('./utils');
@@ -61,9 +61,9 @@ module.exports = class ToolRunner {
     async initialize() {
         // TODO: remove creating 'report-builder-json' when making saving to DB the default save format
         if (this.saveResultsToDb) {
-            //rename the old database because it will get overridden by ReportBuilderSqlite
-            renameDatabaseForReuse(this._reportPath);
-            this._reportBuilder = await ReportBuilderSqlite.create(this._hermione, this._pluginConfig);
+            await mergeDatabasesForReuse(this._reportPath);
+            this._reportBuilder = ReportBuilderSqlite.create(this._hermione, this._pluginConfig, {reuse: true});
+            await this._reportBuilder.init();
             await this._reportBuilder.saveStaticFiles();
         } else {
             this._reportBuilder = ReportBuilderJson.create(this._hermione, this._pluginConfig);
@@ -75,10 +75,10 @@ module.exports = class ToolRunner {
         this._subscribeOnEvents();
     }
 
-    _readTests() {
+    async _readTests() {
         const {grep, set: sets, browser: browsers} = this._globalOpts;
 
-        return this._hermione.readTests(this._testFiles, {grep, sets, browsers});
+        return await this._hermione.readTests(this._testFiles, {grep, sets, browsers});
     }
 
     finalize() {
@@ -140,22 +140,33 @@ module.exports = class ToolRunner {
     }
 
     async _loadReuseData() {
-        if (this.saveResultsToDb) {
-            const pathToDatabase = path.resolve(this._reportPath, constantFileNames.DATABASE_TO_REUSE);
-            if (!fs.existsSync(pathToDatabase)) {
-                logger.warn(chalk.yellow(`Nothing to reuse in ${this._reportPath}`));
-                return {};
-            }
-            return await getDataFromDatabase(pathToDatabase, this._reportBuilder);
-        }
+        let data = {};
 
-        // TODO: remove when making saving to DB the default save format
+        // TODO: remove loading "data.js" with format save "js"
         try {
-            return utils.require(path.resolve(this._reportPath, 'data'));
+            data = utils.require(path.resolve(this._reportPath, 'data'));
         } catch (e) {
-            logger.warn(chalk.yellow(`Nothing to reuse in ${this._reportPath}`));
+            logger.warn(chalk.yellow(`Nothing to reuse in ${this._reportPath}: can not load data.js`));
             return {};
         }
+
+        if (data.saveFormat !== this._pluginConfig.saveFormat) {
+            logger.warn(chalk.yellow(`Nothing to reuse in ${this._reportPath}: saveFormat from data.js does not equal saveFormat from plugin options.`));
+            return {};
+        }
+
+        if (!this.saveResultsToDb) {
+            return data;
+        }
+
+        const dbPath = path.resolve(this._reportPath, constantFileNames.LOCAL_DATABASE_NAME);
+
+        if (await fs.pathExists(dbPath)) {
+            return getDataFromDatabase(dbPath);
+        }
+
+        logger.warn(chalk.yellow(`Nothing to reuse in ${this._reportPath}: can not load data from ${constantFileNames.DATABASE_URLS_JSON_NAME}`));
+        return {};
     }
 
     run(tests = []) {
@@ -167,7 +178,7 @@ module.exports = class ToolRunner {
     }
 
     async _handleRunnableCollection() {
-        await Promise.all(this._collection.mapTests(async (test, browserId) => {
+        this._collection.eachTest((test, browserId) => {
             if (test.disabled || test.silentSkip) {
                 return;
             }
@@ -176,9 +187,9 @@ module.exports = class ToolRunner {
             this._tests[testId] = _.extend(test, {browserId});
 
             test.pending
-                ? await this._reportBuilder.addSkipped(test)
-                : await this._reportBuilder.addIdle(test);
-        }));
+                ? this._reportBuilder.addSkipped(test)
+                : this._reportBuilder.addIdle(test);
+        });
 
         await this._fillTestsTree();
     }

lib/gui/tool-runner/report-subscriber.js

@@ -55,7 +55,7 @@ module.exports = (hermione, reportBuilder, client, reportPath) => {
             formattedResult.attempt = reportBuilder.getCurrAttempt(formattedResult);
 
             await formattedResult.saveTestImages(reportPath, workers);
-            await reportBuilder.addSuccess(formattedResult);
+            reportBuilder.addSuccess(formattedResult);
             const testResult = findTestResult(
                 reportBuilder.getSuites(),
                 formattedResult.prepareTestResult()
@@ -72,8 +72,8 @@ module.exports = (hermione, reportBuilder, client, reportPath) => {
 
             await failHandler(formattedResult);
             formattedResult.hasDiff()
-                ? await reportBuilder.addFail(formattedResult)
-                : await reportBuilder.addError(formattedResult);
+                ? reportBuilder.addFail(formattedResult)
+                : reportBuilder.addError(formattedResult);
             const testResult = findTestResult(reportBuilder.getSuites(), formattedResult.prepareTestResult());
 
             return client.emit(clientEvents.TEST_RESULT, testResult);
@@ -86,7 +86,7 @@ module.exports = (hermione, reportBuilder, client, reportPath) => {
             formattedResult.attempt = reportBuilder.getCurrAttempt(formattedResult);
 
             await failHandler(formattedResult);
-            await reportBuilder.addRetry(formattedResult);
+            reportBuilder.addRetry(formattedResult);
         });
     });
 
@@ -96,7 +96,7 @@ module.exports = (hermione, reportBuilder, client, reportPath) => {
             formattedResult.attempt = reportBuilder.getCurrAttempt(formattedResult);
 
             await failHandler(formattedResult);
-            await reportBuilder.addSkipped(formattedResult);
+            reportBuilder.addSkipped(formattedResult);
             const testResult = findTestResult(reportBuilder.getSuites(), formattedResult.prepareTestResult());
 
             return client.emit(clientEvents.TEST_RESULT, testResult);