[Reporting/Test] move functional tests to apps (#64368)

* Squashed commit of the following: commit 5953089c03bea6b2d091f7723fea25bb1c210ee8 Author: Timothy Sullivan <tsullivan@elastic.co> Date: Thu Apr 9 18:29:55 2020 -0700 move tests to apps commit 39adeaae6d3502d2c6da4e6111c2a396f2a7aedb Author: Timothy Sullivan <tsullivan@elastic.co> Date: Thu Apr 9 17:49:20 2020 -0700 update archive with better dashboard commit 55b600748356c1adc5e75669b0c5588a812e401d Author: Timothy Sullivan <tsullivan@elastic.co> Date: Thu Apr 9 17:16:53 2020 -0700 fix the refactoring bugs commit 11aff10cd8009aeb9bb78aa5ce0b37a72d47776e Author: Timothy Sullivan <tsullivan@elastic.co> Date: Thu Apr 9 17:16:28 2020 -0700 remove unused fixtuers commit 05c33817c8eb67c461ac012cf2f71a9c01f1a91e Author: Timothy Sullivan <tsullivan@elastic.co> Date: Thu Apr 9 16:37:36 2020 -0700 Start of refactoring commit b63c182b5f32b19dc3ca715efdbc5f18fcc02f67 Author: Timothy Sullivan <tsullivan@elastic.co> Date: Thu Apr 9 16:32:50 2020 -0700 Todo comments commit 1e0105e Author: Timothy Sullivan <tsullivan@elastic.co> Date: Thu Apr 9 14:31:58 2020 -0700 revert unrelated change commit 206fd14 Merge: 0d4c2ad 8343064 Author: Timothy Sullivan <tsullivan@elastic.co> Date: Thu Apr 9 14:28:45 2020 -0700 Merge branch 'master' into reporting/test-better commit 0d4c2ad Author: Timothy Sullivan <tsullivan@elastic.co> Date: Wed Apr 8 10:41:19 2020 -0700 fix ts commit 890128c Merge: d9ce402 3598b8c Author: Timothy Sullivan <tsullivan@elastic.co> Date: Wed Apr 8 10:31:09 2020 -0700 Merge branch 'master' into reporting/test-better commit d9ce402 Author: Timothy Sullivan <tsullivan@elastic.co> Date: Tue Apr 7 08:31:58 2020 -0700 [Reporting] convert all server unit tests to TypeScript * fix imports and readmes * remove not-needed readme * remove extra info from readme * correct some comments * log the error that was caught * fix config path in readme * fix readme instructions to point to updated paths Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
elastic · Apr 27, 2020 · 4c460b8 · 4c460b8
1 parent 4e714c2
commit 4c460b8
Show file tree

Hide file tree

Showing 16 changed files with 298 additions and 466 deletions.
diff --git a/x-pack/scripts/functional_tests.js b/x-pack/scripts/functional_tests.js
@@ -14,7 +14,6 @@ const alwaysImportedTests = [
 ];
 const onlyNotInCoverageTests = [
   require.resolve('../test/reporting/configs/chromium_api.js'),
-  require.resolve('../test/reporting/configs/chromium_functional.js'),
   require.resolve('../test/reporting/configs/generate_api.js'),
   require.resolve('../test/api_integration/config_security_basic.js'),
   require.resolve('../test/api_integration/config.js'),

diff --git a/x-pack/test/functional/apps/dashboard/index.ts b/x-pack/test/functional/apps/dashboard/index.ts
@@ -11,5 +11,6 @@ export default function({ loadTestFile }: FtrProviderContext) {
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./preserve_url'));
+    loadTestFile(require.resolve('./reporting'));
   });
 }
diff --git a/x-pack/test/functional/apps/dashboard/reporting/README.md b/x-pack/test/functional/apps/dashboard/reporting/README.md
@@ -0,0 +1,22 @@
+## The Dashboard Reporting Tests
+
+### Baseline snapshots
+
+The reporting tests create a few PNG reports and do a snapshot comparison against stored baselines.  The baseline images are stored in `./reports/baseline`.
+
+### Updating the baselines
+
+Every now and then visual changes will be made that will require the snapshots to be updated.  This is how you go about updating it.
+
+1. **Load the ES Archive containing the dashboard and data.**
+   This will load the test data into an Elasticsearch instance running via the functional test server:
+  ```
+  node scripts/es_archiver load reporting/ecommerce --config=x-pack/test/functional/config.js
+  node scripts/es_archiver load reporting/ecommerce_kibana --config=x-pack/test/functional/config.js
+  ```
+2. **Generate the reports of the E-commerce dashboard in the Kibana UI.**
+   Navigate to `http://localhost:5620`, find the archived dashboard, and generate all the types of reports for which there are stored baseline images.
+3. **Download the reports, and save them into the `reports/baseline` folder.**
+   Change the names of the PNG/PDF files to overwrite the stored baselines.
+
+The next time functional tests run, the generated reports will be compared to the latest image that you have saved :bowtie:
diff --git a/x-pack/test/functional/apps/dashboard/reporting/index.js b/x-pack/test/functional/apps/dashboard/reporting/index.js
@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import path from 'path';
+import fs from 'fs';
+import { promisify } from 'util';
+import { checkIfPngsMatch } from './lib/compare_pngs';
+
+const writeFileAsync = promisify(fs.writeFile);
+const mkdirAsync = promisify(fs.mkdir);
+
+const REPORTS_FOLDER = path.resolve(__dirname, 'reports');
+
+export default function({ getService, getPageObjects }) {
+  const esArchiver = getService('esArchiver');
+  const browser = getService('browser');
+  const log = getService('log');
+  const config = getService('config');
+  const PageObjects = getPageObjects(['reporting', 'common', 'dashboard']);
+
+  describe('Reporting', () => {
+    before('initialize tests', async () => {
+      log.debug('ReportingPage:initTests');
+      await esArchiver.loadIfNeeded('reporting/ecommerce');
+      await esArchiver.loadIfNeeded('reporting/ecommerce_kibana');
+      await browser.setWindowSize(1600, 850);
+    });
+    after('clean up archives', async () => {
+      await esArchiver.unload('reporting/ecommerce');
+      await esArchiver.unload('reporting/ecommerce_kibana');
+    });
+
+    describe('Print PDF button', () => {
+      it('is not available if new', async () => {
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.clickNewDashboard();
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.dashboard.saveDashboard('My PDF Dashboard');
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+    });
+
+    describe('Print Layout', () => {
+      it('downloads a PDF file', async function() {
+        // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs
+        // function is taking about 15 seconds per comparison in jenkins.
+        this.timeout(300000);
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
+        await PageObjects.reporting.openPdfReportingPanel();
+        await PageObjects.reporting.checkUsePrintLayout();
+        await PageObjects.reporting.clickGenerateReportButton();
+
+        const url = await PageObjects.reporting.getReportURL(60000);
+        const res = await PageObjects.reporting.getResponse(url);
+
+        expect(res.statusCode).to.equal(200);
+        expect(res.headers['content-type']).to.equal('application/pdf');
+      });
+    });
+
+    describe('Print PNG button', () => {
+      it('is not available if new', async () => {
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.clickNewDashboard();
+        await PageObjects.reporting.openPngReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.dashboard.saveDashboard('My PNG Dash');
+        await PageObjects.reporting.openPngReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+    });
+
+    describe('Preserve Layout', () => {
+      it('matches baseline report', async function() {
+        const writeSessionReport = async (name, rawPdf, reportExt) => {
+          const sessionDirectory = path.resolve(REPORTS_FOLDER, 'session');
+          await mkdirAsync(sessionDirectory, { recursive: true });
+          const sessionReportPath = path.resolve(sessionDirectory, `${name}.${reportExt}`);
+          await writeFileAsync(sessionReportPath, rawPdf);
+          return sessionReportPath;
+        };
+        const getBaselineReportPath = (fileName, reportExt) => {
+          const baselineFolder = path.resolve(REPORTS_FOLDER, 'baseline');
+          const fullPath = path.resolve(baselineFolder, `${fileName}.${reportExt}`);
+          log.debug(`getBaselineReportPath (${fullPath})`);
+          return fullPath;
+        };
+
+        this.timeout(300000);
+
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
+        await PageObjects.reporting.openPngReportingPanel();
+        await PageObjects.reporting.forceSharedItemsContainerSize({ width: 1405 });
+        await PageObjects.reporting.clickGenerateReportButton();
+        await PageObjects.reporting.removeForceSharedItemsContainerSize();
+
+        const url = await PageObjects.reporting.getReportURL(60000);
+        const reportData = await PageObjects.reporting.getRawPdfReportData(url);
+        const reportFileName = 'dashboard_preserve_layout';
+        const sessionReportPath = await writeSessionReport(reportFileName, reportData, 'png');
+        const percentSimilar = await checkIfPngsMatch(
+          sessionReportPath,
+          getBaselineReportPath(reportFileName, 'png'),
+          config.get('screenshots.directory'),
+          log
+        );
+
+        expect(percentSimilar).to.be.lessThan(0.1);
+      });
+    });
+  });
+}
diff --git a/.../reporting/functional/lib/compare_pngs.js → ...s/dashboard/reporting/lib/compare_pngs.js b/.../reporting/functional/lib/compare_pngs.js → ...s/dashboard/reporting/lib/compare_pngs.js
@@ -7,7 +7,7 @@
 import path from 'path';
 import fs from 'fs';
 import { promisify } from 'bluebird';
-import { comparePngs } from '../../../../../test/functional/services/lib/compare_pngs';
+import { comparePngs } from '../../../../../../../test/functional/services/lib/compare_pngs';
 
 const mkdirAsync = promisify(fs.mkdir);
 

diff --git a/...ts/baseline/dashboard_preserve_layout.png → ...ts/baseline/dashboard_preserve_layout.png b/...ts/baseline/dashboard_preserve_layout.png → ...ts/baseline/dashboard_preserve_layout.png
diff --git a/x-pack/test/functional/apps/discover/index.ts b/x-pack/test/functional/apps/discover/index.ts
@@ -11,5 +11,6 @@ export default function({ loadTestFile }: FtrProviderContext) {
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./preserve_url'));
+    loadTestFile(require.resolve('./reporting'));
   });
 }
diff --git a/x-pack/test/functional/apps/discover/reporting.js b/x-pack/test/functional/apps/discover/reporting.js
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+export default function({ getService, getPageObjects }) {
+  const log = getService('log');
+  const esArchiver = getService('esArchiver');
+  const browser = getService('browser');
+  const PageObjects = getPageObjects(['reporting', 'common', 'discover']);
+  const filterBar = getService('filterBar');
+
+  describe('Discover', () => {
+    before('initialize tests', async () => {
+      log.debug('ReportingPage:initTests');
+      await esArchiver.loadIfNeeded('reporting/ecommerce');
+      await browser.setWindowSize(1600, 850);
+    });
+    after('clean up archives', async () => {
+      await esArchiver.unload('reporting/ecommerce');
+    });
+
+    describe('Generate CSV button', () => {
+      beforeEach(() => PageObjects.common.navigateToApp('discover'));
+
+      it('is not available if new', async () => {
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+
+      it('becomes available/not available when a saved search is created, changed and saved again', async () => {
+        // create new search, csv export is not available
+        await PageObjects.discover.clickNewSearchButton();
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+        // save search, csv export is available
+        await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+        // add filter, csv export is not available
+        await filterBar.addFilter('currency', 'is', 'EUR');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+        // save search again, csv export is available
+        await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+
+      it('generates a report with data', async () => {
+        await PageObjects.discover.clickNewSearchButton();
+        await PageObjects.reporting.setTimepickerInDataRange();
+        await PageObjects.discover.saveSearch('my search - with data - expectReportCanBeCreated');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.canReportBeCreated()).to.be(true);
+      });
+
+      it('generates a report with no data', async () => {
+        await PageObjects.reporting.setTimepickerInNoDataRange();
+        await PageObjects.discover.saveSearch('my search - no data - expectReportCanBeCreated');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.canReportBeCreated()).to.be(true);
+      });
+    });
+  });
+}
diff --git a/x-pack/test/functional/apps/visualize/index.ts b/x-pack/test/functional/apps/visualize/index.ts
@@ -15,5 +15,6 @@ export default function visualize({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./hybrid_visualization'));
     loadTestFile(require.resolve('./precalculated_histogram'));
     loadTestFile(require.resolve('./preserve_url'));
+    loadTestFile(require.resolve('./reporting'));
   });
 }
diff --git a/x-pack/test/functional/apps/visualize/reporting.js b/x-pack/test/functional/apps/visualize/reporting.js
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+export default function({ getService, getPageObjects }) {
+  const esArchiver = getService('esArchiver');
+  const browser = getService('browser');
+  const log = getService('log');
+  const PageObjects = getPageObjects([
+    'reporting',
+    'common',
+    'dashboard',
+    'visualize',
+    'visEditor',
+  ]);
+
+  describe('Visualize', () => {
+    before('initialize tests', async () => {
+      log.debug('ReportingPage:initTests');
+      await esArchiver.loadIfNeeded('reporting/ecommerce');
+      await esArchiver.loadIfNeeded('reporting/ecommerce_kibana');
+      await browser.setWindowSize(1600, 850);
+    });
+    after('clean up archives', async () => {
+      await esArchiver.unload('reporting/ecommerce');
+      await esArchiver.unload('reporting/ecommerce_kibana');
+    });
+
+    describe('Print PDF button', () => {
+      it('is not available if new', async () => {
+        await PageObjects.common.navigateToUrl('visualize', 'new');
+        await PageObjects.visualize.clickAreaChart();
+        await PageObjects.visualize.clickNewSearch('ecommerce');
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.reporting.setTimepickerInDataRange();
+        await PageObjects.visEditor.clickBucket('X-axis');
+        await PageObjects.visEditor.selectAggregation('Date Histogram');
+        await PageObjects.visEditor.clickGo();
+        await PageObjects.visualize.saveVisualization('my viz');
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+
+      it('downloaded PDF has OK status', async function() {
+        // Generating and then comparing reports can take longer than the default 60s timeout
+        this.timeout(180000);
+
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
+        await PageObjects.reporting.openPdfReportingPanel();
+        await PageObjects.reporting.clickGenerateReportButton();
+
+        const url = await PageObjects.reporting.getReportURL(60000);
+        const res = await PageObjects.reporting.getResponse(url);
+
+        expect(res.statusCode).to.equal(200);
+        expect(res.headers['content-type']).to.equal('application/pdf');
+      });
+    });
+  });
+}
diff --git a/x-pack/test/functional/page_objects/reporting_page.js b/x-pack/test/functional/page_objects/reporting_page.js
@@ -17,7 +17,6 @@ import http from 'http';
 export function ReportingPageProvider({ getService, getPageObjects }) {
   const retry = getService('retry');
   const log = getService('log');
-  const config = getService('config');
   const testSubjects = getService('testSubjects');
   const browser = getService('browser');
   const PageObjects = getPageObjects(['common', 'security', 'share', 'timePicker']);
@@ -51,7 +50,7 @@ export function ReportingPageProvider({ getService, getPageObjects }) {
 
     getResponse(url) {
       log.debug(`getResponse for ${url}`);
-      const auth = config.get('servers.elasticsearch.auth');
+      const auth = 'test_user:changeme'; // FIXME not sure why there is no config that can be read for this
       const headers = {
         Authorization: `Basic ${Buffer.from(auth).toString('base64')}`,
       };
@@ -71,6 +70,7 @@ export function ReportingPageProvider({ getService, getPageObjects }) {
             }
           )
           .on('error', e => {
+            log.error(e);
             reject(e);
           });
       });