[Reporting] Fix CSV generator to prevent columns from being shifted (#…

…140971)
elastic · Sep 23, 2022 · 54c9c2d · 54c9c2d
1 parent f3a3243
commit 54c9c2d
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 8 deletions.
diff --git a/...server/export_types/csv_searchsource/generate_csv/__snapshots__/generate_csv.test.ts.snap b/...server/export_types/csv_searchsource/generate_csv/__snapshots__/generate_csv.test.ts.snap
diff --git a/.../plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts b/.../plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts
@@ -353,6 +353,73 @@ it('uses the scrollId to page all the data', async () => {
   });
 });
 
+it('keeps order of the columns during the scroll', async () => {
+  mockDataClient.search = jest.fn().mockImplementation(() =>
+    Rx.of({
+      rawResponse: {
+        _scroll_id: 'awesome-scroll-hero',
+        hits: {
+          hits: [
+            {
+              fields: {
+                a: ['a1'],
+                b: ['b1'],
+              },
+            },
+          ],
+          total: 3,
+        },
+      },
+    })
+  );
+
+  mockEsClient.asCurrentUser.scroll = jest
+    .fn()
+    .mockResolvedValueOnce({
+      hits: {
+        hits: [
+          {
+            fields: {
+              b: ['b2'],
+            },
+          },
+        ],
+      },
+    })
+    .mockResolvedValueOnce({
+      hits: {
+        hits: [
+          {
+            fields: {
+              a: ['a3'],
+              c: ['c3'],
+            },
+          },
+        ],
+      },
+    });
+
+  const generateCsv = new CsvGenerator(
+    createMockJob({ searchSource: {}, columns: [] }),
+    mockConfig,
+    {
+      es: mockEsClient,
+      data: mockDataClient,
+      uiSettings: uiSettingsClient,
+    },
+    {
+      searchSourceStart: mockSearchSourceService,
+      fieldFormatsRegistry: mockFieldFormatsRegistry,
+    },
+    new CancellationToken(),
+    mockLogger,
+    stream
+  );
+  await generateCsv.generateData();
+
+  expect(content).toMatchSnapshot();
+});
+
 describe('fields from job.searchSource.getFields() (7.12 generated)', () => {
   it('cells can be multi-value', async () => {
     mockDataClient.search = jest.fn().mockImplementation(() =>

diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts
@@ -193,12 +193,13 @@ export class CsvGenerator {
    * Use the list of columns to generate the header row
    */
   private generateHeader(
-    columns: string[],
+    columns: Set<string>,
     builder: MaxSizeStringBuilder,
     settings: CsvExportSettings
   ) {
     this.logger.debug(`Building CSV header row...`);
-    const header = columns.map(this.escapeValues(settings)).join(settings.separator) + '\n';
+    const header =
+      Array.from(columns).map(this.escapeValues(settings)).join(settings.separator) + '\n';
 
     if (!builder.tryAppend(header)) {
       return {
@@ -213,7 +214,7 @@ export class CsvGenerator {
    * Format a Datatable into rows of CSV content
    */
   private async generateRows(
-    columns: string[],
+    columns: Set<string>,
     table: Datatable,
     builder: MaxSizeStringBuilder,
     formatters: Record<string, FieldFormat>,
@@ -315,6 +316,7 @@ export class CsvGenerator {
       this.logger.error(err);
     }
 
+    const columns = new Set<string>(this.job.columns ?? []);
     try {
       do {
         if (this.cancellationToken.isCancelled()) {
@@ -366,11 +368,8 @@ export class CsvGenerator {
           break;
         }
 
-        let columns: string[];
-        if (this.job.columns && this.job.columns.length > 0) {
-          columns = this.job.columns;
-        } else {
-          columns = this.getColumnsFromTabify(table);
+        if (!this.job.columns?.length) {
+          this.getColumnsFromTabify(table).forEach((column) => columns.add(column));
         }
 
         if (first) {