[SO Migrations v2] Log doc-count progress (#99865) (#100036)

Co-authored-by: Alejandro Fernández Haro <alejandro.haro@elastic.co>

docs/development/core/public/kibana-plugin-core-public.doclinksstart.md

@@ -17,7 +17,5 @@ export interface DocLinksStart
 |  --- | --- | --- |
 |  [DOC\_LINK\_VERSION](./kibana-plugin-core-public.doclinksstart.doc_link_version.md) | <code>string</code> |  |
 |  [ELASTIC\_WEBSITE\_URL](./kibana-plugin-core-public.doclinksstart.elastic_website_url.md) | <code>string</code> |  |
-
 |  [links](./kibana-plugin-core-public.doclinksstart.links.md) | <code>{</code><br/><code>        readonly canvas: {</code><br/><code>            readonly guide: string;</code><br/><code>        };</code><br/><code>        readonly dashboard: {</code><br/><code>            readonly guide: string;</code><br/><code>            readonly drilldowns: string;</code><br/><code>            readonly drilldownsTriggerPicker: string;</code><br/><code>            readonly urlDrilldownTemplateSyntax: string;</code><br/><code>            readonly urlDrilldownVariables: string;</code><br/><code>        };</code><br/><code>        readonly discover: Record&lt;string, string&gt;;</code><br/><code>        readonly filebeat: {</code><br/><code>            readonly base: string;</code><br/><code>            readonly installation: string;</code><br/><code>            readonly configuration: string;</code><br/><code>            readonly elasticsearchOutput: string;</code><br/><code>            readonly elasticsearchModule: string;</code><br/><code>            readonly startup: string;</code><br/><code>            readonly exportedFields: string;</code><br/><code>        };</code><br/><code>        readonly auditbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly metricbeat: {</code><br/><code>            readonly base: string;</code><br/><code>            readonly configure: string;</code><br/><code>            readonly httpEndpoint: string;</code><br/><code>            readonly install: string;</code><br/><code>            readonly start: string;</code><br/><code>        };</code><br/><code>        readonly enterpriseSearch: {</code><br/><code>            readonly base: string;</code><br/><code>            readonly appSearchBase: string;</code><br/><code>            readonly workplaceSearchBase: string;</code><br/><code>        };</code><br/><code>        readonly heartbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly logstash: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly functionbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly winlogbeat: {</code><br/><code>            readonly base: string;</code><br/><code>        };</code><br/><code>        readonly aggs: {</code><br/><code>            readonly composite: string;</code><br/><code>            readonly composite_missing_bucket: string;</code><br/><code>            readonly date_histogram: string;</code><br/><code>            readonly date_range: string;</code><br/><code>            readonly date_format_pattern: string;</code><br/><code>            readonly filter: string;</code><br/><code>            readonly filters: string;</code><br/><code>            readonly geohash_grid: string;</code><br/><code>            readonly histogram: string;</code><br/><code>            readonly ip_range: string;</code><br/><code>            readonly range: string;</code><br/><code>            readonly significant_terms: string;</code><br/><code>            readonly terms: string;</code><br/><code>            readonly avg: string;</code><br/><code>            readonly avg_bucket: string;</code><br/><code>            readonly max_bucket: string;</code><br/><code>            readonly min_bucket: string;</code><br/><code>            readonly sum_bucket: string;</code><br/><code>            readonly cardinality: string;</code><br/><code>            readonly count: string;</code><br/><code>            readonly cumulative_sum: string;</code><br/><code>            readonly derivative: string;</code><br/><code>            readonly geo_bounds: string;</code><br/><code>            readonly geo_centroid: string;</code><br/><code>            readonly max: string;</code><br/><code>            readonly median: string;</code><br/><code>            readonly min: string;</code><br/><code>            readonly moving_avg: string;</code><br/><code>            readonly percentile_ranks: string;</code><br/><code>            readonly serial_diff: string;</code><br/><code>            readonly std_dev: string;</code><br/><code>            readonly sum: string;</code><br/><code>            readonly top_hits: string;</code><br/><code>        };</code><br/><code>        readonly runtimeFields: {</code><br/><code>            readonly overview: string;</code><br/><code>            readonly mapping: string;</code><br/><code>        };</code><br/><code>        readonly scriptedFields: {</code><br/><code>            readonly scriptFields: string;</code><br/><code>            readonly scriptAggs: string;</code><br/><code>            readonly painless: string;</code><br/><code>            readonly painlessApi: string;</code><br/><code>            readonly painlessLangSpec: string;</code><br/><code>            readonly painlessSyntax: string;</code><br/><code>            readonly painlessWalkthrough: string;</code><br/><code>            readonly luceneExpressions: string;</code><br/><code>        };</code><br/><code>        readonly search: {</code><br/><code>            readonly sessions: string;</code><br/><code>        };</code><br/><code>        readonly indexPatterns: {</code><br/><code>            readonly introduction: string;</code><br/><code>            readonly fieldFormattersNumber: string;</code><br/><code>            readonly fieldFormattersString: string;</code><br/><code>        };</code><br/><code>        readonly addData: string;</code><br/><code>        readonly kibana: string;</code><br/><code>        readonly upgradeAssistant: string;</code><br/><code>        readonly elasticsearch: Record&lt;string, string&gt;;</code><br/><code>        readonly siem: {</code><br/><code>            readonly guide: string;</code><br/><code>            readonly gettingStarted: string;</code><br/><code>        };</code><br/><code>        readonly query: {</code><br/><code>            readonly eql: string;</code><br/><code>            readonly kueryQuerySyntax: string;</code><br/><code>            readonly luceneQuerySyntax: string;</code><br/><code>            readonly percolate: string;</code><br/><code>            readonly queryDsl: string;</code><br/><code>        };</code><br/><code>        readonly date: {</code><br/><code>            readonly dateMath: string;</code><br/><code>            readonly dateMathIndexNames: string;</code><br/><code>        };</code><br/><code>        readonly management: Record&lt;string, string&gt;;</code><br/><code>        readonly ml: Record&lt;string, string&gt;;</code><br/><code>        readonly transforms: Record&lt;string, string&gt;;</code><br/><code>        readonly visualize: Record&lt;string, string&gt;;</code><br/><code>        readonly apis: Readonly&lt;{</code><br/><code>            bulkIndexAlias: string;</code><br/><code>            byteSizeUnits: string;</code><br/><code>            createAutoFollowPattern: string;</code><br/><code>            createFollower: string;</code><br/><code>            createIndex: string;</code><br/><code>            createSnapshotLifecyclePolicy: string;</code><br/><code>            createRoleMapping: string;</code><br/><code>            createRoleMappingTemplates: string;</code><br/><code>            createRollupJobsRequest: string;</code><br/><code>            createApiKey: string;</code><br/><code>            createPipeline: string;</code><br/><code>            createTransformRequest: string;</code><br/><code>            cronExpressions: string;</code><br/><code>            executeWatchActionModes: string;</code><br/><code>            indexExists: string;</code><br/><code>            openIndex: string;</code><br/><code>            putComponentTemplate: string;</code><br/><code>            painlessExecute: string;</code><br/><code>            painlessExecuteAPIContexts: string;</code><br/><code>            putComponentTemplateMetadata: string;</code><br/><code>            putSnapshotLifecyclePolicy: string;</code><br/><code>            putIndexTemplateV1: string;</code><br/><code>            putWatch: string;</code><br/><code>            simulatePipeline: string;</code><br/><code>            timeUnits: string;</code><br/><code>            updateTransform: string;</code><br/><code>        }&gt;;</code><br/><code>        readonly observability: Record&lt;string, string&gt;;</code><br/><code>        readonly alerting: Record&lt;string, string&gt;;</code><br/><code>        readonly maps: Record&lt;string, string&gt;;</code><br/><code>        readonly monitoring: Record&lt;string, string&gt;;</code><br/><code>        readonly security: Readonly&lt;{</code><br/><code>            apiKeyServiceSettings: string;</code><br/><code>            clusterPrivileges: string;</code><br/><code>            elasticsearchSettings: string;</code><br/><code>            elasticsearchEnableSecurity: string;</code><br/><code>            indicesPrivileges: string;</code><br/><code>            kibanaTLS: string;</code><br/><code>            kibanaPrivileges: string;</code><br/><code>            mappingRoles: string;</code><br/><code>            mappingRolesFieldRules: string;</code><br/><code>            runAsPrivilege: string;</code><br/><code>        }&gt;;</code><br/><code>        readonly watcher: Record&lt;string, string&gt;;</code><br/><code>        readonly ccs: Record&lt;string, string&gt;;</code><br/><code>        readonly plugins: Record&lt;string, string&gt;;</code><br/><code>        readonly snapshotRestore: Record&lt;string, string&gt;;</code><br/><code>        readonly ingest: Record&lt;string, string&gt;;</code><br/><code>    }</code> |  |
 
-

src/core/server/saved_objects/migrationsv2/actions/index.ts

@@ -456,6 +456,7 @@ export const openPit = (
 export interface ReadWithPit {
   outdatedDocuments: SavedObjectsRawDoc[];
   readonly lastHitSortValue: number[] | undefined;
+  readonly totalHits: number | undefined;
 }
 
 /*
@@ -481,26 +482,35 @@ export const readWithPit = (
         pit: { id: pitId, keep_alive: pitKeepAlive },
         size: batchSize,
         search_after: searchAfter,
-        // Improve performance by not calculating the total number of hits
-        // matching the query.
-        track_total_hits: false,
+        /**
+         * We want to know how many documents we need to process so we can log the progress.
+         * But we also want to increase the performance of these requests,
+         * so we ask ES to report the total count only on the first request (when searchAfter does not exist)
+         */
+        track_total_hits: typeof searchAfter === 'undefined',
         query,
       },
     })
     .then((response) => {
+      const totalHits =
+        typeof response.body.hits.total === 'number'
+          ? response.body.hits.total // This format is to be removed in 8.0
+          : response.body.hits.total?.value;
       const hits = response.body.hits.hits;
 
       if (hits.length > 0) {
         return Either.right({
           // @ts-expect-error @elastic/elasticsearch _source is optional
           outdatedDocuments: hits as SavedObjectsRawDoc[],
           lastHitSortValue: hits[hits.length - 1].sort as number[],
+          totalHits,
         });
       }
 
       return Either.right({
         outdatedDocuments: [],
         lastHitSortValue: undefined,
+        totalHits,
       });
     })
     .catch(catchRetryableEsClientErrors);