[Infra] Running processes missing from processes table (elastic#209076)

Closes elastic#184582 ## Summary This PR fixes an issue with processes missing from the processes table and not being searchable. - Additional fixes: The chart size was not set correctly and they were not visible so it is fixed now: | Before | After | | ---- | ------ | | ![image](https://github.com/user-attachments/assets/3bd0788d-a89e-4cac-84af-40f6cf101613) | ![image](https://github.com/user-attachments/assets/21ebbdc0-fcfa-4de5-a7f4-cbe855eceb7a) | ⚠️ _UPDATE_ I also updated the archive used in the API test with the one we have for the e2e name (and matched the name with the old one) so we have consistent results and also the correct fields / mappings ### Solution details After some digging into the fields I ended up **not** using the suggested `process.name` because it gave us too generic information for the table (we should still use the command there) also tried the process id but then the search by command won't work properly. So I found a field that is not ignored and contains the same information called `process.command_line`and used it: <img width="538" alt="image" src="https://github.com/user-attachments/assets/ce4d35a6-e912-4656-bee1-8137d5635432" /> ## Testing - Follow the steps in the issue inside `How to reproduce` to create the two processes - Note: in order to see the process summary, you need to enable it in the config ( ⚠️ for the oblt hosts I checked it is not enabled and the summary section showed N/A ) : ![image](https://github.com/user-attachments/assets/ffd5fa1c-d909-4ec1-ab35-99186fc8b5e3) https://github.com/user-attachments/assets/41bdcc8d-6b87-4351-b866-2e706dd11dfd - The metrics charts should be visible - The search should work as expected: - And the missing.... process should be visible
drewdaemon · Feb 6, 2025 · 24641e8 · 24641e8
1 parent 8100230
commit 24641e8
Show file tree

Hide file tree

Showing 10 changed files with 26,409 additions and 53 deletions.
diff --git a/x-pack/solutions/observability/plugins/infra/common/constants.ts b/x-pack/solutions/observability/plugins/infra/common/constants.ts
@@ -35,7 +35,7 @@ export const TIEBREAKER_FIELD = '_doc';
 export const HOST_NAME_FIELD = 'host.name';
 export const CONTAINER_ID_FIELD = 'container.id';
 export const KUBERNETES_POD_UID_FIELD = 'kubernetes.pod.uid';
-export const SYSTEM_PROCESS_CMDLINE_FIELD = 'system.process.cmdline';
+export const PROCESS_COMMANDLINE_FIELD = 'process.command_line';
 export const EVENT_MODULE = 'event.module';
 export const METRICSET_MODULE = 'metricset.module';
 export const METRICSET_NAME = 'metricset.name';

diff --git a/x-pack/solutions/observability/plugins/infra/common/http_api/host_details/process_list.ts b/x-pack/solutions/observability/plugins/infra/common/http_api/host_details/process_list.ts
@@ -58,6 +58,7 @@ export const ProcessListAPIQueryAggregationRT = rt.type({
                   _source: rt.type({
                     process: rt.type({
                       pid: rt.number,
+                      command_line: rt.string,
                     }),
                     system: rt.type({
                       process: rt.type({

diff --git a/...ility/plugins/infra/public/components/asset_details/tabs/processes/parse_search_string.ts b/...ility/plugins/infra/public/components/asset_details/tabs/processes/parse_search_string.ts
@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import { PROCESS_COMMANDLINE_FIELD } from '../../../../../common/constants';
+
 export const parseSearchString = (query: string) => {
   if (query.trim() === '') {
     return [
@@ -22,7 +24,7 @@ export const parseSearchString = (query: string) => {
   return [
     ...cmdlineFilters.map((clause) => ({
       query_string: {
-        fields: ['system.process.cmdline'],
+        fields: [PROCESS_COMMANDLINE_FIELD],
         query: `*${escapeReservedCharacters(clause)}*`,
         minimum_should_match: 1,
       },

diff --git a/...ility/plugins/infra/public/components/asset_details/tabs/processes/process_row_charts.tsx b/...ility/plugins/infra/public/components/asset_details/tabs/processes/process_row_charts.tsx
@@ -166,44 +166,40 @@ const ProcessChart = ({ timeseries, color, label }: ProcessChartProps) => {
     : { max: 0, min: 0 };
 
   return (
-    <div
-      css={css`
-         {
-          width: 100%;
-          height: 140px;
-        }
-      `}
+    <Chart
+      size={{
+        height: '140px',
+        width: '100%',
+      }}
     >
-      <Chart>
-        <MetricExplorerSeriesChart
-          type={MetricsExplorerChartType.area}
-          metric={chartMetric}
-          id="0"
-          series={timeseries}
-          stack={false}
-        />
-        <Axis
-          id={'timestamp'}
-          position={Position.Bottom}
-          showOverlappingTicks={true}
-          tickFormat={dateFormatter}
-        />
-        <Axis
-          id={'values'}
-          position={Position.Left}
-          tickFormat={yAxisFormatter}
-          domain={domain}
-          ticks={6}
-          gridLine={{ visible: true }}
-        />
-        <Tooltip headerFormatter={({ value }) => moment(value).format('Y-MM-DD HH:mm:ss.SSS')} />
-        <Settings
-          baseTheme={chartTheme.baseTheme}
-          theme={chartTheme.theme}
-          locale={i18n.getLocale()}
-        />
-      </Chart>
-    </div>
+      <MetricExplorerSeriesChart
+        type={MetricsExplorerChartType.area}
+        metric={chartMetric}
+        id="0"
+        series={timeseries}
+        stack={false}
+      />
+      <Axis
+        id={'timestamp'}
+        position={Position.Bottom}
+        showOverlappingTicks={true}
+        tickFormat={dateFormatter}
+      />
+      <Axis
+        id={'values'}
+        position={Position.Left}
+        tickFormat={yAxisFormatter}
+        domain={domain}
+        ticks={6}
+        gridLine={{ visible: true }}
+      />
+      <Tooltip headerFormatter={({ value }) => moment(value).format('Y-MM-DD HH:mm:ss.SSS')} />
+      <Settings
+        baseTheme={chartTheme.baseTheme}
+        theme={chartTheme.theme}
+        locale={i18n.getLocale()}
+      />
+    </Chart>
   );
 };
 

diff --git a/x-pack/solutions/observability/plugins/infra/server/lib/host_details/process_list.ts b/x-pack/solutions/observability/plugins/infra/server/lib/host_details/process_list.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { TIMESTAMP_FIELD, SYSTEM_PROCESS_CMDLINE_FIELD } from '../../../common/constants';
+import { TIMESTAMP_FIELD, PROCESS_COMMANDLINE_FIELD } from '../../../common/constants';
 import type {
   ProcessListAPIRequest,
   ProcessListAPIQueryAggregation,
@@ -72,7 +72,7 @@ export const getProcessList = async (
         aggs: {
           filteredProcs: {
             terms: {
-              field: SYSTEM_PROCESS_CMDLINE_FIELD,
+              field: PROCESS_COMMANDLINE_FIELD,
               size: TOP_N,
               order: {
                 [sortBy.name]: sortBy.isAscending ? 'asc' : 'desc',
@@ -104,7 +104,12 @@ export const getProcessList = async (
                       },
                     },
                   ],
-                  _source: ['system.process.state', 'user.name', 'process.pid'],
+                  _source: [
+                    'system.process.state',
+                    'user.name',
+                    'process.pid',
+                    'process.command_line',
+                  ],
                 },
               },
             },

diff --git a/x-pack/solutions/observability/plugins/infra/server/lib/host_details/process_list_chart.ts b/x-pack/solutions/observability/plugins/infra/server/lib/host_details/process_list_chart.ts
@@ -6,7 +6,7 @@
  */
 
 import { first } from 'lodash';
-import { TIMESTAMP_FIELD, SYSTEM_PROCESS_CMDLINE_FIELD } from '../../../common/constants';
+import { TIMESTAMP_FIELD, PROCESS_COMMANDLINE_FIELD } from '../../../common/constants';
 import type {
   ProcessListAPIChartRequest,
   ProcessListAPIChartQueryAggregation,
@@ -48,7 +48,7 @@ export const getProcessListChart = async (
             must: [
               {
                 match: {
-                  [SYSTEM_PROCESS_CMDLINE_FIELD]: command,
+                  [PROCESS_COMMANDLINE_FIELD]: command,
                 },
               },
             ],
@@ -57,7 +57,7 @@ export const getProcessListChart = async (
         aggs: {
           filteredProc: {
             terms: {
-              field: SYSTEM_PROCESS_CMDLINE_FIELD,
+              field: PROCESS_COMMANDLINE_FIELD,
               size: 1,
             },
             aggs: {

diff --git a/...test/api_integration/deployment_agnostic/apis/observability/infra/metrics_process_list.ts b/...test/api_integration/deployment_agnostic/apis/observability/infra/metrics_process_list.ts
@@ -21,15 +21,20 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
 
   describe('API /api/metrics/process_list', () => {
     let supertestWithAdminScope: SupertestWithRoleScopeType;
+
     before(async () => {
       supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
         withInternalHeaders: true,
         useCookieHeader: true,
       });
-      await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
+      await esArchiver.load(
+        'x-pack/test/functional/es_archives/infra/8.0.0/metrics_hosts_processes'
+      );
     });
     after(async () => {
-      await esArchiver.unload('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
+      await esArchiver.unload(
+        'x-pack/test/functional/es_archives/infra/8.0.0/metrics_hosts_processes'
+      );
       await supertestWithAdminScope.destroy();
     });
 
@@ -42,7 +47,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
               'host.name': 'gke-observability-8--observability-8--bc1afd95-nhhw',
             },
             sourceId: 'default',
-            to: 1564432800000,
+            to: 1680027660000,
             sortBy: {
               name: 'cpu',
               isAscending: false,
@@ -59,7 +64,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
       const { processList, summary } = decodeOrThrow(ProcessListAPIResponseRT)(response.body);
 
       expect(processList.length).to.be(10);
-      expect(summary.total).to.be(178);
+      expect(summary.total).to.be(313);
     });
   });
 }
diff --git a/...pi_integration/deployment_agnostic/apis/observability/infra/metrics_process_list_chart.ts b/...pi_integration/deployment_agnostic/apis/observability/infra/metrics_process_list_chart.ts
@@ -27,10 +27,14 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
         withInternalHeaders: true,
         useCookieHeader: true,
       });
-      await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
+      await esArchiver.load(
+        'x-pack/test/functional/es_archives/infra/8.0.0/metrics_hosts_processes'
+      );
     });
     after(async () => {
-      await esArchiver.unload('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
+      await esArchiver.unload(
+        'x-pack/test/functional/es_archives/infra/8.0.0/metrics_hosts_processes'
+      );
       await supertestWithAdminScope.destroy();
     });
 
@@ -43,8 +47,9 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
               'host.name': 'gke-observability-8--observability-8--bc1afd95-nhhw',
             },
             indexPattern: 'metrics-*,metricbeat-*',
-            to: 1564432800000,
-            command: '/usr/lib/systemd/systemd-journald',
+            to: 1680027660000,
+            command:
+              '/System/Library/CoreServices/NotificationCenter.app/Contents/MacOS/NotificationCenter',
           })
         )
         .expect(200);

diff --git a/x-pack/test/functional/es_archives/infra/8.0.0/metrics_hosts_processes/data.json.gz b/x-pack/test/functional/es_archives/infra/8.0.0/metrics_hosts_processes/data.json.gz