Skip to content

Commit

Permalink
[8.17] [Infra] Running processes missing from processes table (#209076)…
Browse files Browse the repository at this point in the history
… (#209473)

# Backport

This will backport the following commits from `main` to `8.17`:
- [[Infra] Running processes missing from processes table
(#209076)](#209076)

<!--- Backport version: 9.6.4 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT
[{"author":{"name":"jennypavlova","email":"dzheni.pavlova@elastic.co"},"sourceCommit":{"committedDate":"2025-02-04T08:25:36Z","message":"[Infra]
Running processes missing from processes table (#209076)\n\nCloses
#184582\r\n\r\n## Summary\r\n\r\nThis PR fixes an issue with processes
missing from the processes table\r\nand not being searchable.\r\n\r\n-
Additional fixes: The chart size was not set correctly and they
were\r\nnot visible so it is fixed now:\r\n\r\n| Before | After |\r\n|
---- | ------
|\r\n|\r\n![image](https://github.com/user-attachments/assets/3bd0788d-a89e-4cac-84af-40f6cf101613)\r\n|\r\n![image](https://github.com/user-attachments/assets/21ebbdc0-fcfa-4de5-a7f4-cbe855eceb7a)\r\n|\r\n\r\n⚠️
_UPDATE_ I also updated the archive used in the API test with the
one\r\nwe have for the e2e name (and matched the name with the old one)
so we\r\nhave consistent results and also the correct fields /
mappings\r\n\r\n### Solution details\r\n\r\nAfter some digging into the
fields I ended up **not** using the\r\nsuggested `process.name` because
it gave us too generic information for\r\nthe table (we should still use
the command there) also tried the process\r\nid but then the search by
command won't work properly. So I found a\r\nfield that is not ignored
and contains the same information called\r\n`process.command_line`and
used it:\r\n<img width=\"538\"
alt=\"image\"\r\nsrc=\"https://github.com/user-attachments/assets/ce4d35a6-e912-4656-bee1-8137d5635432\"\r\n/>\r\n\r\n##
Testing\r\n\r\n- Follow the steps in the issue inside `How to reproduce`
to create the\r\ntwo processes\r\n- Note: in order to see the process
summary, you need to enable it in\r\nthe config ( ⚠️ for the oblt hosts
I checked it is not enabled and the\r\nsummary section showed N/A )
:\r\n\r\n![image](https://github.com/user-attachments/assets/ffd5fa1c-d909-4ec1-ab35-99186fc8b5e3)\r\n\r\n\r\nhttps://github.com/user-attachments/assets/41bdcc8d-6b87-4351-b866-2e706dd11dfd\r\n\r\n
- The metrics charts should be visible\r\n - The search should work as
expected:\r\n - And the missing.... process should be
visible","sha":"1f52febb1cd43724e81514af398049f64c516199","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","backport:prev-minor","backport:prev-major","Team:obs-ux-infra_services","v9.1.0"],"title":"[Infra]
Running processes missing from processes
table","number":209076,"url":"https://github.com/elastic/kibana/pull/209076","mergeCommit":{"message":"[Infra]
Running processes missing from processes table (#209076)\n\nCloses
#184582\r\n\r\n## Summary\r\n\r\nThis PR fixes an issue with processes
missing from the processes table\r\nand not being searchable.\r\n\r\n-
Additional fixes: The chart size was not set correctly and they
were\r\nnot visible so it is fixed now:\r\n\r\n| Before | After |\r\n|
---- | ------
|\r\n|\r\n![image](https://github.com/user-attachments/assets/3bd0788d-a89e-4cac-84af-40f6cf101613)\r\n|\r\n![image](https://github.com/user-attachments/assets/21ebbdc0-fcfa-4de5-a7f4-cbe855eceb7a)\r\n|\r\n\r\n⚠️
_UPDATE_ I also updated the archive used in the API test with the
one\r\nwe have for the e2e name (and matched the name with the old one)
so we\r\nhave consistent results and also the correct fields /
mappings\r\n\r\n### Solution details\r\n\r\nAfter some digging into the
fields I ended up **not** using the\r\nsuggested `process.name` because
it gave us too generic information for\r\nthe table (we should still use
the command there) also tried the process\r\nid but then the search by
command won't work properly. So I found a\r\nfield that is not ignored
and contains the same information called\r\n`process.command_line`and
used it:\r\n<img width=\"538\"
alt=\"image\"\r\nsrc=\"https://github.com/user-attachments/assets/ce4d35a6-e912-4656-bee1-8137d5635432\"\r\n/>\r\n\r\n##
Testing\r\n\r\n- Follow the steps in the issue inside `How to reproduce`
to create the\r\ntwo processes\r\n- Note: in order to see the process
summary, you need to enable it in\r\nthe config ( ⚠️ for the oblt hosts
I checked it is not enabled and the\r\nsummary section showed N/A )
:\r\n\r\n![image](https://github.com/user-attachments/assets/ffd5fa1c-d909-4ec1-ab35-99186fc8b5e3)\r\n\r\n\r\nhttps://github.com/user-attachments/assets/41bdcc8d-6b87-4351-b866-2e706dd11dfd\r\n\r\n
- The metrics charts should be visible\r\n - The search should work as
expected:\r\n - And the missing.... process should be
visible","sha":"1f52febb1cd43724e81514af398049f64c516199"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/209076","number":209076,"mergeCommit":{"message":"[Infra]
Running processes missing from processes table (#209076)\n\nCloses
#184582\r\n\r\n## Summary\r\n\r\nThis PR fixes an issue with processes
missing from the processes table\r\nand not being searchable.\r\n\r\n-
Additional fixes: The chart size was not set correctly and they
were\r\nnot visible so it is fixed now:\r\n\r\n| Before | After |\r\n|
---- | ------
|\r\n|\r\n![image](https://github.com/user-attachments/assets/3bd0788d-a89e-4cac-84af-40f6cf101613)\r\n|\r\n![image](https://github.com/user-attachments/assets/21ebbdc0-fcfa-4de5-a7f4-cbe855eceb7a)\r\n|\r\n\r\n⚠️
_UPDATE_ I also updated the archive used in the API test with the
one\r\nwe have for the e2e name (and matched the name with the old one)
so we\r\nhave consistent results and also the correct fields /
mappings\r\n\r\n### Solution details\r\n\r\nAfter some digging into the
fields I ended up **not** using the\r\nsuggested `process.name` because
it gave us too generic information for\r\nthe table (we should still use
the command there) also tried the process\r\nid but then the search by
command won't work properly. So I found a\r\nfield that is not ignored
and contains the same information called\r\n`process.command_line`and
used it:\r\n<img width=\"538\"
alt=\"image\"\r\nsrc=\"https://github.com/user-attachments/assets/ce4d35a6-e912-4656-bee1-8137d5635432\"\r\n/>\r\n\r\n##
Testing\r\n\r\n- Follow the steps in the issue inside `How to reproduce`
to create the\r\ntwo processes\r\n- Note: in order to see the process
summary, you need to enable it in\r\nthe config ( ⚠️ for the oblt hosts
I checked it is not enabled and the\r\nsummary section showed N/A )
:\r\n\r\n![image](https://github.com/user-attachments/assets/ffd5fa1c-d909-4ec1-ab35-99186fc8b5e3)\r\n\r\n\r\nhttps://github.com/user-attachments/assets/41bdcc8d-6b87-4351-b866-2e706dd11dfd\r\n\r\n
- The metrics charts should be visible\r\n - The search should work as
expected:\r\n - And the missing.... process should be
visible","sha":"1f52febb1cd43724e81514af398049f64c516199"}},{"url":"https://github.com/elastic/kibana/pull/209467","number":209467,"branch":"8.18","state":"OPEN"},{"url":"https://github.com/elastic/kibana/pull/209468","number":209468,"branch":"8.x","state":"OPEN"},{"url":"https://github.com/elastic/kibana/pull/209469","number":209469,"branch":"9.0","state":"OPEN"}]}]
BACKPORT-->
  • Loading branch information
jennypavlova authored Feb 4, 2025
1 parent 795ba16 commit 6957fe2
Show file tree
Hide file tree
Showing 10 changed files with 26,415 additions and 57 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export const TIEBREAKER_FIELD = '_doc';
export const HOST_NAME_FIELD = 'host.name';
export const CONTAINER_ID_FIELD = 'container.id';
export const KUBERNETES_POD_UID_FIELD = 'kubernetes.pod.uid';
export const SYSTEM_PROCESS_CMDLINE_FIELD = 'system.process.cmdline';
export const PROCESS_COMMANDLINE_FIELD = 'process.command_line';
export const EVENT_MODULE = 'event.module';
export const METRICSET_MODULE = 'metricset.module';
export const METRICSET_NAME = 'metricset.name';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ export const ProcessListAPIQueryAggregationRT = rt.type({
_source: rt.type({
process: rt.type({
pid: rt.number,
command_line: rt.string,
}),
system: rt.type({
process: rt.type({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
* 2.0.
*/

import { PROCESS_COMMANDLINE_FIELD } from '../../../../../common/constants';

export const parseSearchString = (query: string) => {
if (query.trim() === '') {
return [
Expand All @@ -22,7 +24,7 @@ export const parseSearchString = (query: string) => {
return [
...cmdlineFilters.map((clause) => ({
query_string: {
fields: ['system.process.cmdline'],
fields: [PROCESS_COMMANDLINE_FIELD],
query: `*${escapeReservedCharacters(clause)}*`,
minimum_should_match: 1,
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,44 +168,40 @@ const ProcessChart = ({ timeseries, color, label }: ProcessChartProps) => {
: { max: 0, min: 0 };

return (
<div
css={css`
{
width: 100%;
height: 140px;
}
`}
<Chart
size={{
height: '140px',
width: '100%',
}}
>
<Chart>
<MetricExplorerSeriesChart
type={MetricsExplorerChartType.area}
metric={chartMetric}
id="0"
series={timeseries}
stack={false}
/>
<Axis
id={'timestamp'}
position={Position.Bottom}
showOverlappingTicks={true}
tickFormat={dateFormatter}
/>
<Axis
id={'values'}
position={Position.Left}
tickFormat={yAxisFormatter}
domain={domain}
ticks={6}
gridLine={{ visible: true }}
/>
<Tooltip headerFormatter={({ value }) => moment(value).format('Y-MM-DD HH:mm:ss.SSS')} />
<Settings
baseTheme={chartTheme.baseTheme}
theme={chartTheme.theme}
locale={i18n.getLocale()}
/>
</Chart>
</div>
<MetricExplorerSeriesChart
type={MetricsExplorerChartType.area}
metric={chartMetric}
id="0"
series={timeseries}
stack={false}
/>
<Axis
id={'timestamp'}
position={Position.Bottom}
showOverlappingTicks={true}
tickFormat={dateFormatter}
/>
<Axis
id={'values'}
position={Position.Left}
tickFormat={yAxisFormatter}
domain={domain}
ticks={6}
gridLine={{ visible: true }}
/>
<Tooltip headerFormatter={({ value }) => moment(value).format('Y-MM-DD HH:mm:ss.SSS')} />
<Settings
baseTheme={chartTheme.baseTheme}
theme={chartTheme.theme}
locale={i18n.getLocale()}
/>
</Chart>
);
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { TIMESTAMP_FIELD, SYSTEM_PROCESS_CMDLINE_FIELD } from '../../../common/constants';
import { ProcessListAPIRequest, ProcessListAPIQueryAggregation } from '../../../common/http_api';
import { ESSearchClient } from '../metrics/types';
import { TIMESTAMP_FIELD, PROCESS_COMMANDLINE_FIELD } from '../../../common/constants';
import type {
ProcessListAPIRequest,
ProcessListAPIQueryAggregation,
} from '../../../common/http_api';
import type { ESSearchClient } from '../metrics/types';
import type { InfraSourceConfiguration } from '../sources';

const TOP_N = 10;
Expand Down Expand Up @@ -69,7 +71,7 @@ export const getProcessList = async (
aggs: {
filteredProcs: {
terms: {
field: SYSTEM_PROCESS_CMDLINE_FIELD,
field: PROCESS_COMMANDLINE_FIELD,
size: TOP_N,
order: {
[sortBy.name]: sortBy.isAscending ? 'asc' : 'desc',
Expand Down Expand Up @@ -101,7 +103,12 @@ export const getProcessList = async (
},
},
],
_source: ['system.process.state', 'user.name', 'process.pid'],
_source: [
'system.process.state',
'user.name',
'process.pid',
'process.command_line',
],
},
},
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
*/

import { first } from 'lodash';
import { TIMESTAMP_FIELD, SYSTEM_PROCESS_CMDLINE_FIELD } from '../../../common/constants';
import {
import { TIMESTAMP_FIELD, PROCESS_COMMANDLINE_FIELD } from '../../../common/constants';
import type {
ProcessListAPIChartRequest,
ProcessListAPIChartQueryAggregation,
ProcessListAPIRow,
Expand Down Expand Up @@ -48,7 +48,7 @@ export const getProcessListChart = async (
must: [
{
match: {
[SYSTEM_PROCESS_CMDLINE_FIELD]: command,
[PROCESS_COMMANDLINE_FIELD]: command,
},
},
],
Expand All @@ -57,7 +57,7 @@ export const getProcessListChart = async (
aggs: {
filteredProc: {
terms: {
field: SYSTEM_PROCESS_CMDLINE_FIELD,
field: PROCESS_COMMANDLINE_FIELD,
size: 1,
},
aggs: {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,20 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {

describe('API /api/metrics/process_list', () => {
let supertestWithAdminScope: SupertestWithRoleScopeType;

before(async () => {
supertestWithAdminScope = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
withInternalHeaders: true,
useCookieHeader: true,
});
await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
await esArchiver.load(
'x-pack/test/functional/es_archives/infra/8.0.0/metrics_hosts_processes'
);
});
after(async () => {
await esArchiver.unload('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
await esArchiver.unload(
'x-pack/test/functional/es_archives/infra/8.0.0/metrics_hosts_processes'
);
await supertestWithAdminScope.destroy();
});

Expand All @@ -42,7 +47,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
'host.name': 'gke-observability-8--observability-8--bc1afd95-nhhw',
},
sourceId: 'default',
to: 1564432800000,
to: 1680027660000,
sortBy: {
name: 'cpu',
isAscending: false,
Expand All @@ -59,7 +64,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
const { processList, summary } = decodeOrThrow(ProcessListAPIResponseRT)(response.body);

expect(processList.length).to.be(10);
expect(summary.total).to.be(178);
expect(summary.total).to.be(313);
});
});
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,14 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
withInternalHeaders: true,
useCookieHeader: true,
});
await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
await esArchiver.load(
'x-pack/test/functional/es_archives/infra/8.0.0/metrics_hosts_processes'
);
});
after(async () => {
await esArchiver.unload('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
await esArchiver.unload(
'x-pack/test/functional/es_archives/infra/8.0.0/metrics_hosts_processes'
);
await supertestWithAdminScope.destroy();
});

Expand All @@ -43,8 +47,9 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
'host.name': 'gke-observability-8--observability-8--bc1afd95-nhhw',
},
indexPattern: 'metrics-*,metricbeat-*',
to: 1564432800000,
command: '/usr/lib/systemd/systemd-journald',
to: 1680027660000,
command:
'/System/Library/CoreServices/NotificationCenter.app/Contents/MacOS/NotificationCenter',
})
)
.expect(200);
Expand Down
Binary file not shown.
Loading

0 comments on commit 6957fe2

Please sign in to comment.