Skip to content

Commit

Permalink
Merge branch 'main' into fix_ui_counters_flaky
Browse files Browse the repository at this point in the history
  • Loading branch information
kibanamachine authored Nov 7, 2023
2 parents 6edccf0 + 0fce67d commit 8f0eace
Show file tree
Hide file tree
Showing 292 changed files with 22,449 additions and 5,027 deletions.
3 changes: 2 additions & 1 deletion .buildkite/ftr_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,6 @@ enabled:
- x-pack/test/detection_engine_api_integration/security_and_spaces/group1/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group4/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group5/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group6/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group10/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/prebuilt_rules/config.ts
Expand Down Expand Up @@ -461,6 +460,8 @@ enabled:
- x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts
- x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts
- x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts
- x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/serverless.config.ts
- x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/ess.config.ts



Expand Down
10 changes: 10 additions & 0 deletions .buildkite/pipelines/artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,16 @@ steps:
- exit_status: '*'
limit: 1

- command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh
label: 'Docker Context Verification'
agents:
queue: n2-2
timeout_in_minutes: 30
retry:
automatic:
- exit_status: '*'
limit: 1

- command: .buildkite/scripts/steps/artifacts/cloud.sh
label: 'Cloud Deployment'
soft_fail:
Expand Down
4 changes: 2 additions & 2 deletions .buildkite/scripts/pipelines/pull_request/pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,8 @@ const uploadPipeline = (pipelineContent: string | object) => {
}

if (
GITHUB_PR_LABELS.includes('ci:project-deploy-es') ||
GITHUB_PR_LABELS.includes('ci:project-deploy-oblt') ||
GITHUB_PR_LABELS.includes('ci:project-deploy-elasticsearch') ||
GITHUB_PR_LABELS.includes('ci:project-deploy-observability') ||
GITHUB_PR_LABELS.includes('ci:project-deploy-security')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_project.yml'));
Expand Down
40 changes: 29 additions & 11 deletions .buildkite/scripts/steps/artifacts/docker_context.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,43 @@ KIBANA_DOCKER_CONTEXT="${KIBANA_DOCKER_CONTEXT:="default"}"

echo "--- Create contexts"
mkdir -p target
node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact "${BUILD_ARGS[@]}"
node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --skip-cdn-assets --docker-context-use-local-artifact

echo "--- Setup context"
DOCKER_BUILD_FOLDER=$(mktemp -d)
DOCKER_BUILD_ARGS=''
case $KIBANA_DOCKER_CONTEXT in
default)
DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
;;
cloud)
DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
;;
ubi8)
DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
;;
ubi9)
DOCKER_CONTEXT_FILE="kibana-ubi9-$FULL_VERSION-docker-build-context.tar.gz"
;;
ironbank)
DOCKER_CONTEXT_FILE="kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz"
DOCKER_BUILD_ARGS='--build-arg BASE_REGISTRY=docker.elastic.co --build-arg BASE_IMAGE=ubi9/ubi --build-arg BASE_TAG=latest'

# See src/dev/build/tasks/os_packages/docker_generator/templates/ironbank/hardening_manifest.yaml
curl --retry 8 -sS -L -o "$DOCKER_BUILD_FOLDER/tini" "https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64"
echo "8053cc21a3a9bdd6042a495349d1856ae8d3b3e7664c9654198de0087af031f5d41139ec85a2f5d7d2febd22ec3f280767ff23b9d5f63d490584e2b7ad3c218c $DOCKER_BUILD_FOLDER/tini" | sha512sum -c -
curl --retry 8 -sS -L -o "$DOCKER_BUILD_FOLDER/NotoSansCJK-Regular.ttc" https://github.com/googlefonts/noto-cjk/raw/NotoSansV2.001/NotoSansCJK-Regular.ttc
echo "0ce56bde1853fed3e53282505bac65707385275a27816c29712ab04c187aa249797c82c58759b2b36c210d4e2683eda92359d739a8045cb8385c2c34d37cc9e1 $DOCKER_BUILD_FOLDER/NotoSansCJK-Regular.ttc" | sha512sum -c -

echo "Warning: this will build an approximation of the Iron Bank context, using a swapped base image"
;;
esac

if [[ "$KIBANA_DOCKER_CONTEXT" == "default" ]]; then
DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
elif [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi8" ]]; then
DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi9" ]]; then
DOCKER_CONTEXT_FILE="kibana-ubi9-$FULL_VERSION-docker-build-context.tar.gz"
fi

tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"
cd $DOCKER_BUILD_FOLDER

download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"

echo "--- Build context"
docker build .
docker build $DOCKER_BUILD_ARGS .
4 changes: 4 additions & 0 deletions .buildkite/scripts/steps/cloud/purge_deployments.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,8 @@

set -euo pipefail

echo '--- Purging Cloud deployments'
ts-node .buildkite/scripts/steps/cloud/purge_deployments.ts

echo '--- Purging Project deployments'
ts-node .buildkite/scripts/steps/cloud/purge_projects.ts
124 changes: 124 additions & 0 deletions .buildkite/scripts/steps/cloud/purge_projects.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { execSync } from 'child_process';
import axios from 'axios';

async function getPrProjects() {
const match = /^kibana-pr-([0-9]+)-(elasticsearch|security|observability)$/;
try {
return (
await Promise.all([
projectRequest.get('/api/v1/serverless/projects/elasticsearch'),
projectRequest.get('/api/v1/serverless/projects/security'),
projectRequest.get('/api/v1/serverless/projects/observability'),
])
)
.map((response) => response.data.items)
.flat()
.filter((project) => project.name.match(match))
.map((project) => {
const [, prNumber, projectType] = project.name.match(match);
return {
id: project.id,
name: project.name,
prNumber,
type: projectType,
};
});
} catch (e) {
if (e.isAxiosError) {
const message = JSON.stringify(e.response.data) || 'unable to fetch projects';
throw new Error(message);
}
throw e;
}
}

async function deleteProject({
type,
id,
}: {
type: 'elasticsearch' | 'observability' | 'security';
id: number;
}) {
try {
await projectRequest.delete(`/api/v1/serverless/projects/${type}/${id}`);
} catch (e) {
if (e.isAxiosError) {
const message =
JSON.stringify(e.response.data) || `unable to delete ${type} project with id ${id}`;
throw new Error(message);
}
throw e;
}
}

async function purgeProjects() {
const prProjects = await getPrProjects();
const projectsToPurge = [];
for (const project of prProjects) {
const NOW = new Date().getTime() / 1000;
const DAY_IN_SECONDS = 60 * 60 * 24;
const prJson = execSync(
`gh pr view '${project.prNumber}' --json state,labels,commits`
).toString();
const pullRequest = JSON.parse(prJson);
const prOpen = pullRequest.state === 'OPEN';
const lastCommit = pullRequest.commits.slice(-1)[0];
const lastCommitTimestamp = new Date(lastCommit.committedDate).getTime() / 1000;

const persistDeployment = Boolean(
pullRequest.labels.filter((label: any) => label.name === 'ci:project-persist-deployment')
.length
);
if (prOpen && persistDeployment) {
continue;
}

if (!prOpen) {
console.log(
`Pull Request #${project.prNumber} is no longer open, will delete associated ${project.type} project`
);
projectsToPurge.push(project);
} else if (
!Boolean(
pullRequest.labels.filter((label: any) =>
/^ci:project-deploy-(elasticearch|security|observability)$/.test(label.name)
).length
)
) {
console.log(
`Pull Request #${project.prNumber} no longer has a project deployment label, will delete associated deployment`
);
projectsToPurge.push(project);
} else if (lastCommitTimestamp < NOW - DAY_IN_SECONDS * 2) {
console.log(
`Pull Request #${project.prNumber} has not been updated in more than 2 days, will delete associated deployment`
);
projectsToPurge.push(project);
}
}
await Promise.all(projectsToPurge.map((p) => deleteProject(p)));
}

if (!process.env.PROJECT_API_DOMAIN || !process.env.PROJECT_API_KEY) {
console.error('missing project authentication');
process.exit(1);
}
const projectRequest = axios.create({
baseURL: process.env.PROJECT_API_DOMAIN,
headers: {
Authorization: `ApiKey ${process.env.PROJECT_API_KEY}`,
},
});

purgeProjects().catch((e) => {
console.error(e.toString());
process.exitCode = 1;
});
4 changes: 2 additions & 2 deletions .buildkite/scripts/steps/serverless/build_and_deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ source .buildkite/scripts/common/util.sh
source .buildkite/scripts/steps/artifacts/docker_image.sh

PROJECT_TYPE=""
is_pr_with_label "ci:project-deploy-es" && PROJECT_TYPE="elasticsearch"
is_pr_with_label "ci:project-deploy-oblt" && PROJECT_TYPE="observability"
is_pr_with_label "ci:project-deploy-elasticsearch" && PROJECT_TYPE="elasticsearch"
is_pr_with_label "ci:project-deploy-observability" && PROJECT_TYPE="observability"
is_pr_with_label "ci:project-deploy-security" && PROJECT_TYPE="security"
if [ -z "${PROJECT_TYPE}" ]; then
echo "Mising project type"
Expand Down
6 changes: 4 additions & 2 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -1312,8 +1312,10 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib
/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics @elastic/security-detection-engine
/x-pack/test/security_solution_cypress/cypress/e2e/exceptions @elastic/security-detection-engine
/x-pack/test/security_solution_cypress/cypress/e2e/overview @elastic/security-detection-engine
x-pack/test/security_solution_api_integration/test_suites/detections_response/exceptions @elastic/security-detection-engine
x-pack/test/security_solution_api_integration/test_suites/detections_response/rule_creation @elastic/security-detection-engine
x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions @elastic/security-detection-engine
x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation @elastic/security-detection-engine
x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions @elastic/security-detection-engine
x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts @elastic/security-detection-engine

## Security Threat Intelligence - Under Security Platform
/x-pack/plugins/security_solution/public/common/components/threat_match @elastic/security-detection-engine
Expand Down
2 changes: 1 addition & 1 deletion packages/kbn-management/cards_navigation/src/consts.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ export const appDefinitions: Record<AppId, AppDefinition> = {
[AppIds.SAVED_OBJECTS]: {
category: appCategories.CONTENT,
description: i18n.translate('management.landing.withCardNavigation.objectsDescription', {
defaultMessage: 'Manage your saved dashboards, maps, data views, and Canvas workpads.',
defaultMessage: 'Manage your saved dashboards, visualizations, maps, and data views.',
}),
icon: 'save',
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,15 +127,15 @@ export const IntegrationsPanel: React.FC<IntegrationsPanelProps> = ({
<EuiTitle size="s">
<h3>
{i18n.translate('searchApiPanels.welcomeBanner.ingestData.connectorsTitle', {
defaultMessage: 'Connector Client',
defaultMessage: 'Connector clients',
})}
</h3>
</EuiTitle>
<EuiSpacer size="s" />
<EuiText size="s">
{i18n.translate('searchApiPanels.welcomeBanner.ingestData.connectorsDescription', {
defaultMessage:
'Specialized integrations for syncing data from third-party sources to Elasticsearch. Use Elastic Connectors to sync content from a range of databases and object stores.',
'Specialized integrations for syncing data from third-party sources to Elasticsearch. Use Elastic connectors to sync content from a range of databases and object stores.',
})}
</EuiText>
<EuiSpacer size="m" />
Expand All @@ -153,7 +153,7 @@ export const IntegrationsPanel: React.FC<IntegrationsPanelProps> = ({
label={i18n.translate(
'searchApiPanels.welcomeBanner.ingestData.connectorsPythonLink',
{
defaultMessage: 'connectors-python',
defaultMessage: 'elastic/connectors',
}
)}
assetBasePath={assetBasePath}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ export const TryInConsoleButton = ({
<EuiButtonEmpty href={consolePreviewLink} iconType="popout" target="_blank" size="s">
<FormattedMessage
id="searchApiPanels.welcomeBanner.tryInConsoleButton"
defaultMessage="Try in console"
defaultMessage="Try in Console"
/>
</EuiButtonEmpty>
);
Expand Down
2 changes: 1 addition & 1 deletion packages/kbn-search-connectors/lib/start_sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ export const startConnectorSync = async (
error: null,
indexed_document_count: 0,
indexed_document_volume: 0,
job_type: jobType,
job_type: jobType || SyncJobType.FULL,
last_seen: null,
metadata: {},
started_at: null,
Expand Down
23 changes: 13 additions & 10 deletions packages/kbn-search-connectors/lib/update_connector_service_type.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,27 +9,30 @@
import { ElasticsearchClient } from '@kbn/core/server';
import { i18n } from '@kbn/i18n';

import { CONNECTORS_INDEX } from '..';
import { CONNECTORS_INDEX, fetchConnectorById } from '..';

import { ConnectorDocument } from '../types/connectors';
import { ConnectorDocument, ConnectorStatus } from '../types/connectors';

export const updateConnectorServiceType = async (
client: ElasticsearchClient,
connectorId: string,
serviceType: string
) => {
const connectorResult = await client.get<ConnectorDocument>({
id: connectorId,
index: CONNECTORS_INDEX,
});
const connector = connectorResult._source;
if (connector) {
const connectorResult = await fetchConnectorById(client, connectorId);

if (connectorResult?.value) {
const result = await client.index<ConnectorDocument>({
document: { ...connector, service_type: serviceType },
document: {
...connectorResult.value,
configuration: {},
service_type: serviceType,
status: ConnectorStatus.NEEDS_CONFIGURATION,
},
id: connectorId,
index: CONNECTORS_INDEX,
if_seq_no: connectorResult.seqNo,
if_primary_term: connectorResult.primaryTerm,
});
await client.indices.refresh({ index: CONNECTORS_INDEX });
return result;
} else {
throw new Error(
Expand Down
Loading

0 comments on commit 8f0eace

Please sign in to comment.