Skip to content

Commit

Permalink
Merge branch 'main' into issue_130199
Browse files Browse the repository at this point in the history
  • Loading branch information
kibanamachine authored Apr 15, 2022
2 parents c512571 + fc7d385 commit 4744a39
Show file tree
Hide file tree
Showing 735 changed files with 54,121 additions and 9,003 deletions.
55 changes: 47 additions & 8 deletions .buildkite/pipelines/artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,29 +4,68 @@ steps:
agents:
queue: c2-16
timeout_in_minutes: 60
retry:
automatic:
- exit_status: '*'
limit: 1

- wait

- command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
label: Package testing for deb
label: Artifact Testing
agents:
queue: n2-4-virt
timeout_in_minutes: 20
timeout_in_minutes: 30
retry:
automatic:
- exit_status: '*'
limit: 1

- command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
label: Package testing for rpm
label: Artifact Testing
agents:
queue: n2-4-virt
timeout_in_minutes: 20
timeout_in_minutes: 30
retry:
automatic:
- exit_status: '*'
limit: 1

- command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
label: Package testing for docker
label: Artifact Testing
agents:
queue: n2-4-virt
timeout_in_minutes: 20
timeout_in_minutes: 30
retry:
automatic:
- exit_status: '*'
limit: 1

- command: .buildkite/scripts/steps/artifacts/docker_context.sh
label: 'Docker Build Context'
label: 'Docker Context Verification'
agents:
queue: n2-2
timeout_in_minutes: 20
timeout_in_minutes: 30
retry:
automatic:
- exit_status: '*'
limit: 1

- command: .buildkite/scripts/steps/artifacts/cloud.sh
label: 'Cloud Deployment'
agents:
queue: n2-2
timeout_in_minutes: 30
if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
retry:
automatic:
- exit_status: '*'
limit: 1

- wait

- command: .buildkite/scripts/steps/artifacts/publish.sh
label: 'Publish Kibana Artifacts'
agents:
queue: n2-2
timeout_in_minutes: 30
14 changes: 1 addition & 13 deletions .buildkite/pipelines/es_snapshots/verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ steps:

- command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
label: 'Default CI Group'
parallelism: 27
parallelism: 29
agents:
queue: n2-4
depends_on: build
Expand All @@ -40,18 +40,6 @@ steps:
- exit_status: '*'
limit: 1

- command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
label: 'Docker CI Group'
agents:
queue: n2-4
depends_on: build
timeout_in_minutes: 120
key: default-cigroup-docker
retry:
automatic:
- exit_status: '*'
limit: 1

- command: .buildkite/scripts/steps/functional/oss_cigroup.sh
label: 'OSS CI Group'
parallelism: 11
Expand Down
8 changes: 2 additions & 6 deletions .buildkite/pipelines/flaky_tests/groups.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,7 @@
{
"key": "xpack/cigroup",
"name": "Default CI Group",
"ciGroups": 27
},
{
"key": "xpack/cigroup/Docker",
"name": "Default CI Group Docker"
"ciGroups": 29
},
{
"key": "xpack/firefox",
Expand All @@ -47,4 +43,4 @@
"name": "Default Accessibility"
}
]
}
}
16 changes: 1 addition & 15 deletions .buildkite/pipelines/on_merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ steps:

- command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
label: 'Default CI Group'
parallelism: 27
parallelism: 29
agents:
queue: n2-4-spot
depends_on: build
Expand All @@ -64,20 +64,6 @@ steps:
- exit_status: '*'
limit: 1

- command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
label: 'Docker CI Group'
agents:
queue: n2-4-spot
depends_on: build
timeout_in_minutes: 120
key: default-cigroup-docker
retry:
automatic:
- exit_status: '-1'
limit: 3
- exit_status: '*'
limit: 1

- command: .buildkite/scripts/steps/functional/oss_cigroup.sh
label: 'OSS CI Group'
parallelism: 11
Expand Down
14 changes: 1 addition & 13 deletions .buildkite/pipelines/pull_request/base.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ steps:

- command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
label: 'Default CI Group'
parallelism: 27
parallelism: 29
agents:
queue: n2-4
depends_on: build
Expand All @@ -28,18 +28,6 @@ steps:
- exit_status: '*'
limit: 1

- command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
label: 'Docker CI Group'
agents:
queue: n2-4
depends_on: build
timeout_in_minutes: 120
key: default-cigroup-docker
retry:
automatic:
- exit_status: '*'
limit: 1

- command: .buildkite/scripts/steps/functional/oss_cigroup.sh
label: 'OSS CI Group'
parallelism: 11
Expand Down
11 changes: 11 additions & 0 deletions .buildkite/pipelines/pull_request/response_ops.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
steps:
- command: .buildkite/scripts/steps/functional/response_ops_cases.sh
label: 'Cases Cypress Tests on Security Solution'
agents:
queue: ci-group-6
depends_on: build
timeout_in_minutes: 120
retry:
automatic:
- exit_status: '*'
limit: 1
8 changes: 8 additions & 0 deletions .buildkite/scripts/lifecycle/pre_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,11 @@ export CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-iss
export CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"

node "$(dirname "${0}")/ci_stats_start.js"

# We resolve the latest manifest URL at the beginning of the build to ensure that all steps in the build will use the same manifest
# Otherwise, the manifest could change if a step is running around the time that a new one is promoted
if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
BUCKET=$(curl -s "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(cat package.json | jq -r .version)/manifest-latest-verified.json" | jq -r .bucket)
ES_SNAPSHOT_MANIFEST_DEFAULT="https://storage.googleapis.com/$BUCKET/manifest.json"
buildkite-agent meta-data set ES_SNAPSHOT_MANIFEST_DEFAULT "$ES_SNAPSHOT_MANIFEST_DEFAULT"
fi
9 changes: 9 additions & 0 deletions .buildkite/scripts/lifecycle/pre_command.sh
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,15 @@ EOF
fi
}

# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build
{
if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')}
export ES_SNAPSHOT_MANIFEST
echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST"
fi
}

# Setup CI Stats
{
CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"
Expand Down
8 changes: 7 additions & 1 deletion .buildkite/scripts/pipelines/pull_request/pipeline.js
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ const uploadPipeline = (pipelineContent) => {
if (
(await doAnyChangesMatch([
/^x-pack\/plugins\/security_solution/,
/^x-pack\/plugins\/cases/,
/^x-pack\/plugins\/lists/,
/^x-pack\/plugins\/timelines/,
/^x-pack\/test\/security_solution_cypress/,
Expand All @@ -77,6 +76,13 @@ const uploadPipeline = (pipelineContent) => {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution.yml'));
}

if (
(await doAnyChangesMatch([/^x-pack\/plugins\/cases/])) ||
process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
) {
pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops.yml'));
}

if (
(await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
Expand Down
51 changes: 15 additions & 36 deletions .buildkite/scripts/steps/artifacts/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,44 +4,23 @@ set -euo pipefail

.buildkite/scripts/bootstrap.sh

if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
VERSION="$(jq -r '.version' package.json)"
RELEASE_ARG="--release"
else
VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
RELEASE_ARG=""
fi
source .buildkite/scripts/steps/artifacts/env.sh

echo "--- Build Kibana Distribution"
node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile --skip-docker-cloud
echo "--- Build Kibana artifacts"
node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS")

echo "--- Build dependencies report"
node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv"

# Release verification
if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
echo "--- Build and push Kibana Cloud Distribution"
# This doesn't meet the requirements for a release image, implementation TBD
# Beats artifacts will need to match a specific commit sha that matches other stack iamges
# For now this is a placeholder step that will allow us to run automated Cloud tests
# against a best guess approximation of a release image
echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
trap 'docker logout docker.elastic.co' EXIT
echo "--- Extract default i18n messages"
mkdir -p target/i18n
node scripts/i18n_extract --output-dir=target/i18n

node scripts/build \
"$RELEASE_ARG" \
--skip-initialize \
--skip-generic-folders \
--skip-platform-folders \
--skip-archives \
--docker-images \
--docker-tag-qualifier="$GIT_COMMIT" \
--docker-push \
--skip-docker-ubi \
--skip-docker-ubuntu \
--skip-docker-contexts
fi
echo "--- Build dependencies report"
node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv"
(cd target; sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt")

echo "--- Upload Kibana Artifacts"
cd target
buildkite-agent artifact upload "*"
cd -
buildkite-agent artifact upload 'kibana-*'
buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv"
buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv.sha512.txt"
buildkite-agent artifact upload 'i18n/*.json'
cd -
81 changes: 81 additions & 0 deletions .buildkite/scripts/steps/artifacts/cloud.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#!/bin/bash

set -euo pipefail

.buildkite/scripts/bootstrap.sh

source .buildkite/scripts/steps/artifacts/env.sh

echo "--- Publish Cloud image"
mkdir -p target
cd target

buildkite-agent artifact download "kibana-cloud-$FULL_VERSION-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
docker load --input kibana-cloud-$FULL_VERSION-docker-image.tar.gz

TAG="$FULL_VERSION-$GIT_COMMIT"
KIBANA_BASE_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$FULL_VERSION"
KIBANA_TEST_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$TAG"

docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE"

echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
trap 'docker logout docker.elastic.co' EXIT

docker push "$KIBANA_TEST_IMAGE"
docker logout docker.elastic.co

cd -

echo "--- Create deployment"
CLOUD_DEPLOYMENT_NAME="kibana-artifacts-$TAG"

LOGS=$(mktemp --suffix ".json")
DEPLOYMENT_SPEC=$(mktemp --suffix ".json")

jq '
.name = "'$CLOUD_DEPLOYMENT_NAME'" |
.resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_TEST_IMAGE'" |
.resources.kibana[0].plan.kibana.version = "'$FULL_VERSION'" |
.resources.elasticsearch[0].plan.elasticsearch.version = "'$FULL_VERSION'" |
.resources.enterprise_search[0].plan.enterprise_search.version = "'$FULL_VERSION'" |
.resources.integrations_server[0].plan.integrations_server.version = "'$FULL_VERSION'"
' .buildkite/scripts/steps/cloud/deploy.json > "$DEPLOYMENT_SPEC"

ecctl deployment create --track --output json --file "$DEPLOYMENT_SPEC" &> "$LOGS"
CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS")
CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS")
CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS")
CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$LOGS")

CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')

# NOTE: disabled pending log sanitization
# echo "--- Setup FTR"
# export TEST_KIBANA_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol)")
# export TEST_KIBANA_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
# export TEST_KIBANA_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
# export TEST_KIBANA_USERNAME=$CLOUD_DEPLOYMENT_USERNAME"
# export TEST_KIBANA_PASS=$CLOUD_DEPLOYMENT_PASSWORD"

# export TEST_ES_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol)")
# export TEST_ES_HOSTNAME==$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
# export TEST_ES_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
# export TEST_ES_USER="$CLOUD_DEPLOYMENT_USERNAME"
# export TEST_ES_PASS="$CLOUD_DEPLOYMENT_PASSWORD"

# export TEST_BROWSER_HEADLESS=1

# Error: attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ConnectionError: self signed certificate in certificate chain
# export NODE_TLS_REJECT_UNAUTHORIZED=0

# echo "--- Run default functional tests"
# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud

# echo "--- Run x-pack functional tests"
# cd x-pack
# node --no-warnings scripts/functional_test_runner.js --include-tag=cloud -exclude-tag=skipCloud

echo "--- Shutdown deployment"
ecctl deployment shutdown "$CLOUD_DEPLOYMENT_ID" --force --track --output json &> "$LOGS"
Loading

0 comments on commit 4744a39

Please sign in to comment.