From f13b32941a27a9bcf7cd3289572398ff2e9542c2 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 14 Apr 2021 16:53:36 +0100 Subject: [PATCH 1/5] packer-cache: for the 7.x and latest minor --- .ci/packer_cache.sh | 129 ++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index 86f3a51c3273..cda511951d2b 100644 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -9,75 +9,23 @@ source /usr/local/bin/bash_standard_lib.sh # shellcheck disable=SC1091 source ./dev-tools/common.bash -# Docker images used on Dockerfiles 2019-07-12 -# aerospike:3.9.0 -# alpine:edge -# apache/couchdb:1.7 -# busybox:latest -# ceph/daemon:master-6373c6a-jewel-centos-7-x86_64 -# cockroachdb/cockroach:v19.1.1 -# consul:1.4.2 -# coredns/coredns:1.5.0 -# couchbase:4.5.1 -# debian:latest -# debian:stretch -# docker.elastic.co/beats/metricbeat:6.5.4 -# docker.elastic.co/beats/metricbeat:7.2.0 -# envoyproxy/envoy:v1.7.0 -# exekias/localkube-image -# haproxy:1.8 -# httpd:2.4.20 -# java:8-jdk-alpine -# jplock/zookeeper:3.4.8 -# maven:3.3-jdk-8 -# memcached:1.4.35-alpine -# microsoft/mssql-server-linux:2017-GA -# mongo:3.4 -# mysql:5.7.12 -# nats:1.3.0 -# nginx:1.9 -# oraclelinux:7 -# postgres:9.5.3 -# prom/prometheus:v2.6.0 -# python:3.6-alpine -# quay.io/coreos/etcd:v3.3.10 -# rabbitmq:3.7.4-management -# redis:3.2.12-alpine -# redis:3.2.4-alpine -# store/oracle/database-enterprise:12.2.0.1 -# traefik:1.6-alpine -# tsouza/nginx-php-fpm:php-7.1 -# ubuntu:16.04 -# ubuntu:trusty +###################### +############ FUNCTIONS +###################### +function getBeatsVersion() { + grep 'defaultBeatVersion' libbeat/version/version.go | cut -d= -f2 | sed 's#"##g' | tr -d " " +} -get_go_version - -DOCKER_IMAGES="docker.elastic.co/observability-ci/database-instantclient:12.2.0.1 -docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 -docker.elastic.co/beats-dev/fpm:1.11.0 -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9 -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian7 -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian8 -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian9 -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-mips -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-ppc -docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-s390x -docker.elastic.co/elasticsearch/elasticsearch:8.0.0-SNAPSHOT -docker.elastic.co/kibana/kibana:8.0.0-SNAPSHOT -docker.elastic.co/logstash/logstash:8.0.0-SNAPSHOT -golang:${GO_VERSION} -golang:1.14.12-stretch -centos:7 -" -if [ -x "$(command -v docker)" ]; then - for image in ${DOCKER_IMAGES} - do +function dockerPullCommonImages() { + DOCKER_IMAGES="docker.elastic.co/observability-ci/database-instantclient:12.2.0.1 + docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 + docker.elastic.co/beats-dev/fpm:1.11.0 + golang:1.14.12-stretch + centos:7 + " + for image in ${DOCKER_IMAGES} ; do (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing." done - docker tag \ docker.elastic.co/observability-ci/database-instantclient:12.2.0.1 \ store/oracle/database-instantclient:12.2.0.1 \ @@ -86,4 +34,53 @@ if [ -x "$(command -v docker)" ]; then docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 \ store/oracle/database-enterprise:12.2.0.1 \ || echo "Error setting the Oracle Database tag" +} + +function dockerPullImages() { + SNAPSHOT=$1 + get_go_version + + DOCKER_IMAGES=" + docker.elastic.co/elasticsearch/elasticsearch:${SNAPSHOT} + docker.elastic.co/kibana/kibana:${SNAPSHOT} + docker.elastic.co/logstash/logstash:${SNAPSHOT} + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9 + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian7 + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian8 + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian9 + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-mips + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-ppc + docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-s390x + golang:${GO_VERSION} + " + for image in ${DOCKER_IMAGES} + do + (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing." + done +} + +################# +############ MAIN +################# +if [ -x "$(command -v docker)" ]; then + + echo "Docker pull common docker images" + dockerPullCommonImages + + ## GitHub api returns up to 100 entries. + ## Probably we need a different approach to search the latest minor. + latestMinor=$(curl -s https://api.github.com/repos/elastic/beats/branches\?per_page=100 | jq -r '.[].name' | grep "^7." | tail -1) + + for value in master 7.x $latestMinor ; do + if [ "$branch" != "master" ] ; then + echo "Checkout the branch $branch" + git checkout "$branch" + fi + + VERSION=$(getBeatsVersion) + dockerPullImages "${VERSION}-SNAPSHOT" + done fi From f1f5c27e2b1f8a91e4f098074da5467ded06ad4b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 14 Apr 2021 16:54:18 +0100 Subject: [PATCH 2/5] test the packer-cache --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 30479827d915..79084399fdbe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -64,6 +64,8 @@ pipeline { withEnv(["HOME=${env.WORKSPACE}"]) { retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } } + sh '.ci/packer_cache.sh' + sh 'exit 1' } } } From 3b6cff5ccccea234893d2c12bf8df41fe5846b33 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 14 Apr 2021 17:30:36 +0100 Subject: [PATCH 3/5] chmod +x --- .ci/packer_cache.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .ci/packer_cache.sh diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh old mode 100644 new mode 100755 From ad7971818d360296e3af14564a1b149c28ad55ab Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 14 Apr 2021 21:31:04 +0100 Subject: [PATCH 4/5] Apply suggestions from code review --- .ci/packer_cache.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index cda511951d2b..e4248118fae4 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -66,7 +66,7 @@ function dockerPullImages() { ############ MAIN ################# if [ -x "$(command -v docker)" ]; then - + set -x echo "Docker pull common docker images" dockerPullCommonImages @@ -74,7 +74,7 @@ if [ -x "$(command -v docker)" ]; then ## Probably we need a different approach to search the latest minor. latestMinor=$(curl -s https://api.github.com/repos/elastic/beats/branches\?per_page=100 | jq -r '.[].name' | grep "^7." | tail -1) - for value in master 7.x $latestMinor ; do + for branch in master 7.x $latestMinor ; do if [ "$branch" != "master" ] ; then echo "Checkout the branch $branch" git checkout "$branch" From fa3793709b4481246c6f25b4ec0c4003f5a854c0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 14 Apr 2021 22:06:39 +0100 Subject: [PATCH 5/5] Apply suggestions from code review --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 79084399fdbe..30479827d915 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -64,8 +64,6 @@ pipeline { withEnv(["HOME=${env.WORKSPACE}"]) { retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } } - sh '.ci/packer_cache.sh' - sh 'exit 1' } } }