From 54b3df4db653a8ad7b80a432d8ba0c0b7644eece Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 12 May 2020 11:31:47 +0200 Subject: [PATCH 1/5] improved demo --- .../docker-entrypoint.elastic-agent.tmpl | 10 +++++- x-pack/elastic-agent/magefile.go | 31 +++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl index 457ff659745e..3b20b58407a9 100644 --- a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl @@ -30,11 +30,19 @@ function enroll(){ if [[ -n "${FLEET_ENROLLMENT_TOKEN}" ]] && [[ ${FLEET_ENROLLMENT_TOKEN} == 1 ]]; then apikey = "${FLEET_ENROLLMENT_TOKEN}" else + configsResp=$(curl -X GET ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/agent_configs \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true' \ + -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme}) + + configId=$(echo "$configsResp" | jq -r '.items | map(select(.name | contains("Default"))) | .[0].id') + default=${configId:-default} + enrollResp=$(curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/enrollment-api-keys \ -H 'Content-Type: application/json' \ -H 'kbn-xsrf: true' \ -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} \ - -d '{"name":"'"${FLEET_TOKEN_NAME:-demotoken}"'","config_id":"'"${FLEET_CONFIG_ID:-default}"'"}') + -d '{"name":"'"${FLEET_TOKEN_NAME:-demotoken}"'","config_id":"'"${FLEET_CONFIG_ID:-$default}"'"}') local exitCode=$? if [ $exitCode -ne 0 ]; then diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 8733b0a3d108..8e28472b051c 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "time" @@ -443,6 +444,13 @@ func (Demo) NoEnroll() error { } func runAgent(env map[string]string) error { + prevPlatforms := os.Getenv("PLATFORMS") + defer os.Setenv("PLATFORMS", prevPlatforms) + + // setting this improves build time + os.Setenv("PLATFORMS", "+all linux/amd64") + devtools.Platforms = devtools.NewPlatformList("+all linux/amd64") + supportedEnvs := map[string]int{"FLEET_ADMIN_PASSWORD": 0, "FLEET_ADMIN_USERNAME": 0, "FLEET_CONFIG_ID": 0, "FLEET_ENROLLMENT_TOKEN": 0, "FLEET_ENROLL": 0, "FLEET_SETUP": 0, "FLEET_TOKEN_NAME": 0, "KIBANA_HOST": 0, "KIBANA_PASSWORD": 0, "KIBANA_USERNAME": 0} tag := dockerTag() @@ -455,7 +463,6 @@ func runAgent(env map[string]string) error { if !strings.Contains(dockerImageOut, tag) { // produce docker package packageAgent([]string{ - "linux-x86.tar.gz", "linux-x86_64.tar.gz", }, devtools.UseElasticAgentDemoPackaging) @@ -471,7 +478,11 @@ func runAgent(env map[string]string) error { } // prepare env variables - var envs []string + envs := []string{ + // providing default kibana to be fixed for os-es if not provided + "KIBANA_HOST=http://localhost:5601", + } + envs = append(envs, os.Environ()...) for k, v := range env { envs = append(envs, fmt.Sprintf("%s=%s", k, v)) @@ -485,6 +496,9 @@ func runAgent(env map[string]string) error { continue } + // fix value + e = fmt.Sprintf("%s=%s", parts[0], fixOsEnv(parts[0], parts[1])) + dockerCmdArgs = append(dockerCmdArgs, "-e", e) } @@ -493,6 +507,7 @@ func runAgent(env map[string]string) error { } func packageAgent(requiredPackages []string, packagingFn func()) { + version, found := os.LookupEnv("BEAT_VERSION") if !found { version = release.Version() @@ -540,3 +555,15 @@ func dockerTag() string { return tagBase } + +func fixOsEnv(k, v string) string { + switch k { + case "KIBANA_HOST": + // network host works in a weird way here + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + return strings.Replace(strings.ToLower(v), "localhost", "host.docker.internal", 1) + } + } + + return v +} From ec4514da78ae0fe3b359f3c55b49976d0bbb87c6 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 12 May 2020 11:48:18 +0200 Subject: [PATCH 2/5] optimization 1 request less --- .../docker/docker-entrypoint.elastic-agent.tmpl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl index 3b20b58407a9..e5caff5ceb12 100644 --- a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl @@ -30,13 +30,15 @@ function enroll(){ if [[ -n "${FLEET_ENROLLMENT_TOKEN}" ]] && [[ ${FLEET_ENROLLMENT_TOKEN} == 1 ]]; then apikey = "${FLEET_ENROLLMENT_TOKEN}" else - configsResp=$(curl -X GET ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/agent_configs \ - -H 'Content-Type: application/json' \ - -H 'kbn-xsrf: true' \ - -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme}) - - configId=$(echo "$configsResp" | jq -r '.items | map(select(.name | contains("Default"))) | .[0].id') - default=${configId:-default} + if [[ -z "${FLEET_CONFIG_ID}" ]]; then + configsResp=$(curl -X GET ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/agent_configs \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true' \ + -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme}) + + configId=$(echo "$configsResp" | jq -r '.items | map(select(.name | contains("Default"))) | .[0].id') + default=${configId:-default} + fi enrollResp=$(curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/enrollment-api-keys \ -H 'Content-Type: application/json' \ From b26e9b704ee597d64952553e0b42ca0e6e18bab4 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 14 May 2020 08:14:14 +0200 Subject: [PATCH 3/5] redesigned scripts after latest fix --- .../docker/docker-entrypoint.elastic-agent.tmpl | 14 ++------------ go.sum | 1 + x-pack/elastic-agent/magefile.go | 2 +- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl index 6f5621eb7be5..ee43dfdd6e5f 100644 --- a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl @@ -5,7 +5,6 @@ set -eo pipefail # Environment variables used # FLEET_ADMIN_PASSWORD - used for new fleet user [elastic] # FLEET_ADMIN_USERNAME - used for new fleet user [changeme] -# FLEET_CONFIG_ID - config related to new token [defaul] # FLEET_ENROLLMENT_TOKEN - existing enrollment token to be used for enroll # FLEET_ENROLL - if set to 1 enroll will be performed # FLEET_SETUP - if set to 1 fleet setup will be performed @@ -30,16 +29,6 @@ function enroll(){ if [[ -n "${FLEET_ENROLLMENT_TOKEN}" ]] && [[ ${FLEET_ENROLLMENT_TOKEN} == 1 ]]; then apikey = "${FLEET_ENROLLMENT_TOKEN}" else - if [[ -z "${FLEET_CONFIG_ID}" ]]; then - configsResp=$(curl -X GET ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/agent_configs \ - -H 'Content-Type: application/json' \ - -H 'kbn-xsrf: true' \ - -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme}) - - configId=$(echo "$configsResp" | jq -r '.items | map(select(.name | contains("Default"))) | .[0].id') - default=${configId:-default} - fi - enrollResp=$(curl ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/enrollment-api-keys \ -H 'Content-Type: application/json' \ -H 'kbn-xsrf: true' \ @@ -62,8 +51,9 @@ function enroll(){ if [ $exitCode -ne 0 ]; then exit $exitCode fi + + apikey=$(echo $enrollResp | jq -r '.item.api_key') fi - apikey=$(echo $enrollResp | jq -r '.item.api_key') echo $apikey ./{{ .BeatName }} enroll ${KIBANA_HOST:-http://localhost:5601} $apikey -f diff --git a/go.sum b/go.sum index c669cc0a92e9..94de43bdedcd 100644 --- a/go.sum +++ b/go.sum @@ -221,6 +221,7 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/elastic/beats v7.6.2+incompatible h1:jHdLv83KURaqWUC6f55iMyVP6LYZrgElfeqxKWcskVE= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/ecs v1.5.0 h1:/VEIBsRU4ecq2+U3RPfKNc6bFyomP6qnthYEcQZu8GU= diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 8e28472b051c..5dbf8c18b691 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -451,7 +451,7 @@ func runAgent(env map[string]string) error { os.Setenv("PLATFORMS", "+all linux/amd64") devtools.Platforms = devtools.NewPlatformList("+all linux/amd64") - supportedEnvs := map[string]int{"FLEET_ADMIN_PASSWORD": 0, "FLEET_ADMIN_USERNAME": 0, "FLEET_CONFIG_ID": 0, "FLEET_ENROLLMENT_TOKEN": 0, "FLEET_ENROLL": 0, "FLEET_SETUP": 0, "FLEET_TOKEN_NAME": 0, "KIBANA_HOST": 0, "KIBANA_PASSWORD": 0, "KIBANA_USERNAME": 0} + supportedEnvs := map[string]int{"FLEET_ADMIN_PASSWORD": 0, "FLEET_ADMIN_USERNAME": 0, "FLEET_ENROLLMENT_TOKEN": 0, "FLEET_ENROLL": 0, "FLEET_SETUP": 0, "FLEET_TOKEN_NAME": 0, "KIBANA_HOST": 0, "KIBANA_PASSWORD": 0, "KIBANA_USERNAME": 0} tag := dockerTag() dockerImageOut, err := sh.Output("docker", "image", "ls") From ab891406a14aa0d4f6e7274fcdbb696db08dfcab Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 14 May 2020 08:15:09 +0200 Subject: [PATCH 4/5] tidy --- go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/go.sum b/go.sum index 94de43bdedcd..c669cc0a92e9 100644 --- a/go.sum +++ b/go.sum @@ -221,7 +221,6 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA= github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/elastic/beats v7.6.2+incompatible h1:jHdLv83KURaqWUC6f55iMyVP6LYZrgElfeqxKWcskVE= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/ecs v1.5.0 h1:/VEIBsRU4ecq2+U3RPfKNc6bFyomP6qnthYEcQZu8GU= From 138769e29aeb090014b88b993fc0f6deec6a1728 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 14 May 2020 08:16:47 +0200 Subject: [PATCH 5/5] newline --- x-pack/elastic-agent/magefile.go | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 5dbf8c18b691..8caaca1276c6 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -507,7 +507,6 @@ func runAgent(env map[string]string) error { } func packageAgent(requiredPackages []string, packagingFn func()) { - version, found := os.LookupEnv("BEAT_VERSION") if !found { version = release.Version()