From 308c65d055bcf7bb5ad07d9b62eaea976a51ce79 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Wed, 3 Jul 2024 17:44:21 +0200 Subject: [PATCH 01/19] Refactor --- .gitlab-ci.yml | 169 ++++++++++++++++----------- ci/scripts/ci-deploy-demo-staging.sh | 40 ++++--- ci/values-dd-ingest-staging.yaml | 4 - ci/values-ds-gateway.yaml | 4 - ci/values-ingest-staging.yaml | 4 - ci/values-staging.yaml | 4 - 6 files changed, 125 insertions(+), 100 deletions(-) delete mode 100644 ci/values-dd-ingest-staging.yaml delete mode 100644 ci/values-ds-gateway.yaml delete mode 100644 ci/values-ingest-staging.yaml delete mode 100644 ci/values-staging.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99326a7d9c..04d6fc557b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,14 +7,13 @@ variables: BUILD_DEMO_REGISTRY: 172597598159.dkr.ecr.us-east-1.amazonaws.com/otel-demo BUILD_SANDBOX_REGISTRY: 601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-demo stages: - - build + - build-push - prod-deploy - staging-deploy -# ======================================================================= -# Build and deploy the images used for CI -# ======================================================================= -.build-ci-image: &build-ci-image - stage: build + +# BUILD AND PUSH IMAGES +.build-push-ci-image: &build-push-ci-image + stage: build-push tags: ["runner:docker"] image: $CI_IMAGE script: @@ -27,98 +26,98 @@ stages: - if [[ $CI_COMMIT_REF_NAME == "prod" ]]; then docker push $BUILD_DEMO_REGISTRY:$TAG; fi -build-ci-image-accountingservice: - !!merge <<: *build-ci-image +build-push-ci-image-accountingservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/accountingservice/Dockerfile IMAGE_TAG_SUFFIX: accountingservice CONTEXT: . -build-ci-image-adservice: - !!merge <<: *build-ci-image +build-push-ci-image-adservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/adservice/Dockerfile IMAGE_TAG_SUFFIX: adservice CONTEXT: . -build-ci-image-cartservice: - !!merge <<: *build-ci-image +build-push-ci-image-cartservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/cartservice/src/Dockerfile IMAGE_TAG_SUFFIX: cartservice CONTEXT: . -build-ci-image-checkoutservice: - !!merge <<: *build-ci-image +build-push-ci-image-checkoutservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/checkoutservice/Dockerfile IMAGE_TAG_SUFFIX: checkoutservice CONTEXT: . -build-ci-image-currencyservice: - !!merge <<: *build-ci-image +build-push-ci-image-currencyservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/currencyservice/Dockerfile IMAGE_TAG_SUFFIX: currencyservice CONTEXT: src/currencyservice -build-ci-image-emailservice: - !!merge <<: *build-ci-image +build-push-ci-image-emailservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/emailservice/Dockerfile IMAGE_TAG_SUFFIX: emailservice CONTEXT: src/emailservice -build-ci-image-frauddetectionservice: - !!merge <<: *build-ci-image +build-push-ci-image-frauddetectionservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/frauddetectionservice/Dockerfile IMAGE_TAG_SUFFIX: frauddetectionservice CONTEXT: . -build-ci-image-frontend: - !!merge <<: *build-ci-image +build-push-ci-image-frontend: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/frontend/Dockerfile IMAGE_TAG_SUFFIX: frontend CONTEXT: . -build-ci-image-frontendproxy: - !!merge <<: *build-ci-image +build-push-ci-image-frontendproxy: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/frontendproxy/Dockerfile IMAGE_TAG_SUFFIX: frontendproxy CONTEXT: . -build-ci-image-imageprovider: - !!merge <<: *build-ci-image +build-push-ci-image-imageprovider: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/imageprovider/Dockerfile IMAGE_TAG_SUFFIX: imageprovider CONTEXT: . -build-ci-image-loadgenerator: - !!merge <<: *build-ci-image +build-push-ci-image-loadgenerator: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/loadgenerator/Dockerfile IMAGE_TAG_SUFFIX: loadgenerator CONTEXT: . -build-ci-image-paymentservice: - !!merge <<: *build-ci-image +build-push-ci-image-paymentservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/paymentservice/Dockerfile IMAGE_TAG_SUFFIX: paymentservice CONTEXT: . -build-ci-image-productcatalogservice: - !!merge <<: *build-ci-image +build-push-ci-image-productcatalogservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/productcatalogservice/Dockerfile IMAGE_TAG_SUFFIX: productcatalogservice CONTEXT: . -build-ci-image-quoteservice: - !!merge <<: *build-ci-image +build-push-ci-image-quoteservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/quoteservice/Dockerfile IMAGE_TAG_SUFFIX: quoteservice CONTEXT: . -build-ci-image-recommendationservice: - !!merge <<: *build-ci-image +build-push-ci-image-recommendationservice: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/recommendationservice/Dockerfile IMAGE_TAG_SUFFIX: recommendationservice CONTEXT: . -build-ci-image-shippingservice: - stage: build +build-push-ci-image-shippingservice: + stage: build-push tags: ["runner:docker"] image: $CI_IMAGE script: @@ -131,18 +130,20 @@ build-ci-image-shippingservice: DOCKERFILE: src/shippingservice/Dockerfile IMAGE_TAG_SUFFIX: shippingservice CONTEXT: . -build-ci-image-kafka: - !!merge <<: *build-ci-image +build-push-ci-image-kafka: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/kafka/Dockerfile IMAGE_TAG_SUFFIX: kafka CONTEXT: . -build-ci-image-orderproducer: - !!merge <<: *build-ci-image +build-push-ci-image-orderproducer: + !!merge <<: *build-push-ci-image variables: DOCKERFILE: src/orderproducerservice/Dockerfile IMAGE_TAG_SUFFIX: orderproducer CONTEXT: . + +# PROD DEPLOY DEMO / AGENT .prod-deploy: &prod-deploy stage: prod-deploy tags: ["runner:docker", "size:large"] @@ -161,6 +162,8 @@ build-ci-image-orderproducer: # # For debugging # - aws sts get-caller-identity - bash $SCRIPT $CLUSTER_NAME $CLUSTER_ARN $REGION $NAMESPACE + +# Demo env:otel prod-deploy-otel-demo-eks: !!merge <<: *prod-deploy variables: @@ -169,6 +172,11 @@ prod-deploy-otel-demo-eks: CLUSTER_ARN: arn:aws:eks:us-east-1:172597598159:cluster/prod-eks-otel-a-us-east-1 REGION: us-east-1 NAMESPACE: otel + ZOOKEEPER_DEPLOYMENT: deployment.yaml + ORDERPRODUCER_DEPLOYMENT: deployment.yaml + REGISTRY: $BUILD_DEMO_REGISTRY + +# Demo env:otel-ingest prod-deploy-otel-ingest-demo-eks: !!merge <<: *prod-deploy variables: @@ -177,6 +185,11 @@ prod-deploy-otel-ingest-demo-eks: CLUSTER_ARN: arn:aws:eks:us-east-1:172597598159:cluster/prod-eks-otel-ingest-a-usea-1 REGION: us-east-1 NAMESPACE: otel-ingest + ZOOKEEPER_DEPLOYMENT: deployment.yaml + ORDERPRODUCER_DEPLOYMENT: deployment.yaml + REGISTRY: $BUILD_DEMO_REGISTRY + +# Agent env:otel-ingest prod-deploy-otel-ingest-agent-eks: !!merge <<: *prod-deploy variables: @@ -188,6 +201,8 @@ prod-deploy-otel-ingest-agent-eks: VALUES: ./ci/datadog-agent-values-prod.yaml RELEASE_NAME: datadog-agent DD_VALUES: ./ci/datadog-agent-ddvalues-prod.yaml + +# STAGING DEPLOY DEMO / AGENT .staging-deploy: &staging-deploy stage: staging-deploy tags: ["runner:docker", "size:large"] @@ -212,34 +227,76 @@ staging-deploy-otel-demo-eks: !!merge <<: *staging-deploy variables: NAMESPACE: otel-staging - VALUES: ./ci/values-staging.yaml + VALUES: NODE_GROUP: ng-3 SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 + ZOOKEEPER_DEPLOYMENT: deployment-staging.yaml + ORDERPRODUCER_DEPLOYMENT: deployment-staging.yaml + REGISTRY: $BUILD_SANDBOX_REGISTRY + # Demo env:otel-ingest-staging staging-deploy-otel-ingest-demo-eks: !!merge <<: *staging-deploy variables: NAMESPACE: otel-ingest-staging - VALUES: ./ci/values-ingest-staging.yaml + VALUES: NODE_GROUP: ng-4 SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 + ZOOKEEPER_DEPLOYMENT: deployment-staging.yaml + ORDERPRODUCER_DEPLOYMENT: deployment-staging.yaml + REGISTRY: $BUILD_SANDBOX_REGISTRY + +# Demo env:otel-gateway +staging-deploy-gateway-demo-eks: + !!merge <<: *staging-deploy + variables: + NAMESPACE: otel-gateway + VALUES: ./ci/values-gateway.yaml + NODE_GROUP: ng-5 + SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh + CLUSTER_NAME: dd-otel + CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" + REGION: us-east-1 + ZOOKEEPER_DEPLOYMENT: deployment-staging.yaml + ORDERPRODUCER_DEPLOYMENT: deployment-staging.yaml + REGISTRY: $BUILD_SANDBOX_REGISTRY + +# Demo env:otel-ds-gateway +staging-deploy-ds-gateway-demo-eks: + !!merge <<: *staging-deploy + variables: + NAMESPACE: otel-ds-gateway + VALUES: + NODE_GROUP: ng-6 + SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh + CLUSTER_NAME: dd-otel + CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" + REGION: us-east-1 + ZOOKEEPER_DEPLOYMENT: deployment-staging.yaml + ORDERPRODUCER_DEPLOYMENT: deployment-staging.yaml + REGISTRY: $BUILD_SANDBOX_REGISTRY + # Demo env:dd-ingest-staging staging-deploy-dd-ingest-demo-eks: !!merge <<: *staging-deploy variables: NAMESPACE: dd-ingest-staging - VALUES: ./ci/values-dd-ingest-staging.yaml + VALUES: NODE_GROUP: ng-7 SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 + ZOOKEEPER_DEPLOYMENT: deployment-staging.yaml + ORDERPRODUCER_DEPLOYMENT: deployment-staging.yaml + REGISTRY: $BUILD_SANDBOX_REGISTRY + # Agent env:otel-ingest-staging staging-deploy-otel-ingest-agent-eks: !!merge <<: *staging-deploy @@ -264,25 +321,3 @@ staging-deploy-dd-ingest-agent-eks: VALUES: ./ci/datadog-agent-values-dd-staging.yaml DD_VALUES: ./ci/datadog-agent-values-dd.yaml RELEASE_NAME: datadog-agent-dd -# Demo env:otel-gateway -staging-deploy-gateway-demo-eks: - !!merge <<: *staging-deploy - variables: - NAMESPACE: otel-gateway - VALUES: ./ci/values-gateway.yaml - NODE_GROUP: ng-5 - SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh - CLUSTER_NAME: dd-otel - CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" - REGION: us-east-1 -# Demo env:otel-ds-gateway -staging-deploy-ds-gateway-demo-eks: - !!merge <<: *staging-deploy - variables: - NAMESPACE: otel-ds-gateway - VALUES: ./ci/values-ds-gateway.yaml - NODE_GROUP: ng-6 - SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh - CLUSTER_NAME: dd-otel - CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" - REGION: us-east-1 diff --git a/ci/scripts/ci-deploy-demo-staging.sh b/ci/scripts/ci-deploy-demo-staging.sh index 460173a2f4..ed0643870d 100644 --- a/ci/scripts/ci-deploy-demo-staging.sh +++ b/ci/scripts/ci-deploy-demo-staging.sh @@ -15,29 +15,35 @@ region=$REGION namespace=$NAMESPACE nodeGroup=$NODE_GROUP values=$VALUES +zookeeper_deployment=$ZOOKEEPER_DEPLOYMENT +orderproducer_deployment=$ORDERPRODUCER_DEPLOYMENT +registry=$REGISTRY install_demo() { - # Set the namespace and release name release_name="opentelemetry-demo" - # Deploy zookeeper which is not a default component. - sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/zookeeperservice/deployment-staging.yaml - kubectl apply -f ./src/zookeeperservice/deployment-staging.yaml -n "${namespace}" - - # if repo already exists, helm 3+ will skip - helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts - - # --install will run `helm install` if not already present. - helm --debug upgrade "${release_name}" -n "${namespace}" open-telemetry/opentelemetry-demo --install \ + # HELM COMMAND + helm_cmd= "helm --debug upgrade ${release_name} -n ${namespace} open-telemetry/opentelemetry-demo --install \ -f ./ci/values.yaml \ - -f $values \ --set-string default.image.tag="v$CI_COMMIT_SHORT_SHA" \ - --set-string default.image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-demo" - - # Deploy java order producer which is not a default component. - sed -i "s/PLACEHOLDER_COMMIT_SHA/v$CI_COMMIT_SHORT_SHA/g" ./src/orderproducerservice/deployment-staging.yaml - sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/orderproducerservice/deployment-staging.yaml - kubectl apply -f ./src/orderproducerservice/deployment-staging.yaml -n "${namespace}" + --set-string default.image.repository=${REGISTRY}" + + # REPLACEMENTS + if [ -n "$nodegroup" ]; then + sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/zookeeperservice/${zookeeper_deployment} + sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/orderproducerservice/${orderproducer_deployment} + helm_cmd+=" --set default.schedulingRules.nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodegroup}" + fi + if [ -n "$values" ]; then + helm_cmd+=" -f $values" + fi + sed -i "s/PLACEHOLDER_COMMIT_SHA/v$CI_COMMIT_SHORT_SHA/g" ./src/orderproducerservice/${orderproducer_deployment} + + # COMMANDS + kubectl apply -f ./src/zookeeperservice/${zookeeper_deployment} -n "${namespace}" + helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts + eval $helm_cmd + kubectl apply -f ./src/orderproducerservice/${orderproducer_deployment} -n "${namespace}" } ########################################################################################################### diff --git a/ci/values-dd-ingest-staging.yaml b/ci/values-dd-ingest-staging.yaml deleted file mode 100644 index 9fdfa9e911..0000000000 --- a/ci/values-dd-ingest-staging.yaml +++ /dev/null @@ -1,4 +0,0 @@ -default: - schedulingRules: - nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-7 diff --git a/ci/values-ds-gateway.yaml b/ci/values-ds-gateway.yaml deleted file mode 100644 index 08cf8df5fa..0000000000 --- a/ci/values-ds-gateway.yaml +++ /dev/null @@ -1,4 +0,0 @@ -default: - schedulingRules: - nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-6 diff --git a/ci/values-ingest-staging.yaml b/ci/values-ingest-staging.yaml deleted file mode 100644 index a0fb68ece0..0000000000 --- a/ci/values-ingest-staging.yaml +++ /dev/null @@ -1,4 +0,0 @@ -default: - schedulingRules: - nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-4 diff --git a/ci/values-staging.yaml b/ci/values-staging.yaml deleted file mode 100644 index dbafbd746e..0000000000 --- a/ci/values-staging.yaml +++ /dev/null @@ -1,4 +0,0 @@ -default: - schedulingRules: - nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-3 From 6d355220eb86684f46e588737b607aa39916c7d4 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Tue, 16 Jul 2024 16:46:40 +0200 Subject: [PATCH 02/19] remove space --- ci/scripts/ci-deploy-demo-staging.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/ci-deploy-demo-staging.sh b/ci/scripts/ci-deploy-demo-staging.sh index ed0643870d..36053f1949 100644 --- a/ci/scripts/ci-deploy-demo-staging.sh +++ b/ci/scripts/ci-deploy-demo-staging.sh @@ -23,7 +23,7 @@ install_demo() { release_name="opentelemetry-demo" # HELM COMMAND - helm_cmd= "helm --debug upgrade ${release_name} -n ${namespace} open-telemetry/opentelemetry-demo --install \ + helm_cmd="helm --debug upgrade ${release_name} -n ${namespace} open-telemetry/opentelemetry-demo --install \ -f ./ci/values.yaml \ --set-string default.image.tag="v$CI_COMMIT_SHORT_SHA" \ --set-string default.image.repository=${REGISTRY}" From bec439841009448ec0cdd2829dce493c20a3233b Mon Sep 17 00:00:00 2001 From: mackjmr Date: Tue, 16 Jul 2024 16:54:35 +0200 Subject: [PATCH 03/19] fix syntax --- ci/scripts/ci-deploy-demo-staging.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/ci-deploy-demo-staging.sh b/ci/scripts/ci-deploy-demo-staging.sh index 36053f1949..3782d5eb6b 100644 --- a/ci/scripts/ci-deploy-demo-staging.sh +++ b/ci/scripts/ci-deploy-demo-staging.sh @@ -29,10 +29,10 @@ install_demo() { --set-string default.image.repository=${REGISTRY}" # REPLACEMENTS - if [ -n "$nodegroup" ]; then + if [ -n "$nodeGroup" ]; then sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/zookeeperservice/${zookeeper_deployment} sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/orderproducerservice/${orderproducer_deployment} - helm_cmd+=" --set default.schedulingRules.nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodegroup}" + helm_cmd+=" --set default.schedulingRules.nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodeGroup}" fi if [ -n "$values" ]; then helm_cmd+=" -f $values" From 9a5d804824700631b828b2a1a1881356d4e02508 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Tue, 16 Jul 2024 17:03:45 +0200 Subject: [PATCH 04/19] remove redis --- ci/values.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci/values.yaml b/ci/values.yaml index 1f542cc0c7..3a156f9c5e 100644 --- a/ci/values.yaml +++ b/ci/values.yaml @@ -144,10 +144,6 @@ components: resources: limits: memory: 200Mi - redis: - resources: - limits: - memory: 200Mi loadgenerator: resources: limits: From 8878bb5c1ba45e849165daf2ee8677186004c700 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Tue, 16 Jul 2024 17:21:24 +0200 Subject: [PATCH 05/19] fixes --- ci/scripts/ci-deploy-demo-staging.sh | 1 - ci/values-gateway.yaml | 2 -- 2 files changed, 3 deletions(-) diff --git a/ci/scripts/ci-deploy-demo-staging.sh b/ci/scripts/ci-deploy-demo-staging.sh index 3782d5eb6b..2ec300971a 100644 --- a/ci/scripts/ci-deploy-demo-staging.sh +++ b/ci/scripts/ci-deploy-demo-staging.sh @@ -37,7 +37,6 @@ install_demo() { if [ -n "$values" ]; then helm_cmd+=" -f $values" fi - sed -i "s/PLACEHOLDER_COMMIT_SHA/v$CI_COMMIT_SHORT_SHA/g" ./src/orderproducerservice/${orderproducer_deployment} # COMMANDS kubectl apply -f ./src/zookeeperservice/${zookeeper_deployment} -n "${namespace}" diff --git a/ci/values-gateway.yaml b/ci/values-gateway.yaml index b6ff65989b..045c8b76e5 100644 --- a/ci/values-gateway.yaml +++ b/ci/values-gateway.yaml @@ -2,8 +2,6 @@ default: envOverrides: - name: OTEL_COLLECTOR_NAME value: opentelemetry-collector - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector:4317 schedulingRules: nodeSelector: alpha.eksctl.io/nodegroup-name: ng-5 \ No newline at end of file From ae515cabaceaeb1b9f4c171c90ec8b207d5d518b Mon Sep 17 00:00:00 2001 From: mackjmr Date: Wed, 17 Jul 2024 13:30:44 +0200 Subject: [PATCH 06/19] rmove env --- ci/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/values.yaml b/ci/values.yaml index 3a156f9c5e..228fc52618 100644 --- a/ci/values.yaml +++ b/ci/values.yaml @@ -39,8 +39,6 @@ default: value: $(HOST_IP) - name: OTEL_COLLECTOR_NAME value: $(HOST_IP) - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://$(OTEL_COLLECTOR_NAME):4317 - name: OTEL_RESOURCE_ATTRIBUTES value: >- service.name=$(OTEL_SERVICE_NAME), From 2da8460105f3a55fe9b9d835510c633c7945fcb9 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Wed, 17 Jul 2024 16:23:24 +0200 Subject: [PATCH 07/19] fix --- ci/values.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/values.yaml b/ci/values.yaml index 228fc52618..ce79cb6ec8 100644 --- a/ci/values.yaml +++ b/ci/values.yaml @@ -36,9 +36,13 @@ default: fieldRef: fieldPath: status.hostIP - name: DD_AGENT_HOST - value: $(HOST_IP) + valueFrom: + fieldRef: + fieldPath: status.hostIP - name: OTEL_COLLECTOR_NAME - value: $(HOST_IP) + valueFrom: + fieldRef: + fieldPath: status.hostIP - name: OTEL_RESOURCE_ATTRIBUTES value: >- service.name=$(OTEL_SERVICE_NAME), From b42b80d19c6a938b294532d0cda8db8d91f56e7f Mon Sep 17 00:00:00 2001 From: mackjmr Date: Wed, 17 Jul 2024 16:28:15 +0200 Subject: [PATCH 08/19] fix --- .gitlab-ci.yml | 10 ++++---- ci/scripts/ci-deploy-demo.sh | 49 ++++++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 04d6fc557b..8c4935c016 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -229,7 +229,7 @@ staging-deploy-otel-demo-eks: NAMESPACE: otel-staging VALUES: NODE_GROUP: ng-3 - SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh + SCRIPT: ./ci/scripts/ci-deploy-demo.sh CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 @@ -244,7 +244,7 @@ staging-deploy-otel-ingest-demo-eks: NAMESPACE: otel-ingest-staging VALUES: NODE_GROUP: ng-4 - SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh + SCRIPT: ./ci/scripts/ci-deploy-demo.sh CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 @@ -259,7 +259,7 @@ staging-deploy-gateway-demo-eks: NAMESPACE: otel-gateway VALUES: ./ci/values-gateway.yaml NODE_GROUP: ng-5 - SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh + SCRIPT: ./ci/scripts/ci-deploy-demo.sh CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 @@ -274,7 +274,7 @@ staging-deploy-ds-gateway-demo-eks: NAMESPACE: otel-ds-gateway VALUES: NODE_GROUP: ng-6 - SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh + SCRIPT: ./ci/scripts/ci-deploy-demo.sh CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 @@ -289,7 +289,7 @@ staging-deploy-dd-ingest-demo-eks: NAMESPACE: dd-ingest-staging VALUES: NODE_GROUP: ng-7 - SCRIPT: ./ci/scripts/ci-deploy-demo-staging.sh + SCRIPT: ./ci/scripts/ci-deploy-demo.sh CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 diff --git a/ci/scripts/ci-deploy-demo.sh b/ci/scripts/ci-deploy-demo.sh index 70351d5948..63e09ab328 100644 --- a/ci/scripts/ci-deploy-demo.sh +++ b/ci/scripts/ci-deploy-demo.sh @@ -9,29 +9,40 @@ set -euo pipefail IFS=$'\n\t' set -x -clusterName=$1 -clusterArn=$2 -region=$3 -namespace=$4 +clusterName=$CLUSTER_NAME +clusterArn=$CLUSTER_ARN +region=$REGION +namespace=$NAMESPACE +nodeGroup=$NODE_GROUP +values=$VALUES +zookeeper_deployment=$ZOOKEEPER_DEPLOYMENT +orderproducer_deployment=$ORDERPRODUCER_DEPLOYMENT +registry=$REGISTRY install_demo() { - # Set the namespace and release name release_name="opentelemetry-demo" - # Deploy zookeeper which is not a default component. - kubectl apply -f ./src/zookeeperservice/deployment.yaml -n "${namespace}" - - # if repo already exists, helm 3+ will skip - helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts - - # --install will run `helm install` if not already present. - helm --debug upgrade "${release_name}" -n "${namespace}" open-telemetry/opentelemetry-demo --install \ + # HELM COMMAND + helm_cmd="helm --debug upgrade ${release_name} -n ${namespace} open-telemetry/opentelemetry-demo --install \ -f ./ci/values.yaml \ - --set-string default.image.tag="v$CI_COMMIT_SHORT_SHA" - - # Deploy java order producer which is not a default component. - sed -i "s/PLACEHOLDER_COMMIT_SHA/v$CI_COMMIT_SHORT_SHA/g" ./src/orderproducerservice/deployment.yaml - kubectl apply -f ./src/orderproducerservice/deployment.yaml -n "${namespace}" + --set-string default.image.tag="v$CI_COMMIT_SHORT_SHA" \ + --set-string default.image.repository=${REGISTRY}" + + # REPLACEMENTS + if [ -n "$nodeGroup" ]; then + # sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/zookeeperservice/${zookeeper_deployment} + # sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/orderproducerservice/${orderproducer_deployment} + helm_cmd+=" --set default.schedulingRules.nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodeGroup}" + fi + if [ -n "$values" ]; then + helm_cmd+=" -f $values" + fi + + # COMMANDS + kubectl apply -f ./src/zookeeperservice/${zookeeper_deployment} -n "${namespace}" + helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts + eval $helm_cmd + kubectl apply -f ./src/orderproducerservice/${orderproducer_deployment} -n "${namespace}" } ########################################################################################################### @@ -39,4 +50,4 @@ install_demo() { aws eks --region "${region}" update-kubeconfig --name "${clusterName}" kubectl config use-context "${clusterArn}" -install_demo +install_demo \ No newline at end of file From 801cd9e9340cc4671200ca675ee8e8ebd2db28a8 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Wed, 17 Jul 2024 16:57:28 +0200 Subject: [PATCH 09/19] fixes --- .gitlab-ci.yml | 14 ++--- .../datadog-agent-ddvalues-prod.yaml | 0 .../datadog-agent-ddvalues-staging.yaml | 0 .../datadog-agent-values-dd-staging.yaml | 0 .../datadog-agent-values-dd.yaml | 0 .../datadog-agent-values-prod.yaml | 0 .../datadog-agent-values-staging.yaml | 0 ci/{ => demo-values}/values-gateway.yaml | 0 ci/{ => demo-values}/values.yaml | 0 ci/scripts/ci-deploy-demo-staging.sh | 53 ------------------- ci/scripts/ci-deploy-demo.sh | 6 +-- 11 files changed, 10 insertions(+), 63 deletions(-) rename ci/{ => agent-values}/datadog-agent-ddvalues-prod.yaml (100%) rename ci/{ => agent-values}/datadog-agent-ddvalues-staging.yaml (100%) rename ci/{ => agent-values}/datadog-agent-values-dd-staging.yaml (100%) rename ci/{ => agent-values}/datadog-agent-values-dd.yaml (100%) rename ci/{ => agent-values}/datadog-agent-values-prod.yaml (100%) rename ci/{ => agent-values}/datadog-agent-values-staging.yaml (100%) rename ci/{ => demo-values}/values-gateway.yaml (100%) rename ci/{ => demo-values}/values.yaml (100%) delete mode 100644 ci/scripts/ci-deploy-demo-staging.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8c4935c016..f1ea2697d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -198,9 +198,9 @@ prod-deploy-otel-ingest-agent-eks: CLUSTER_ARN: arn:aws:eks:us-east-1:172597598159:cluster/prod-eks-otel-ingest-a-usea-1 REGION: us-east-1 NAMESPACE: otel-ingest - VALUES: ./ci/datadog-agent-values-prod.yaml + VALUES: ./ci/agent-values/datadog-agent-values-prod.yaml RELEASE_NAME: datadog-agent - DD_VALUES: ./ci/datadog-agent-ddvalues-prod.yaml + DD_VALUES: ./ci/agent-values/datadog-agent-ddvalues-prod.yaml # STAGING DEPLOY DEMO / AGENT .staging-deploy: &staging-deploy @@ -257,7 +257,7 @@ staging-deploy-gateway-demo-eks: !!merge <<: *staging-deploy variables: NAMESPACE: otel-gateway - VALUES: ./ci/values-gateway.yaml + VALUES: ./ci/demo-values/values-gateway.yaml NODE_GROUP: ng-5 SCRIPT: ./ci/scripts/ci-deploy-demo.sh CLUSTER_NAME: dd-otel @@ -306,8 +306,8 @@ staging-deploy-otel-ingest-agent-eks: CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 - VALUES: ./ci/datadog-agent-values-staging.yaml - DD_VALUES: ./ci/datadog-agent-ddvalues-staging.yaml + VALUES: ./ci/agent-values/datadog-agent-values-staging.yaml + DD_VALUES: ./ci/agent-values/datadog-agent-ddvalues-staging.yaml RELEASE_NAME: datadog-agent # Agent env:dd-ingest-staging staging-deploy-dd-ingest-agent-eks: @@ -318,6 +318,6 @@ staging-deploy-dd-ingest-agent-eks: CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 - VALUES: ./ci/datadog-agent-values-dd-staging.yaml - DD_VALUES: ./ci/datadog-agent-values-dd.yaml + VALUES: ./ci/agent-values/datadog-agent-values-dd-staging.yaml + DD_VALUES: ./ci/agent-values/datadog-agent-values-dd.yaml RELEASE_NAME: datadog-agent-dd diff --git a/ci/datadog-agent-ddvalues-prod.yaml b/ci/agent-values/datadog-agent-ddvalues-prod.yaml similarity index 100% rename from ci/datadog-agent-ddvalues-prod.yaml rename to ci/agent-values/datadog-agent-ddvalues-prod.yaml diff --git a/ci/datadog-agent-ddvalues-staging.yaml b/ci/agent-values/datadog-agent-ddvalues-staging.yaml similarity index 100% rename from ci/datadog-agent-ddvalues-staging.yaml rename to ci/agent-values/datadog-agent-ddvalues-staging.yaml diff --git a/ci/datadog-agent-values-dd-staging.yaml b/ci/agent-values/datadog-agent-values-dd-staging.yaml similarity index 100% rename from ci/datadog-agent-values-dd-staging.yaml rename to ci/agent-values/datadog-agent-values-dd-staging.yaml diff --git a/ci/datadog-agent-values-dd.yaml b/ci/agent-values/datadog-agent-values-dd.yaml similarity index 100% rename from ci/datadog-agent-values-dd.yaml rename to ci/agent-values/datadog-agent-values-dd.yaml diff --git a/ci/datadog-agent-values-prod.yaml b/ci/agent-values/datadog-agent-values-prod.yaml similarity index 100% rename from ci/datadog-agent-values-prod.yaml rename to ci/agent-values/datadog-agent-values-prod.yaml diff --git a/ci/datadog-agent-values-staging.yaml b/ci/agent-values/datadog-agent-values-staging.yaml similarity index 100% rename from ci/datadog-agent-values-staging.yaml rename to ci/agent-values/datadog-agent-values-staging.yaml diff --git a/ci/values-gateway.yaml b/ci/demo-values/values-gateway.yaml similarity index 100% rename from ci/values-gateway.yaml rename to ci/demo-values/values-gateway.yaml diff --git a/ci/values.yaml b/ci/demo-values/values.yaml similarity index 100% rename from ci/values.yaml rename to ci/demo-values/values.yaml diff --git a/ci/scripts/ci-deploy-demo-staging.sh b/ci/scripts/ci-deploy-demo-staging.sh deleted file mode 100644 index 2ec300971a..0000000000 --- a/ci/scripts/ci-deploy-demo-staging.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# This script is used to deploy collector on demo account cluster - -set -euo pipefail -IFS=$'\n\t' -set -x - -clusterName=$CLUSTER_NAME -clusterArn=$CLUSTER_ARN -region=$REGION -namespace=$NAMESPACE -nodeGroup=$NODE_GROUP -values=$VALUES -zookeeper_deployment=$ZOOKEEPER_DEPLOYMENT -orderproducer_deployment=$ORDERPRODUCER_DEPLOYMENT -registry=$REGISTRY - -install_demo() { - release_name="opentelemetry-demo" - - # HELM COMMAND - helm_cmd="helm --debug upgrade ${release_name} -n ${namespace} open-telemetry/opentelemetry-demo --install \ - -f ./ci/values.yaml \ - --set-string default.image.tag="v$CI_COMMIT_SHORT_SHA" \ - --set-string default.image.repository=${REGISTRY}" - - # REPLACEMENTS - if [ -n "$nodeGroup" ]; then - sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/zookeeperservice/${zookeeper_deployment} - sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/orderproducerservice/${orderproducer_deployment} - helm_cmd+=" --set default.schedulingRules.nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodeGroup}" - fi - if [ -n "$values" ]; then - helm_cmd+=" -f $values" - fi - - # COMMANDS - kubectl apply -f ./src/zookeeperservice/${zookeeper_deployment} -n "${namespace}" - helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts - eval $helm_cmd - kubectl apply -f ./src/orderproducerservice/${orderproducer_deployment} -n "${namespace}" -} - -########################################################################################################### - -aws eks --region "${region}" update-kubeconfig --name "${clusterName}" -kubectl config use-context "${clusterArn}" - -install_demo \ No newline at end of file diff --git a/ci/scripts/ci-deploy-demo.sh b/ci/scripts/ci-deploy-demo.sh index 63e09ab328..dda4d6a6ce 100644 --- a/ci/scripts/ci-deploy-demo.sh +++ b/ci/scripts/ci-deploy-demo.sh @@ -24,14 +24,14 @@ install_demo() { # HELM COMMAND helm_cmd="helm --debug upgrade ${release_name} -n ${namespace} open-telemetry/opentelemetry-demo --install \ - -f ./ci/values.yaml \ + -f ./ci/demo-values/values.yaml \ --set-string default.image.tag="v$CI_COMMIT_SHORT_SHA" \ --set-string default.image.repository=${REGISTRY}" # REPLACEMENTS if [ -n "$nodeGroup" ]; then - # sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/zookeeperservice/${zookeeper_deployment} - # sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/orderproducerservice/${orderproducer_deployment} + sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/zookeeperservice/${zookeeper_deployment} + sed -i "s/PLACEHOLDER_NODE_GROUP/$nodeGroup/g" ./src/orderproducerservice/${orderproducer_deployment} helm_cmd+=" --set default.schedulingRules.nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodeGroup}" fi if [ -n "$values" ]; then From a2246171011c0c5bed8eda5fd2f9c9a3666ce23d Mon Sep 17 00:00:00 2001 From: mackjmr Date: Wed, 17 Jul 2024 17:10:22 +0200 Subject: [PATCH 10/19] simplify --- .gitlab-ci.yml | 9 +++------ ci/agent-values/datadog-agent-values-dd-staging.yaml | 6 ------ ci/agent-values/datadog-agent-values-prod.yaml | 3 --- ci/agent-values/datadog-agent-values-staging.yaml | 6 ------ ...adog-agent-values-dd.yaml => dd-staging-unified.yaml} | 4 ++++ ...t-ddvalues-staging.yaml => otel-staging-unified.yaml} | 5 +++++ ...atadog-agent-ddvalues-prod.yaml => prod-unified.yaml} | 2 ++ ci/scripts/ci-deploy-agent.sh | 2 -- 8 files changed, 14 insertions(+), 23 deletions(-) delete mode 100644 ci/agent-values/datadog-agent-values-dd-staging.yaml delete mode 100644 ci/agent-values/datadog-agent-values-prod.yaml delete mode 100644 ci/agent-values/datadog-agent-values-staging.yaml rename ci/agent-values/{datadog-agent-values-dd.yaml => dd-staging-unified.yaml} (76%) rename ci/agent-values/{datadog-agent-ddvalues-staging.yaml => otel-staging-unified.yaml} (82%) rename ci/agent-values/{datadog-agent-ddvalues-prod.yaml => prod-unified.yaml} (92%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f1ea2697d7..3ff36d0d4e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -198,9 +198,8 @@ prod-deploy-otel-ingest-agent-eks: CLUSTER_ARN: arn:aws:eks:us-east-1:172597598159:cluster/prod-eks-otel-ingest-a-usea-1 REGION: us-east-1 NAMESPACE: otel-ingest - VALUES: ./ci/agent-values/datadog-agent-values-prod.yaml + VALUES: ./ci/agent-values/prod-unified.yaml RELEASE_NAME: datadog-agent - DD_VALUES: ./ci/agent-values/datadog-agent-ddvalues-prod.yaml # STAGING DEPLOY DEMO / AGENT .staging-deploy: &staging-deploy @@ -306,8 +305,7 @@ staging-deploy-otel-ingest-agent-eks: CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 - VALUES: ./ci/agent-values/datadog-agent-values-staging.yaml - DD_VALUES: ./ci/agent-values/datadog-agent-ddvalues-staging.yaml + VALUES: ./ci/agent-values/otel-staging-unified.yaml RELEASE_NAME: datadog-agent # Agent env:dd-ingest-staging staging-deploy-dd-ingest-agent-eks: @@ -318,6 +316,5 @@ staging-deploy-dd-ingest-agent-eks: CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 - VALUES: ./ci/agent-values/datadog-agent-values-dd-staging.yaml - DD_VALUES: ./ci/agent-values/datadog-agent-values-dd.yaml + VALUES: ./ci/agent-values/dd-staging-unified.yaml RELEASE_NAME: datadog-agent-dd diff --git a/ci/agent-values/datadog-agent-values-dd-staging.yaml b/ci/agent-values/datadog-agent-values-dd-staging.yaml deleted file mode 100644 index 6facdbd064..0000000000 --- a/ci/agent-values/datadog-agent-values-dd-staging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -agents: - nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-7 -datadog: - tags: - - "env:dd-ingest-staging" diff --git a/ci/agent-values/datadog-agent-values-prod.yaml b/ci/agent-values/datadog-agent-values-prod.yaml deleted file mode 100644 index 735aad63a5..0000000000 --- a/ci/agent-values/datadog-agent-values-prod.yaml +++ /dev/null @@ -1,3 +0,0 @@ -datadog: - tags: - - "env:otel-ingest" diff --git a/ci/agent-values/datadog-agent-values-staging.yaml b/ci/agent-values/datadog-agent-values-staging.yaml deleted file mode 100644 index e6f2760a82..0000000000 --- a/ci/agent-values/datadog-agent-values-staging.yaml +++ /dev/null @@ -1,6 +0,0 @@ -agents: - nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-4 -datadog: - tags: - - "env:otel-ingest-staging" diff --git a/ci/agent-values/datadog-agent-values-dd.yaml b/ci/agent-values/dd-staging-unified.yaml similarity index 76% rename from ci/agent-values/datadog-agent-values-dd.yaml rename to ci/agent-values/dd-staging-unified.yaml index c62be9de9b..35ce955d9b 100644 --- a/ci/agent-values/datadog-agent-values-dd.yaml +++ b/ci/agent-values/dd-staging-unified.yaml @@ -1,8 +1,12 @@ agents: + nodeSelector: + alpha.eksctl.io/nodegroup-name: ng-7 image: tag: 7.51.0-rc.1 tagSuffix: jmx datadog: + tags: + - "env:dd-ingest-staging" apiKey: $DD_API_KEY apm: portEnabled: true diff --git a/ci/agent-values/datadog-agent-ddvalues-staging.yaml b/ci/agent-values/otel-staging-unified.yaml similarity index 82% rename from ci/agent-values/datadog-agent-ddvalues-staging.yaml rename to ci/agent-values/otel-staging-unified.yaml index c04a37faad..e5c26b01af 100644 --- a/ci/agent-values/datadog-agent-ddvalues-staging.yaml +++ b/ci/agent-values/otel-staging-unified.yaml @@ -1,9 +1,13 @@ agents: + nodeSelector: + alpha.eksctl.io/nodegroup-name: ng-4 image: repository: docker.io/datadog/agent-dev tag: stanley-liu-top-level-spans-poc-py3-jmx doNotCheckTag: true datadog: + tags: + - "env:otel-ingest-staging" apiKey: $DD_API_KEY logs: enabled: true @@ -21,3 +25,4 @@ datadog: name: datadog-secrets key: api-key optional: false + diff --git a/ci/agent-values/datadog-agent-ddvalues-prod.yaml b/ci/agent-values/prod-unified.yaml similarity index 92% rename from ci/agent-values/datadog-agent-ddvalues-prod.yaml rename to ci/agent-values/prod-unified.yaml index d2576f3dc0..c9b4ce6f20 100644 --- a/ci/agent-values/datadog-agent-ddvalues-prod.yaml +++ b/ci/agent-values/prod-unified.yaml @@ -3,6 +3,8 @@ agents: tag: 7.51.0-rc.1 tagSuffix: jmx datadog: + tags: + - "env:otel-ingest" apiKey: $DD_API_KEY logs: enabled: true diff --git a/ci/scripts/ci-deploy-agent.sh b/ci/scripts/ci-deploy-agent.sh index db41b943c9..99c985f24e 100644 --- a/ci/scripts/ci-deploy-agent.sh +++ b/ci/scripts/ci-deploy-agent.sh @@ -13,7 +13,6 @@ clusterArn=$CLUSTER_ARN region=$REGION namespace=$NAMESPACE releaseName=$RELEASE_NAME -ddValues=$DD_VALUES values=$VALUES install_agent() { @@ -22,7 +21,6 @@ install_agent() { # --install will run `helm install` if not already present. helm upgrade "${releaseName}" -n "${namespace}" datadog/datadog --install \ - -f "${ddValues}" \ -f "${values}" } From d330ae69336fd4ccd4298ce9080a9cdda0d2fec5 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Wed, 17 Jul 2024 17:19:20 +0200 Subject: [PATCH 11/19] simplify further --- .gitlab-ci.yml | 8 ++++-- ci/agent-values/dd-staging-unified.yaml | 22 --------------- ci/agent-values/otel-staging-unified.yaml | 28 ------------------- .../{prod-unified.yaml => values.yaml} | 2 -- ci/scripts/ci-deploy-agent.sh | 11 ++++++-- 5 files changed, 14 insertions(+), 57 deletions(-) delete mode 100644 ci/agent-values/dd-staging-unified.yaml delete mode 100644 ci/agent-values/otel-staging-unified.yaml rename ci/agent-values/{prod-unified.yaml => values.yaml} (92%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3ff36d0d4e..c9c4e85c51 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -198,8 +198,9 @@ prod-deploy-otel-ingest-agent-eks: CLUSTER_ARN: arn:aws:eks:us-east-1:172597598159:cluster/prod-eks-otel-ingest-a-usea-1 REGION: us-east-1 NAMESPACE: otel-ingest - VALUES: ./ci/agent-values/prod-unified.yaml RELEASE_NAME: datadog-agent + NODE_GROUP: + # STAGING DEPLOY DEMO / AGENT .staging-deploy: &staging-deploy @@ -305,8 +306,9 @@ staging-deploy-otel-ingest-agent-eks: CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 - VALUES: ./ci/agent-values/otel-staging-unified.yaml RELEASE_NAME: datadog-agent + NODE_GROUP: ng-4 + # Agent env:dd-ingest-staging staging-deploy-dd-ingest-agent-eks: !!merge <<: *staging-deploy @@ -316,5 +318,5 @@ staging-deploy-dd-ingest-agent-eks: CLUSTER_NAME: dd-otel CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel" REGION: us-east-1 - VALUES: ./ci/agent-values/dd-staging-unified.yaml RELEASE_NAME: datadog-agent-dd + NODE_GROUP: ng-7 diff --git a/ci/agent-values/dd-staging-unified.yaml b/ci/agent-values/dd-staging-unified.yaml deleted file mode 100644 index 35ce955d9b..0000000000 --- a/ci/agent-values/dd-staging-unified.yaml +++ /dev/null @@ -1,22 +0,0 @@ -agents: - nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-7 - image: - tag: 7.51.0-rc.1 - tagSuffix: jmx -datadog: - tags: - - "env:dd-ingest-staging" - apiKey: $DD_API_KEY - apm: - portEnabled: true - port: 8126 - logs: - enabled: true - env: - - name: DD_API_KEY - valueFrom: - secretKeyRef: - name: datadog-secrets - key: api-key - optional: false diff --git a/ci/agent-values/otel-staging-unified.yaml b/ci/agent-values/otel-staging-unified.yaml deleted file mode 100644 index e5c26b01af..0000000000 --- a/ci/agent-values/otel-staging-unified.yaml +++ /dev/null @@ -1,28 +0,0 @@ -agents: - nodeSelector: - alpha.eksctl.io/nodegroup-name: ng-4 - image: - repository: docker.io/datadog/agent-dev - tag: stanley-liu-top-level-spans-poc-py3-jmx - doNotCheckTag: true -datadog: - tags: - - "env:otel-ingest-staging" - apiKey: $DD_API_KEY - logs: - enabled: true - otlp: - receiver: - protocols: - grpc: - enabled: true - http: - enabled: true - env: - - name: DD_API_KEY - valueFrom: - secretKeyRef: - name: datadog-secrets - key: api-key - optional: false - diff --git a/ci/agent-values/prod-unified.yaml b/ci/agent-values/values.yaml similarity index 92% rename from ci/agent-values/prod-unified.yaml rename to ci/agent-values/values.yaml index c9b4ce6f20..d2576f3dc0 100644 --- a/ci/agent-values/prod-unified.yaml +++ b/ci/agent-values/values.yaml @@ -3,8 +3,6 @@ agents: tag: 7.51.0-rc.1 tagSuffix: jmx datadog: - tags: - - "env:otel-ingest" apiKey: $DD_API_KEY logs: enabled: true diff --git a/ci/scripts/ci-deploy-agent.sh b/ci/scripts/ci-deploy-agent.sh index 99c985f24e..8961a5d85b 100644 --- a/ci/scripts/ci-deploy-agent.sh +++ b/ci/scripts/ci-deploy-agent.sh @@ -14,14 +14,21 @@ region=$REGION namespace=$NAMESPACE releaseName=$RELEASE_NAME values=$VALUES +nodegroup=$NODE_GROUP install_agent() { # if repo already exists, helm 3+ will skip helm repo add datadog https://helm.datadoghq.com # --install will run `helm install` if not already present. - helm upgrade "${releaseName}" -n "${namespace}" datadog/datadog --install \ - -f "${values}" + helm_cmd=helm upgrade "${releaseName}" -n "${namespace}" datadog/datadog --install \ + -f ./ci/agent-values/values.yaml \ + --set datadog.tags=env:"${namespace}" \ + + if [ -n "$nodegroup" ]; then + helm_cmd+=" --set agents.nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodegroup}" + fi + } ########################################################################################################### From 1e6fcbe06bb913b3c37c2f5aa77d8a77425f9f3b Mon Sep 17 00:00:00 2001 From: mackjmr Date: Wed, 17 Jul 2024 17:25:04 +0200 Subject: [PATCH 12/19] fix unbound var --- ci/scripts/ci-deploy-agent.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/scripts/ci-deploy-agent.sh b/ci/scripts/ci-deploy-agent.sh index 8961a5d85b..c02a818188 100644 --- a/ci/scripts/ci-deploy-agent.sh +++ b/ci/scripts/ci-deploy-agent.sh @@ -13,7 +13,6 @@ clusterArn=$CLUSTER_ARN region=$REGION namespace=$NAMESPACE releaseName=$RELEASE_NAME -values=$VALUES nodegroup=$NODE_GROUP install_agent() { From ff6f463a8447c5d31efdf71f388613664ffe3c7a Mon Sep 17 00:00:00 2001 From: mackjmr Date: Wed, 17 Jul 2024 17:31:29 +0200 Subject: [PATCH 13/19] fix --- ci/scripts/ci-deploy-agent.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/ci-deploy-agent.sh b/ci/scripts/ci-deploy-agent.sh index c02a818188..70e1e89dae 100644 --- a/ci/scripts/ci-deploy-agent.sh +++ b/ci/scripts/ci-deploy-agent.sh @@ -20,9 +20,9 @@ install_agent() { helm repo add datadog https://helm.datadoghq.com # --install will run `helm install` if not already present. - helm_cmd=helm upgrade "${releaseName}" -n "${namespace}" datadog/datadog --install \ + helm_cmd="helm upgrade "${releaseName}" -n "${namespace}" datadog/datadog --install \ -f ./ci/agent-values/values.yaml \ - --set datadog.tags=env:"${namespace}" \ + --set datadog.tags=env:"${namespace}"" if [ -n "$nodegroup" ]; then helm_cmd+=" --set agents.nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodegroup}" From 180a496c6b92aa458d387bc9f398dbf8666fba01 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Wed, 17 Jul 2024 17:37:38 +0200 Subject: [PATCH 14/19] eval --- ci/agent-values/values.yaml | 2 +- ci/scripts/ci-deploy-agent.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/agent-values/values.yaml b/ci/agent-values/values.yaml index d2576f3dc0..e85f74fad3 100644 --- a/ci/agent-values/values.yaml +++ b/ci/agent-values/values.yaml @@ -1,6 +1,6 @@ agents: image: - tag: 7.51.0-rc.1 + tag: 7.55.0 tagSuffix: jmx datadog: apiKey: $DD_API_KEY diff --git a/ci/scripts/ci-deploy-agent.sh b/ci/scripts/ci-deploy-agent.sh index 70e1e89dae..07ec7622d1 100644 --- a/ci/scripts/ci-deploy-agent.sh +++ b/ci/scripts/ci-deploy-agent.sh @@ -20,7 +20,7 @@ install_agent() { helm repo add datadog https://helm.datadoghq.com # --install will run `helm install` if not already present. - helm_cmd="helm upgrade "${releaseName}" -n "${namespace}" datadog/datadog --install \ + helm_cmd="helm --debug upgrade "${releaseName}" -n "${namespace}" datadog/datadog --install \ -f ./ci/agent-values/values.yaml \ --set datadog.tags=env:"${namespace}"" @@ -28,6 +28,7 @@ install_agent() { helm_cmd+=" --set agents.nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodegroup}" fi + eval $helm_cmd } ########################################################################################################### From 7fb7b56c0f14b9507ffb6bbacec30ce72e46d8b3 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Thu, 18 Jul 2024 13:43:21 +0200 Subject: [PATCH 15/19] remove duplicate env --- ci/agent-values/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/agent-values/values.yaml b/ci/agent-values/values.yaml index e85f74fad3..4c63bf1ede 100644 --- a/ci/agent-values/values.yaml +++ b/ci/agent-values/values.yaml @@ -3,7 +3,6 @@ agents: tag: 7.55.0 tagSuffix: jmx datadog: - apiKey: $DD_API_KEY logs: enabled: true otlp: From f2b6cd3434e155321d0b28b6802959367d76200a Mon Sep 17 00:00:00 2001 From: mackjmr Date: Thu, 18 Jul 2024 14:16:58 +0200 Subject: [PATCH 16/19] fix --- ci/demo-values/values.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ci/demo-values/values.yaml b/ci/demo-values/values.yaml index ce79cb6ec8..228fc52618 100644 --- a/ci/demo-values/values.yaml +++ b/ci/demo-values/values.yaml @@ -36,13 +36,9 @@ default: fieldRef: fieldPath: status.hostIP - name: DD_AGENT_HOST - valueFrom: - fieldRef: - fieldPath: status.hostIP + value: $(HOST_IP) - name: OTEL_COLLECTOR_NAME - valueFrom: - fieldRef: - fieldPath: status.hostIP + value: $(HOST_IP) - name: OTEL_RESOURCE_ATTRIBUTES value: >- service.name=$(OTEL_SERVICE_NAME), From c85a0073bea0aa8ceb410cf287832792915f9838 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Thu, 18 Jul 2024 14:46:07 +0200 Subject: [PATCH 17/19] uninstall --- ci/scripts/ci-deploy-demo.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/scripts/ci-deploy-demo.sh b/ci/scripts/ci-deploy-demo.sh index dda4d6a6ce..82a3f2b1bb 100644 --- a/ci/scripts/ci-deploy-demo.sh +++ b/ci/scripts/ci-deploy-demo.sh @@ -22,6 +22,8 @@ registry=$REGISTRY install_demo() { release_name="opentelemetry-demo" + helm uninstall ${release_name} -n ${namespace} + # HELM COMMAND helm_cmd="helm --debug upgrade ${release_name} -n ${namespace} open-telemetry/opentelemetry-demo --install \ -f ./ci/demo-values/values.yaml \ From a7be2a856c297759ddda04222a71c1da0a33dcb8 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Thu, 18 Jul 2024 14:50:13 +0200 Subject: [PATCH 18/19] fix api --- ci/agent-values/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/agent-values/values.yaml b/ci/agent-values/values.yaml index 4c63bf1ede..8d1193efc3 100644 --- a/ci/agent-values/values.yaml +++ b/ci/agent-values/values.yaml @@ -3,6 +3,7 @@ agents: tag: 7.55.0 tagSuffix: jmx datadog: + apiKey: $KEY logs: enabled: true otlp: @@ -13,7 +14,7 @@ datadog: http: enabled: true env: - - name: DD_API_KEY + - name: KEY valueFrom: secretKeyRef: name: datadog-secrets From 43644ae54d701ebcbaf94fe809a98eb8ec2a738c Mon Sep 17 00:00:00 2001 From: mackjmr Date: Thu, 18 Jul 2024 15:46:41 +0200 Subject: [PATCH 19/19] fix --- ci/agent-values/values.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ci/agent-values/values.yaml b/ci/agent-values/values.yaml index 8d1193efc3..bfb14a79e7 100644 --- a/ci/agent-values/values.yaml +++ b/ci/agent-values/values.yaml @@ -3,7 +3,7 @@ agents: tag: 7.55.0 tagSuffix: jmx datadog: - apiKey: $KEY + apiKeyExistingSecret: datadog-secrets logs: enabled: true otlp: @@ -13,10 +13,3 @@ datadog: enabled: true http: enabled: true - env: - - name: KEY - valueFrom: - secretKeyRef: - name: datadog-secrets - key: api-key - optional: false