Merge branch 'main' into mackjmr/add-profiling-endpoints-config

.gitlab/container_build/docker_linux.yml

@@ -226,6 +226,34 @@ docker_build_ot_agent7_jmx_arm64:
     TAG_SUFFIX: -7-ot-beta-jmx
     BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-*-7*-arm64.tar.xz
 
+docker_build_agent7_full:
+  extends: [.docker_build_job_definition_amd64, .docker_build_artifact]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  needs:
+    - job: datadog-agent-7-x64
+    - job: datadog-otel-agent-x64
+  variables:
+    IMAGE: registry.ddbuild.io/ci/datadog-agent/agent
+    BUILD_CONTEXT: Dockerfiles/agent
+    TAG_SUFFIX: -7-full
+    BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-*-7*-amd64.tar.xz
+
+docker_build_agent7_full_arm64:
+  extends: [.docker_build_job_definition_arm64, .docker_build_artifact]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  needs:
+    - job: datadog-agent-7-arm64
+    - job: datadog-otel-agent-arm64
+  variables:
+    IMAGE: registry.ddbuild.io/ci/datadog-agent/agent
+    BUILD_CONTEXT: Dockerfiles/agent
+    TAG_SUFFIX: -7-full
+    BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-*-7*-arm64.tar.xz
+
 # build the cluster-agent image
 docker_build_cluster_agent_amd64:
   extends: [.docker_build_job_definition_amd64]

.gitlab/deploy_containers/deploy_containers_a7.yml

@@ -88,6 +88,18 @@ include:
     - export IMG_SOURCES="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips${JMX}-amd64,${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips${JMX}-arm64"
     - export IMG_DESTINATIONS="${AGENT_REPOSITORY}:${VERSION}${JMX}"
 
+.deploy_containers-a7-full-base:
+  extends: .docker_publish_job_definition
+  stage: deploy_containers
+  dependencies: []
+  before_script:
+    - if [[ "$VERSION" == "" ]]; then VERSION="$(inv agent.version --url-safe --pipeline-id $PARENT_PIPELINE_ID)" || exit $?; fi
+    - export IMG_BASE_SRC="${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
+    - export IMG_LINUX_SOURCES="${IMG_BASE_SRC}-7-full-amd64,${IMG_BASE_SRC}-7-full-arm64"
+    - export IMG_WINDOWS_SOURCES="${IMG_BASE_SRC}-7-jmx-win1809-amd64,${IMG_BASE_SRC}-7-jmx-winltsc2022-amd64"
+    - export IMG_SOURCES="${IMG_LINUX_SOURCES},${IMG_WINDOWS_SOURCES}"
+    - export IMG_DESTINATIONS="${AGENT_REPOSITORY}:${VERSION}-full"
+
 deploy_containers-a7:
   extends: .deploy_containers-a7_external
   rules:
@@ -98,6 +110,11 @@ deploy_containers-a7-win-only:
   rules:
     !reference [.manual_on_deploy_auto_on_rc]
 
+deploy_containers-a7-full:
+  extends: .deploy_containers-a7-full-base
+  rules:
+    !reference [.manual_on_deploy_auto_on_rc]
+
 deploy_containers-a7-ot:
   extends: .deploy_containers-a7-base-ot
   rules:
@@ -144,6 +161,13 @@ deploy_containers-a7-ot-rc:
           - ""
           - "-jmx"
 
+deploy_containers-a7-full-rc:
+  extends: .deploy_containers-a7-full-base
+  rules:
+    !reference [.on_rc]
+  variables:
+    VERSION: 7-full-rc
+
 deploy_containers-a7-fips-rc:
   extends: .deploy_containers-a7-base-fips
   rules:
@@ -264,6 +288,10 @@ deploy_containers_latest-a7:
       - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx"
         IMG_SOURCES: "%BASE%-win1809-servercore-amd64"
         IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-servercore-ltsc2019-jmx,${AGENT_REPOSITORY}:latest-servercore-ltsc2019-jmx
+      # Full image: for linux we need 7-full tag, for windows we need 7-jmx
+      - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7"
+        IMG_SOURCES: "%BASE%-full-amd64,%BASE%-full-arm64,%BASE%-jmx-win1809-amd64,%BASE%-jmx-winltsc2022-amd64"
+        IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-full,${AGENT_REPOSITORY}:latest-full
 
 deploy_containers_latest-a7_internal:
   extends: .docker_publish_job_definition
@@ -276,6 +304,11 @@ deploy_containers_latest-a7_internal:
       - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx"
         IMG_SOURCES: "%BASE%-amd64,%BASE%-arm64,%BASE%-win1809-amd64,%BASE%-winltsc2022-amd64"
         IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-jmx
+      # Full image: for linux we need 7-full tag, for windows we need 7-jmx
+      - IMG_VARIABLES: "BASE=${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7"
+        IMG_SOURCES: "%BASE%-full-amd64,%BASE%-full-arm64,%BASE%-jmx-win1809-amd64,%BASE%-jmx-winltsc2022-amd64"
+        IMG_DESTINATIONS: ${AGENT_REPOSITORY}:7-full
+
 
 deploy_containers_latest-dogstatsd:
   extends: .docker_publish_job_definition

.gitlab/dev_container_deploy/docker_linux.yml

@@ -177,7 +177,6 @@ dev_nightly-a7:
       - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64
         IMG_DESTINATIONS: agent-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-py3-jmx,agent-dev:nightly-${CI_COMMIT_REF_SLUG}-py3-jmx
 
-# deploys nightlies to agent-dev
 dev_nightly-ot-a7:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy
@@ -196,6 +195,18 @@ dev_nightly-ot-a7:
       - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta-jmx-arm64
         IMG_DESTINATIONS: agent-dev:nightly-ot-beta-${CI_COMMIT_REF_SLUG}-jmx
 
+dev_nightly-a7-full:
+  extends: .docker_publish_job_definition
+  stage: dev_container_deploy
+  rules: !reference [.on_deploy_nightly_repo_branch]
+  needs:
+    - docker_build_agent7_full
+    - docker_build_agent7_full_arm64
+  variables:
+    IMG_REGISTRIES: dev
+    IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-full-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-full-arm64
+    IMG_DESTINATIONS: agent-dev:nightly-full-${CI_COMMIT_REF_SLUG}-jmx
+
 # deploys nightlies to agent-dev
 dev_nightly-dogstatsd:
   extends: .docker_publish_job_definition
@@ -228,3 +239,15 @@ dev_branch-ot:
       - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta-jmx-arm64
         IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-7-ot-beta-jmx
 
+dev_branch-full:
+  extends: .docker_publish_job_definition
+  stage: dev_container_deploy
+  rules:
+    - !reference [.manual]
+  needs:
+    - docker_build_agent7_full
+    - docker_build_agent7_full_arm64
+  variables:
+    IMG_REGISTRIES: dev
+    IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-full-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-full-arm64
+    IMG_DESTINATIONS: agent-dev:${CI_COMMIT_REF_SLUG}-full

.gitlab/dev_container_deploy/e2e.yml

@@ -80,6 +80,20 @@ qa_agent_ot:
     IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta-arm64
     IMG_DESTINATIONS: agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta
 
+qa_agent_full:
+  extends: .docker_publish_job_definition
+  stage: dev_container_deploy
+  rules:
+    - !reference [.on_otel_or_e2e_changes]
+    - !reference [.manual]
+  needs:
+    - docker_build_agent7_full
+    - docker_build_agent7_full_arm64
+  variables:
+    IMG_REGISTRIES: agent-qa
+    IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-full-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-full-arm64
+    IMG_DESTINATIONS: agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-full
+
 qa_dca:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy

.gitlab/internal_image_deploy/internal_image_deploy.yml

@@ -233,3 +233,49 @@ docker_trigger_cluster_agent_internal-fips:
       --variable TARGET_ENV
       --variable DYNAMIC_BUILD_RENDER_TARGET_FORWARD_PARAMETERS"
   retry: 2
+
+docker_trigger_internal-full:
+  stage: internal_image_deploy
+  rules: !reference [.on_deploy_internal_or_manual]
+  needs:
+    - job: docker_build_ot_agent7_jmx
+      artifacts: false
+    - job: docker_build_ot_agent7_jmx_arm64
+      artifacts: false
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  variables:
+    DYNAMIC_BUILD_RENDER_RULES: agent-build-only # fake rule to not trigger the ones in the images repo
+    IMAGE_VERSION: tmpl-v12
+    IMAGE_NAME: datadog-agent
+    RELEASE_TAG: ${CI_COMMIT_REF_SLUG}-full
+    BUILD_TAG: ${CI_COMMIT_REF_SLUG}-full
+    TMPL_SRC_IMAGE: v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-full
+    TMPL_SRC_REPO: ci/datadog-agent/agent
+    RELEASE_STAGING: "true"
+  script:
+    - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN
+    - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi
+    - |
+      if [ "$BUCKET_BRANCH" = "nightly" ]; then
+        RELEASE_TAG="${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"
+        TMPL_SRC_REPO="${TMPL_SRC_REPO}-nightly"
+      fi
+    - if [ "$BUCKET_BRANCH" = "dev" ]; then RELEASE_TAG="dev-${RELEASE_TAG}-${CI_COMMIT_SHORT_SHA}"; fi
+    - "inv pipeline.trigger-child-pipeline --project-name DataDog/images --git-ref master --timeout 3600
+      --variable IMAGE_VERSION
+      --variable IMAGE_NAME
+      --variable RELEASE_TAG
+      --variable BUILD_TAG
+      --variable TMPL_SRC_IMAGE
+      --variable TMPL_SRC_REPO
+      --variable RELEASE_STAGING
+      --variable RELEASE_PROD
+      --variable DYNAMIC_BUILD_RENDER_RULES
+      --variable APPS
+      --variable BAZEL_TARGET
+      --variable DDR
+      --variable DDR_WORKFLOW_ID
+      --variable TARGET_ENV
+      --variable DYNAMIC_BUILD_RENDER_TARGET_FORWARD_PARAMETERS"
+  retry: 2

.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml

@@ -17,7 +17,7 @@ internal_kubernetes_deploy_experimental:
   needs:
     - job: docker_trigger_internal
       artifacts: false
-    - job: docker_trigger_internal-ot
+    - job: docker_trigger_internal-full
       artifacts: false
     - job: docker_trigger_cluster_agent_internal
       artifacts: false
@@ -44,7 +44,7 @@ internal_kubernetes_deploy_experimental:
   tags: ["arch:amd64"]
   variables:
     OPTION_AUTOMATIC_ROLLOUT: "true"
-    OPTION_PRE_SCRIPT: "patch-cluster-images-operator.sh env=ci ${CI_COMMIT_REF_SLUG}-ot-beta-jmx-${CI_COMMIT_SHORT_SHA} ${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}"
+    OPTION_PRE_SCRIPT: "patch-cluster-images-operator.sh env=ci ${CI_COMMIT_REF_SLUG}-full-${CI_COMMIT_SHORT_SHA} ${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}"
     SKIP_PLAN_CHECK: "true"
     EXPLICIT_WORKFLOWS: "//workflows:beta_builds.agents_nightly.staging-deploy.publish,//workflows:beta_builds.agents_nightly.staging-validate.publish,//workflows:beta_builds.agents_nightly.prod-wait-business-hours.publish,//workflows:beta_builds.agents_nightly.prod-deploy.publish,//workflows:beta_builds.agents_nightly.prod-validate.publish,//workflows:beta_builds.agents_nightly.publish-image-confirmation.publish"
     BUNDLE_VERSION_OVERRIDE: "v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"

.gitlab/package_build/windows.yml

@@ -40,6 +40,7 @@
       -e AGENT_FLAVOR=${AGENT_FLAVOR}
       -e OMNIBUS_SOFTWARE_VERSION="${OMNIBUS_SOFTWARE_VERSION}"
       -e OMNIBUS_RUBY_VERSION="${OMNIBUS_RUBY_VERSION}"
+      -e PYTHONUTF8=1
       ${WINBUILDIMAGE}
       powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1 -BuildUpgrade 1"
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }