Deliver Cilium debug symbols as separate files (#550)

Provides debug symbols and ensures `release` and `debug` binaries are from the same build - so symbol files always match.

- If the build is invoked with `NOSTRIP=0`, then the `release` image has stripped binaries and the `debug` image has stripped binaries + debug symbol files in `/usr/lib/debug`.
- If the build is invoked with `NOSTRIP=1`, then the `release` image has non-stripped binaries and the `debug` image has non-stripped binaries + debug symbol files in `/usr/lib/debug`.

The latter is somewhat redundant in the `debug` image since the debug symbols are present both in the binaires and in `/usr/lib/debug`. I'm hoping this will be acceptable upstream.

Part of the idea is that we should no longer need to touch `NOSTRIP`. In our current builds, the problem is that both our `release` and `debug` builds are lacking symbols by default. To get builds with symbols, we need to go and hack `.gitlab-ci.yml` and flip `NOSTRIP`, branch, tag and push. We can’t use our default `debug` images because they have no symbols, so delve doesn’t understand anything about the binary it’s running.

With this change, all tagged builds deliver a `release` image that is either stripped or not according to `NOSTRIP` (so stripped with our default `.gitlab-ci.yml`), and a `debug` image that includes the symbol files.
So the `debug` image works: delve understands the binary, you can do remote debugging from your IDE with port-forward etc.

Also, by systematically delivering the debug symbol files in the `debug` image in `/usr/lib/debug`, it makes it easy to split them out for post-processing. If, for instance, I didn’t include the debug files in the non-stripped version of the `debug` image, then it would be harder to identify the files that should be post-processed.

So I think this strikes a good balance in terms of making it easy to know where to find debug symbols and something I hope upstream should be OK with (it improves usability of the `debug` image, since currently if `NOSTRIP=0`, then the `debug` image has no symbols and it’s useless with Delve and Co).

.gitlab-ci.yml

@@ -56,27 +56,13 @@ build-docker-image-runtime:
     DOCKER_CTX: "./images/runtime"
     TARGET: release
 
-build-docker-image-cilium:
-  <<: *build-docker-image
-  needs:
-    # The cilium image depends on the runtime image
-    - build-docker-image-runtime
-  variables:
-    IMAGE_NAME: cilium
-    DOCKERFILE_PATH: images/cilium/Dockerfile
-    DOCKER_BUILD_ARGS: |
-      CILIUM_RUNTIME_IMAGE=registry.ddbuild.io/cilium-runtime:$CI_COMMIT_TAG
-      CILIUM_BUILDER_IMAGE=registry.ddbuild.io/images/mirror/cilium/cilium-builder:f229913ec72a183640bd46d0dd0579ebea3bb1c6@sha256:6ec80f7123cbf83008420b34c458f2e18e2091a648c0926ae3a601820468d902
-      CILIUM_ENVOY_IMAGE=registry.ddbuild.io/images/mirror/cilium/cilium-envoy:v1.26-39dc41f86c465d2a2d16386339dc0bf4d425babc@sha256:e77adfe8a263fe4b8c56dcb9bd0f4d68bb36067602e7be1388528c02fb8765c5
-    TARGET: release
-
 # Caveats:
 # * The build image is single-arch amd64 and we're doing cross-compilation, so the dlv copy is only valid on amd64. In
 #   other words, the arm64 image does not work.
-build-docker-image-cilium-debug:
+build-docker-image-cilium:
   <<: *build-docker-image
   needs:
-    # The debug image depends on the runtime image
+    # The cilium image depends on the runtime image
     - build-docker-image-runtime
   variables:
     IMAGE_NAME: cilium
@@ -85,8 +71,20 @@ build-docker-image-cilium-debug:
       CILIUM_RUNTIME_IMAGE=registry.ddbuild.io/cilium-runtime:$CI_COMMIT_TAG
       CILIUM_BUILDER_IMAGE=registry.ddbuild.io/images/mirror/cilium/cilium-builder:f229913ec72a183640bd46d0dd0579ebea3bb1c6@sha256:6ec80f7123cbf83008420b34c458f2e18e2091a648c0926ae3a601820468d902
       CILIUM_ENVOY_IMAGE=registry.ddbuild.io/images/mirror/cilium/cilium-envoy:v1.26-39dc41f86c465d2a2d16386339dc0bf4d425babc@sha256:e77adfe8a263fe4b8c56dcb9bd0f4d68bb36067602e7be1388528c02fb8765c5
-      NOSTRIP=1
-    TARGET: debug
+    TARGET: release
+    NOSTRIP: 0
+  script:
+    - set -x
+    # Construct valid --build-args arguments from the DOCKER_BUILD_ARGS variable
+    - BUILD_ARGS=""; IFS=$'\n'; for arg in $DOCKER_BUILD_ARGS; do BUILD_ARGS+=" $(echo "--build-arg $arg")"; done; IFS=$' ';
+    - IMAGE_TAG="$CI_COMMIT_TAG"
+    - IMAGE_REF="registry.ddbuild.io/$IMAGE_NAME:$IMAGE_TAG"
+    - METADATA_FILE1=$(mktemp)
+    - METADATA_FILE2=$(mktemp)
+    - docker buildx build --platform linux/amd64,linux/arm64 --tag $IMAGE_REF --file $DOCKERFILE_PATH $BUILD_ARGS --label CILIUM_VERSION=$(cat VERSION) --label target=prod --target $TARGET --push --metadata-file $METADATA_FILE1 $DOCKER_CTX
+    - ddsign sign $IMAGE_REF --docker-metadata-file $METADATA_FILE1
+    - docker buildx build --platform linux/amd64,linux/arm64 --tag $IMAGE_REF-debug --file $DOCKERFILE_PATH $BUILD_ARGS --label CILIUM_VERSION=$(cat VERSION) --label target=debug --target debug --push --metadata-file $METADATA_FILE2 $DOCKER_CTX
+    - ddsign sign $IMAGE_REF-debug --docker-metadata-file $METADATA_FILE2
 
 build-docker-image-hubble-relay:
   <<: *build-docker-image

images/cilium/Dockerfile

@@ -47,17 +47,30 @@ ARG LIBNETWORK_PLUGIN
 #
 WORKDIR /go/src/github.com/cilium/cilium
 RUN --mount=type=bind,readwrite,target=/go/src/github.com/cilium/cilium --mount=target=/root/.cache,type=cache --mount=target=/go/pkg,type=cache \
-    make GOARCH=${TARGETARCH} RACE=${RACE} NOSTRIP=${NOSTRIP} NOOPT=${NOOPT} LOCKDEBUG=${LOCKDEBUG} PKG_BUILD=1 V=${V} LIBNETWORK_PLUGIN=${LIBNETWORK_PLUGIN} \
+    make GOARCH=${TARGETARCH} RACE=${RACE} NOSTRIP=1 NOOPT=${NOOPT} LOCKDEBUG=${LOCKDEBUG} PKG_BUILD=1 V=${V} LIBNETWORK_PLUGIN=${LIBNETWORK_PLUGIN} \
     DESTDIR=/tmp/install/${TARGETOS}/${TARGETARCH} build-container install-container-binary
 
 RUN --mount=type=bind,readwrite,target=/go/src/github.com/cilium/cilium --mount=target=/root/.cache,type=cache --mount=target=/go/pkg,type=cache \
     # install-bash-completion will execute the bash_completion script. It is
     # fine to run this with same architecture as BUILDARCH since the output of
     # bash_completion is the same for both architectures.
-    make GOARCH=${BUILDARCH} RACE=${RACE} NOSTRIP=${NOSTRIP} NOOPT=${NOOPT} LOCKDEBUG=${LOCKDEBUG} PKG_BUILD=1 V=${V} LIBNETWORK_PLUGIN=${LIBNETWORK_PLUGIN} \
+    make GOARCH=${BUILDARCH} RACE=${RACE} NOSTRIP=1 NOOPT=${NOOPT} LOCKDEBUG=${LOCKDEBUG} PKG_BUILD=1 V=${V} LIBNETWORK_PLUGIN=${LIBNETWORK_PLUGIN} \
     DESTDIR=/tmp/install/${TARGETOS}/${TARGETARCH} install-bash-completion licenses-all && \
     mv LICENSE.all /tmp/install/${TARGETOS}/${TARGETARCH}/LICENSE.all
 
+RUN set -xe && \
+    export D=/tmp/debug/${TARGETOS}/${TARGETARCH} && \
+    mkdir -p $D && \
+    cd /tmp/install/${TARGETOS}/${TARGETARCH} && \
+    find . -type f \
+      -executable \
+      -exec sh -c \
+        'filename=$(basename ${0}) && \
+         objcopy --only-keep-debug ${0} ${0}.debug && \
+         if [ "$NOSTRIP" != "1" ] ; then objcopy --strip-all ${0} && (cd $(dirname ${0}) && objcopy --add-gnu-debuglink=${filename}.debug ${filename}) ; fi && \
+         mv -v ${0}.debug ${D}/${filename}.debug' \
+      {} \;
+
 COPY images/cilium/init-container.sh \
      plugins/cilium-cni/cni-install.sh \
      plugins/cilium-cni/install-plugin.sh \
@@ -110,3 +123,12 @@ ARG TARGETARCH
 COPY --from=builder /go/bin/dlv /usr/bin/dlv
 RUN mv /usr/bin/cilium-agent /usr/bin/cilium-agent-bin
 COPY images/scripts/debug-wrapper.sh /usr/bin/cilium-agent
+
+# Copy in the debug symbols in case the binaries were stripped
+COPY --from=builder /tmp/debug/${TARGETOS}/${TARGETARCH} /usr/lib/debug
+
+# Ensure dlv finds the debug symbols. Due to CGO_ENABLED=0, we have no GNU build-id, so Delve's default search path
+# is insufficient.
+RUN mkdir -p ${HOME}/.config/dlv && \
+    echo 'debug-info-directories: ["/usr/lib/debug/.build-id","/usr/lib/debug"]' > ${HOME}/.config/dlv/config.yml && \
+    ln -s /usr/lib/debug/cilium-agent.debug /usr/lib/debug/cilium-agent-bin.debug