Skip to content

Commit

Permalink
buildkite: clean up docker containers from previous jobs
Browse files Browse the repository at this point in the history
When buildkite cancels jobs, it doesn't run the post-coomand hook, so some
docker containers can continue running.

PiperOrigin-RevId: 721001481
  • Loading branch information
avagin authored and gvisor-bot committed Jan 29, 2025
1 parent ddebbe5 commit c191ceb
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 8 deletions.
26 changes: 26 additions & 0 deletions .buildkite/hooks/libhook
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#!/bin/bash

# Copyright 2025 The gVisor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

clear_docker_containers() {
# Kill any running containers (clear state), except for "bootstrap".
for id_and_name in $(docker ps --format='{{.ID}}/{{.Names}}'); do
if [[ "$(echo "$id_and_name" | cut -d'/' -f2-)" == 'bootstrap' ]]; then
continue
fi
timeout --kill-after=10s --preserve-status 30s \
docker container kill "$(echo "$id_and_name" | cut -d'/' -f1)"
done
}
11 changes: 3 additions & 8 deletions .buildkite/hooks/post-command
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
set -x

source .buildkite/hooks/libhook

# Clear any downloaded credentials.
rm -f repo.key

Expand Down Expand Up @@ -70,14 +72,7 @@ if test "${BUILDKITE_COMMAND_EXIT_STATUS}" -ne "0"; then
sudo rm -rf "${HOME}/go"
fi

# Kill any running containers (clear state), except for "bootstrap".
for id_and_name in $(docker ps --format='{{.ID}}/{{.Names}}'); do
if [[ "$(echo "$id_and_name" | cut -d'/' -f2-)" == 'bootstrap' ]]; then
continue
fi
timeout --kill-after=10s --preserve-status 8s \
docker container kill "$(echo "$id_and_name" | cut -d'/' -f1)"
done
clear_docker_containers

set -euo pipefail

Expand Down
4 changes: 4 additions & 0 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
set -euo pipefail

source .buildkite/hooks/libhook

clear_docker_containers

# Use a per-day bazel remote cache. As the cache object's TTL expires, they are
# deleted on an ongoing basis. Such partial deletion can break the cache state.
# Using per day cache will ensure that builds triggered on a certain day have
Expand Down

0 comments on commit c191ceb

Please sign in to comment.