Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[oadp-1.1] OADP-774 must-gather: add timeout to velero logs/describe, var typos, remove duplicate logs, add make run #821

Merged
merged 5 commits into from
Sep 20, 2022
Merged

[oadp-1.1] OADP-774 must-gather: add timeout to velero logs/describe, var typos, remove duplicate logs, add make run #821

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5b34707
Typo shell variable
kaovilai Sep 8, 2022
a11aa38
mustgather `make run` target
kaovilai Sep 8, 2022
100968c
do not gather restic logs twice
kaovilai Sep 8, 2022
d885f5d
force a timeout of velero command execution
kaovilai Sep 9, 2022
a0db18b
Use value for timeout from $timeout
kaovilai Sep 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions must-gather/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,13 @@ PROMETHEUS_DUMP_PATH ?= $(shell find ./must-gather.local* -name prom_data.tar.gz

build: docker-build docker-push

run: IMAGE_REGISTRY:=ttl.sh
run: IMAGE_NAME:=oadp/must-gather-$(shell git rev-parse --short HEAD)-$(shell echo $$RANDOM)
run: IMAGE_TAG:=1h
run:
IMAGE_REGISTRY=$(IMAGE_REGISTRY) IMAGE_NAME=$(IMAGE_NAME) IMAGE_TAG=$(IMAGE_TAG) make build && \
oc adm must-gather --image ${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}

docker-build:
docker build -t ${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG} .

Expand Down
6 changes: 3 additions & 3 deletions must-gather/collection-scripts/logs/gather_logs_backup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ skip_tls=$8
mkdir -p "{object_collection_path}"
echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero backup describe ${backup}'"
if [ "$timeout" = "0s" ]; then
oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" &
oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" &
else
oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" &
oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout ${timeout} /velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" &
fi
echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero backup logs ${backup}'"
oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero backup logs ${backup} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/backup-${backup}.log" &
oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero backup logs ${backup} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/backup-${backup}.log" &

wait
9 changes: 1 addition & 8 deletions must-gather/collection-scripts/logs/gather_logs_pvb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,5 @@ object_collection_path=$6
node=$(oc get podvolumebackup $pvb --namespace $ns -o jsonpath='{.spec.node}')
mkdir -p ${object_collection_path}
oc describe podvolumebackup ${pvb} --namespace ${ns} &> "${object_collection_path}/pvb-describe-${pvb}.txt" &
for pod in $(oc get pods -o wide --field-selector spec.nodeName=${node} --selector name=restic --no-headers --namespace $ns | awk '{print $1}'); do
echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting Pod logs..."
oc logs --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/current.log" &
echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting previous Pod logs..."
oc logs --previous --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/previous.log" &
pwait $max_parallelism
done

# logs covered by restic pod logs in gather_logs_pods
wait
8 changes: 2 additions & 6 deletions must-gather/collection-scripts/logs/gather_logs_pvr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,8 @@ max_parallelism=$4
pvr=$5
object_collection_path=$6

# Gather PVR describe and logs
# Gather PVR describe
mkdir -p ${object_collection_path}
oc describe podvolumerestores.velero.io ${pvr} --namespace ${ns} &> "${object_collection_path}/pvr-describe-${pvr}.txt"
echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting Pod logs..."
oc logs --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/current.log" &
echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting previous Pod logs..."
oc logs --previous --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/previous.log" &

# logs covered by restic pod logs in gather_logs_pods
wait
8 changes: 4 additions & 4 deletions must-gather/collection-scripts/logs/gather_logs_restore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ timeout=$7
skip_tls=$8

# Gather restore describe and logs
mkdir -p "{object_collection_path}"
mkdir -p "${object_collection_path}"
echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero restore describe ${restore}'"
if [ "$timeout" = "0s" ]; then
oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" &
oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" &
weshayutin marked this conversation as resolved.
Show resolved Hide resolved
else
oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" &
oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout ${timeout} /velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" &
fi
echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero restore logs ${restore}'"
oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero restore logs ${restore} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/restore-${restore}.log" &
oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero restore logs ${restore} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/restore-${restore}.log" &

wait