From 4271a6ab687911a5fed927027849795e04b92596 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Thu, 8 Sep 2022 13:50:08 -0400 Subject: [PATCH 1/5] Typo shell variable --- must-gather/collection-scripts/logs/gather_logs_restore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/must-gather/collection-scripts/logs/gather_logs_restore b/must-gather/collection-scripts/logs/gather_logs_restore index 1e8ba1bedd..855d8b7e6e 100755 --- a/must-gather/collection-scripts/logs/gather_logs_restore +++ b/must-gather/collection-scripts/logs/gather_logs_restore @@ -12,7 +12,7 @@ timeout=$7 skip_tls=$8 # Gather restore describe and logs -mkdir -p "{object_collection_path}" +mkdir -p "${object_collection_path}" echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero restore describe ${restore}'" if [ "$timeout" = "0s" ]; then oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" & From 45956dbc3d073f02534d664d2d2feec2c8f533eb Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Thu, 8 Sep 2022 13:58:08 -0400 Subject: [PATCH 2/5] mustgather `make run` target Signed-off-by: Tiger Kaovilai --- must-gather/Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/must-gather/Makefile b/must-gather/Makefile index 4305a2506e..1a316efacc 100644 --- a/must-gather/Makefile +++ b/must-gather/Makefile @@ -8,6 +8,13 @@ PROMETHEUS_DUMP_PATH ?= $(shell find ./must-gather.local* -name prom_data.tar.gz build: docker-build docker-push +run: IMAGE_REGISTRY:=ttl.sh +run: IMAGE_NAME:=oadp/must-gather-$(shell git rev-parse --short HEAD)-$(shell echo $$RANDOM) +run: IMAGE_TAG:=1h +run: + IMAGE_REGISTRY=$(IMAGE_REGISTRY) IMAGE_NAME=$(IMAGE_NAME) IMAGE_TAG=$(IMAGE_TAG) make build && \ + oc adm must-gather --image ${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG} + docker-build: docker build -t ${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG} . From 228195cb32caad7406770725b5707e177b549d1c Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Thu, 8 Sep 2022 15:44:38 -0400 Subject: [PATCH 3/5] do not gather restic logs twice Signed-off-by: Tiger Kaovilai --- must-gather/collection-scripts/logs/gather_logs_pvb | 9 +-------- must-gather/collection-scripts/logs/gather_logs_pvr | 8 ++------ 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/must-gather/collection-scripts/logs/gather_logs_pvb b/must-gather/collection-scripts/logs/gather_logs_pvb index 137f2d4d56..c805045c14 100755 --- a/must-gather/collection-scripts/logs/gather_logs_pvb +++ b/must-gather/collection-scripts/logs/gather_logs_pvb @@ -13,12 +13,5 @@ object_collection_path=$6 node=$(oc get podvolumebackup $pvb --namespace $ns -o jsonpath='{.spec.node}') mkdir -p ${object_collection_path} oc describe podvolumebackup ${pvb} --namespace ${ns} &> "${object_collection_path}/pvb-describe-${pvb}.txt" & -for pod in $(oc get pods -o wide --field-selector spec.nodeName=${node} --selector name=restic --no-headers --namespace $ns | awk '{print $1}'); do - echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting Pod logs..." - oc logs --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/current.log" & - echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting previous Pod logs..." - oc logs --previous --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/previous.log" & - pwait $max_parallelism -done - +# logs covered by restic pod logs in gather_logs_pods wait \ No newline at end of file diff --git a/must-gather/collection-scripts/logs/gather_logs_pvr b/must-gather/collection-scripts/logs/gather_logs_pvr index f6f3d547d8..bfdb24b808 100755 --- a/must-gather/collection-scripts/logs/gather_logs_pvr +++ b/must-gather/collection-scripts/logs/gather_logs_pvr @@ -9,12 +9,8 @@ max_parallelism=$4 pvr=$5 object_collection_path=$6 -# Gather PVR describe and logs +# Gather PVR describe mkdir -p ${object_collection_path} oc describe podvolumerestores.velero.io ${pvr} --namespace ${ns} &> "${object_collection_path}/pvr-describe-${pvr}.txt" -echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting Pod logs..." -oc logs --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/current.log" & -echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting previous Pod logs..." -oc logs --previous --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/previous.log" & - +# logs covered by restic pod logs in gather_logs_pods wait \ No newline at end of file From cf13ed2e06434a8da2904254545f8e50cb0738ff Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 9 Sep 2022 13:44:12 -0400 Subject: [PATCH 4/5] force a timeout of velero command execution --- must-gather/collection-scripts/logs/gather_logs_backup | 6 +++--- must-gather/collection-scripts/logs/gather_logs_restore | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/must-gather/collection-scripts/logs/gather_logs_backup b/must-gather/collection-scripts/logs/gather_logs_backup index e5ec86350a..e688f26758 100755 --- a/must-gather/collection-scripts/logs/gather_logs_backup +++ b/must-gather/collection-scripts/logs/gather_logs_backup @@ -15,11 +15,11 @@ skip_tls=$8 mkdir -p "{object_collection_path}" echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero backup describe ${backup}'" if [ "$timeout" = "0s" ]; then - oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" & + oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" & else - oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" & + oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" & fi echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero backup logs ${backup}'" -oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero backup logs ${backup} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/backup-${backup}.log" & +oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero backup logs ${backup} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/backup-${backup}.log" & wait \ No newline at end of file diff --git a/must-gather/collection-scripts/logs/gather_logs_restore b/must-gather/collection-scripts/logs/gather_logs_restore index 855d8b7e6e..c12dce56d4 100755 --- a/must-gather/collection-scripts/logs/gather_logs_restore +++ b/must-gather/collection-scripts/logs/gather_logs_restore @@ -15,11 +15,11 @@ skip_tls=$8 mkdir -p "${object_collection_path}" echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero restore describe ${restore}'" if [ "$timeout" = "0s" ]; then - oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" & + oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" & else - oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" & + oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" & fi echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero restore logs ${restore}'" -oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero restore logs ${restore} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/restore-${restore}.log" & +oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero restore logs ${restore} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/restore-${restore}.log" & wait \ No newline at end of file From fbfd433c1d000ee947ff3dd2db8c723c0bf1bd9f Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 9 Sep 2022 16:54:01 -0400 Subject: [PATCH 5/5] Use value for timeout from $timeout --- must-gather/collection-scripts/logs/gather_logs_backup | 2 +- must-gather/collection-scripts/logs/gather_logs_restore | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/must-gather/collection-scripts/logs/gather_logs_backup b/must-gather/collection-scripts/logs/gather_logs_backup index e688f26758..3366398fe1 100755 --- a/must-gather/collection-scripts/logs/gather_logs_backup +++ b/must-gather/collection-scripts/logs/gather_logs_backup @@ -17,7 +17,7 @@ echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero backup describe ${backup} if [ "$timeout" = "0s" ]; then oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" & else - oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" & + oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout ${timeout} /velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" & fi echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero backup logs ${backup}'" oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero backup logs ${backup} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/backup-${backup}.log" & diff --git a/must-gather/collection-scripts/logs/gather_logs_restore b/must-gather/collection-scripts/logs/gather_logs_restore index c12dce56d4..be27612621 100755 --- a/must-gather/collection-scripts/logs/gather_logs_restore +++ b/must-gather/collection-scripts/logs/gather_logs_restore @@ -17,7 +17,7 @@ echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero restore describe ${restor if [ "$timeout" = "0s" ]; then oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" & else - oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" & + oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout ${timeout} /velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" & fi echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero restore logs ${restore}'" oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero restore logs ${restore} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/restore-${restore}.log" &