-
Notifications
You must be signed in to change notification settings - Fork 324
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: Restore upload_logs script in use by acir bench
Had been accidentally deleted as part of #11202
- Loading branch information
1 parent
f2885ec
commit 2d88497
Showing
2 changed files
with
69 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
#!/usr/bin/env bash | ||
# Downloads the log files uploaded in upload_logs_to_s3 | ||
|
||
set -eu | ||
|
||
BUCKET_NAME="aztec-ci-artifacts" | ||
LOG_FOLDER="${LOG_FOLDER:-log}" | ||
COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" | ||
|
||
echo "Downloading logs from S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}" | ||
|
||
# Paths from upload_logs_to_s3 | ||
if [ "${BRANCH:-}" = "master" ]; then | ||
LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" | ||
BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH" | ||
BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json" | ||
BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json" | ||
elif [ -n "${PULL_REQUEST:-}" ]; then | ||
LOG_SOURCE_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" | ||
BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}" | ||
BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${PULL_REQUEST##*/}.json" | ||
else | ||
echo "Skipping benchmark run on branch ${BRANCH:-unknown}." | ||
exit 0 | ||
fi | ||
|
||
mkdir -p $LOG_FOLDER | ||
|
||
# Download benchmark log files from S3 LOG_SOURCE_FOLDER into local LOG_FOLDER | ||
echo "Downloading benchmark log files from $BUCKET_NAME/$LOG_SOURCE_FOLDER to $LOG_FOLDER" | ||
aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include 'bench*.jsonl' --recursive | ||
|
||
# Download barretenberg log files, these are direct benchmarks and separate from the above | ||
aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive | ||
|
||
echo "Downloaded log files $(ls $LOG_FOLDER)" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
#!/usr/bin/env bash | ||
|
||
# Uploads to S3 the contents of the log file mounted on the end-to-end container, | ||
# which contains log entries with an associated event and metrics for it. | ||
# Logs are uploaded to aztec-ci-artifacts/logs-v1/master/$COMMIT/$JOB.jsonl | ||
# or to aztec-ci-artifacts/logs-v1/pulls/$PRNUMBER/$JOB.jsonl if on a PR | ||
|
||
[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace | ||
set -eu | ||
|
||
LOG_FOLDER=$1 | ||
BUCKET_NAME="aztec-ci-artifacts" | ||
COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" | ||
|
||
echo "Uploading logs to S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}" | ||
|
||
if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then | ||
echo "No logs in folder $LOG_FOLDER to upload" | ||
exit 0 | ||
fi | ||
|
||
# Paths used in scripts/ci/assemble_e2e_benchmark.sh | ||
if [ "${BRANCH:-}" = "master" ]; then | ||
TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" | ||
elif [ -n "${PULL_REQUEST:-}" ]; then | ||
TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" | ||
fi | ||
|
||
if [ -n "${TARGET_FOLDER:-}" ]; then | ||
aws s3 cp $LOG_FOLDER "s3://${BUCKET_NAME}/${TARGET_FOLDER}" --include "*.jsonl" --recursive | ||
else | ||
echo Skipping upload since no target folder was defined | ||
fi |