Skip to content

Commit

Permalink
fix: Restore upload_logs script in use by acir bench
Browse files Browse the repository at this point in the history
Had been accidentally deleted as part of #11202
  • Loading branch information
spalladino committed Jan 14, 2025
1 parent f2885ec commit 2d88497
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 0 deletions.
36 changes: 36 additions & 0 deletions scripts/logs/download_logs_from_s3.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
# Downloads the log files uploaded in upload_logs_to_s3

set -eu

BUCKET_NAME="aztec-ci-artifacts"
LOG_FOLDER="${LOG_FOLDER:-log}"
COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}"

echo "Downloading logs from S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}"

# Paths from upload_logs_to_s3
if [ "${BRANCH:-}" = "master" ]; then
LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH"
BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH"
BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json"
BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json"
elif [ -n "${PULL_REQUEST:-}" ]; then
LOG_SOURCE_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}"
BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}"
BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${PULL_REQUEST##*/}.json"
else
echo "Skipping benchmark run on branch ${BRANCH:-unknown}."
exit 0
fi

mkdir -p $LOG_FOLDER

# Download benchmark log files from S3 LOG_SOURCE_FOLDER into local LOG_FOLDER
echo "Downloading benchmark log files from $BUCKET_NAME/$LOG_SOURCE_FOLDER to $LOG_FOLDER"
aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include 'bench*.jsonl' --recursive

# Download barretenberg log files, these are direct benchmarks and separate from the above
aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive

echo "Downloaded log files $(ls $LOG_FOLDER)"
33 changes: 33 additions & 0 deletions scripts/logs/upload_logs_to_s3.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env bash

# Uploads to S3 the contents of the log file mounted on the end-to-end container,
# which contains log entries with an associated event and metrics for it.
# Logs are uploaded to aztec-ci-artifacts/logs-v1/master/$COMMIT/$JOB.jsonl
# or to aztec-ci-artifacts/logs-v1/pulls/$PRNUMBER/$JOB.jsonl if on a PR

[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
set -eu

LOG_FOLDER=$1
BUCKET_NAME="aztec-ci-artifacts"
COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}"

echo "Uploading logs to S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}"

if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then
echo "No logs in folder $LOG_FOLDER to upload"
exit 0
fi

# Paths used in scripts/ci/assemble_e2e_benchmark.sh
if [ "${BRANCH:-}" = "master" ]; then
TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/"
elif [ -n "${PULL_REQUEST:-}" ]; then
TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}"
fi

if [ -n "${TARGET_FOLDER:-}" ]; then
aws s3 cp $LOG_FOLDER "s3://${BUCKET_NAME}/${TARGET_FOLDER}" --include "*.jsonl" --recursive
else
echo Skipping upload since no target folder was defined
fi

0 comments on commit 2d88497

Please sign in to comment.