diff --git a/scripts/logs/download_logs_from_s3.sh b/scripts/logs/download_logs_from_s3.sh new file mode 100755 index 00000000000..a31979e3968 --- /dev/null +++ b/scripts/logs/download_logs_from_s3.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Downloads the log files uploaded in upload_logs_to_s3 + +set -eu + +BUCKET_NAME="aztec-ci-artifacts" +LOG_FOLDER="${LOG_FOLDER:-log}" +COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" + +echo "Downloading logs from S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}" + +# Paths from upload_logs_to_s3 +if [ "${BRANCH:-}" = "master" ]; then + LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" + BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH" + BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json" + BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json" +elif [ -n "${PULL_REQUEST:-}" ]; then + LOG_SOURCE_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" + BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}" + BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${PULL_REQUEST##*/}.json" +else + echo "Skipping benchmark run on branch ${BRANCH:-unknown}." + exit 0 +fi + +mkdir -p $LOG_FOLDER + +# Download benchmark log files from S3 LOG_SOURCE_FOLDER into local LOG_FOLDER +echo "Downloading benchmark log files from $BUCKET_NAME/$LOG_SOURCE_FOLDER to $LOG_FOLDER" +aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include 'bench*.jsonl' --recursive + +# Download barretenberg log files, these are direct benchmarks and separate from the above +aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive + +echo "Downloaded log files $(ls $LOG_FOLDER)" \ No newline at end of file diff --git a/scripts/logs/upload_logs_to_s3.sh b/scripts/logs/upload_logs_to_s3.sh new file mode 100755 index 00000000000..4a2c77c22d8 --- /dev/null +++ b/scripts/logs/upload_logs_to_s3.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Uploads to S3 the contents of the log file mounted on the end-to-end container, +# which contains log entries with an associated event and metrics for it. +# Logs are uploaded to aztec-ci-artifacts/logs-v1/master/$COMMIT/$JOB.jsonl +# or to aztec-ci-artifacts/logs-v1/pulls/$PRNUMBER/$JOB.jsonl if on a PR + +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +LOG_FOLDER=$1 +BUCKET_NAME="aztec-ci-artifacts" +COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" + +echo "Uploading logs to S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}" + +if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then + echo "No logs in folder $LOG_FOLDER to upload" + exit 0 +fi + +# Paths used in scripts/ci/assemble_e2e_benchmark.sh +if [ "${BRANCH:-}" = "master" ]; then + TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" +elif [ -n "${PULL_REQUEST:-}" ]; then + TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" +fi + +if [ -n "${TARGET_FOLDER:-}" ]; then + aws s3 cp $LOG_FOLDER "s3://${BUCKET_NAME}/${TARGET_FOLDER}" --include "*.jsonl" --recursive +else + echo Skipping upload since no target folder was defined +fi \ No newline at end of file