From 70d2ad2d8f0769988cfb0597c580f0c5c5f2d622 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Wed, 1 May 2024 19:49:42 +0000 Subject: [PATCH 01/26] test --- .github/actions/fc/fc.yml | 36 +++++++++++++++++++++++++++++++ .github/workflows/clusterfuzz.yml | 28 ++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 .github/actions/fc/fc.yml diff --git a/.github/actions/fc/fc.yml b/.github/actions/fc/fc.yml new file mode 100644 index 0000000000..d29972f606 --- /dev/null +++ b/.github/actions/fc/fc.yml @@ -0,0 +1,36 @@ +name: Generate Firedancer Fuzz Targets +on: + workflow_call: + inputs: + previous_action_outputs: + description: 'Outputs from the previous action' + required: true + type: string +jobs: + generate-fuzz-targets: + runs-on: ubuntu-latest + steps: + # - uses: actions/checkout@v4 + + - name: Download artifacts from previous action + id: get-artifacts + uses: actions/download-artifact@v4 + with: + path: build + merge-multiple: true + + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y zip unzip + + - name: Generate fuzz targets and .fc files + run: | + chmod +x bundle-binaries-fc.sh + ./contrib/test/bundle-binaries-fc.sh ${{ steps.get-artifacts.outputs.download-path }} fc-fd-${{ github.run_id }}.zip + + - name: Upload merged zipfile + uses: actions/upload-artifact@v4 + with: + name: fc-fd-${{ github.run_id }}.zip + path: fc-fd-${{ github.run_id }}.zip \ No newline at end of file diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index b99914e52c..aa94cacdfc 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -50,3 +50,31 @@ jobs: project-id: isol-clusterfuzz qualifier: ${{ matrix.qualifier }} service-account-credentials: ${{ secrets.FUZZ_SERVICE_ACCT_JSON_BUNDLE }} + + - name: Find the tmpdir containing our binaries and rewritten libs + id: find-fuzzdir + run: | + FD_DIR=$(find -d /tmp/ | grep fdfuzz | head -n 1) + echo "fd-dir=$FD_DIR" >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@v4 + name: Upload harness binaries + with: + name: ${{ github.run_id }}-${{ matrix.machine }} + path: ./build/ + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload .a dependency libraries + with: + name: ${{ github.run_id }}-${{ matrix.machine }}-lib + path: ${{ steps.find-fuzzdir.outputs.fd-dir }}/lib + retention-days: 1 + + fc-publish: + needs: clusterfuzz-publish + uses: ./.github/actions/fc/fc.yml + with: + previous_action_outputs: | + ${{ github.run_id }}-linux_clang_haswell + ${{ github.run_id }}-linux_clang_icelake + ${{ github.run_id }}-linux_clang_haswell-lib + ${{ github.run_id }}-linux_clang_icelake-lib From d76b16e40a31b12bbbd2d338a4060d1fcaf0a8b8 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Wed, 1 May 2024 20:37:43 +0000 Subject: [PATCH 02/26] we go againe --- .github/actions/deps/action.yml | 2 +- .github/workflows/clusterfuzz.yml | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/actions/deps/action.yml b/.github/actions/deps/action.yml index 576d094316..9a895a68a2 100644 --- a/.github/actions/deps/action.yml +++ b/.github/actions/deps/action.yml @@ -24,7 +24,7 @@ runs: - name: apt-get update shell: bash - run: sudo apt-get update + run: sudo apt-get update && sudo apt-get install -y cmake if: env.HAS_APT_GET == '1' - id: deps-sh-hash diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index aa94cacdfc..6b2cce9fa8 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -2,6 +2,7 @@ name: Publish to ClusterFuzz on: workflow_call: workflow_dispatch: + pull_request: jobs: clusterfuzz-publish: environment: @@ -20,17 +21,17 @@ jobs: artifact_dir: build/linux/clang/icelake qualifier: highend runs-on: - group: github-v1 + ubuntu-latest env: MACHINE: ${{ matrix.machine }} EXTRAS: fuzz asan ubsan steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/hugepages + # - uses: ./.github/actions/hugepages - uses: ./.github/actions/deps - - run: sudo apt update && sudo apt install -y zip + - run: sudo apt update && sudo apt install -y zip patchelf - uses: asymmetric-research/clusterfuzz-fuzzbot-builder@main name: Build fuzz tests From 56cbdbbae3cb8de0d450d7542e29607f8ab54a0c Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Wed, 1 May 2024 20:43:12 +0000 Subject: [PATCH 03/26] woop --- .github/workflows/clusterfuzz.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index 6b2cce9fa8..6f5d6ff978 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -72,10 +72,14 @@ jobs: fc-publish: needs: clusterfuzz-publish - uses: ./.github/actions/fc/fc.yml - with: - previous_action_outputs: | - ${{ github.run_id }}-linux_clang_haswell - ${{ github.run_id }}-linux_clang_icelake - ${{ github.run_id }}-linux_clang_haswell-lib - ${{ github.run_id }}-linux_clang_icelake-lib + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v4 + - uses: ./.github/actions/fc/fc.yml + with: + previous_action_outputs: | + ${{ github.run_id }}-linux_clang_haswell + ${{ github.run_id }}-linux_clang_icelake + ${{ github.run_id }}-linux_clang_haswell-lib + ${{ github.run_id }}-linux_clang_icelake-lib From dfc7b4ab839183a19cc148aafb80f7b8311447d1 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 2 May 2024 15:52:11 +0000 Subject: [PATCH 04/26] pls --- .github/workflows/clusterfuzz.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index 6f5d6ff978..43ce49471e 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -55,7 +55,7 @@ jobs: - name: Find the tmpdir containing our binaries and rewritten libs id: find-fuzzdir run: | - FD_DIR=$(find -d /tmp/ | grep fdfuzz | head -n 1) + FD_DIR=$(find /tmp/ | grep fdfuzz | head -n 1) echo "fd-dir=$FD_DIR" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@v4 name: Upload harness binaries @@ -67,7 +67,7 @@ jobs: name: Upload .a dependency libraries with: name: ${{ github.run_id }}-${{ matrix.machine }}-lib - path: ${{ steps.find-fuzzdir.outputs.fd-dir }}/lib + path: ${{ steps.find-fuzzdir.outputs.fd-dir }}/lib/*.a retention-days: 1 fc-publish: From 7cc893684050bcbac2a595ed63516f34db0931b5 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 2 May 2024 16:07:58 +0000 Subject: [PATCH 05/26] rename --- .github/actions/fc/{fc.yml => action.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/actions/fc/{fc.yml => action.yml} (100%) diff --git a/.github/actions/fc/fc.yml b/.github/actions/fc/action.yml similarity index 100% rename from .github/actions/fc/fc.yml rename to .github/actions/fc/action.yml From 191f5757dbebb2cc9125d9f615505e2596d51a32 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 2 May 2024 16:16:06 +0000 Subject: [PATCH 06/26] Facepalm --- .github/workflows/clusterfuzz.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index 43ce49471e..df55d2a1fe 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -76,7 +76,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/fc/fc.yml + - uses: ./.github/actions/fc with: previous_action_outputs: | ${{ github.run_id }}-linux_clang_haswell From c962d427cd3c34dabb26a33750fa3e2565838019 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 2 May 2024 16:26:26 +0000 Subject: [PATCH 07/26] please last one --- .github/actions/fc/action.yml | 6 +++--- .github/workflows/clusterfuzz.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index d29972f606..3be4bf719e 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -27,10 +27,10 @@ jobs: - name: Generate fuzz targets and .fc files run: | chmod +x bundle-binaries-fc.sh - ./contrib/test/bundle-binaries-fc.sh ${{ steps.get-artifacts.outputs.download-path }} fc-fd-${{ github.run_id }}.zip + ./contrib/test/bundle-binaries-fc.sh ./build fc-fd-latest.zip - name: Upload merged zipfile uses: actions/upload-artifact@v4 with: - name: fc-fd-${{ github.run_id }}.zip - path: fc-fd-${{ github.run_id }}.zip \ No newline at end of file + name: fc-fd-latest.zip + path: fc-fd-latest.zip \ No newline at end of file diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index df55d2a1fe..59fdf1bc8f 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -28,7 +28,7 @@ jobs: steps: - uses: actions/checkout@v4 - # - uses: ./.github/actions/hugepages + - uses: ./.github/actions/hugepages - uses: ./.github/actions/deps - run: sudo apt update && sudo apt install -y zip patchelf From 31c936fffac12355835c1609d8ccb9ccba7da4a2 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 2 May 2024 16:32:07 +0000 Subject: [PATCH 08/26] whoops --- .github/workflows/clusterfuzz.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index 59fdf1bc8f..df55d2a1fe 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -28,7 +28,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/hugepages + # - uses: ./.github/actions/hugepages - uses: ./.github/actions/deps - run: sudo apt update && sudo apt install -y zip patchelf From ceed2c35489d2c7b500f0b48bff4d6e91b509c9b Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 2 May 2024 16:41:59 +0000 Subject: [PATCH 09/26] please lort --- .github/actions/fc/action.yml | 62 +++++++++++++++++------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index 3be4bf719e..2435071210 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -1,36 +1,36 @@ name: Generate Firedancer Fuzz Targets -on: - workflow_call: - inputs: - previous_action_outputs: - description: 'Outputs from the previous action' - required: true - type: string -jobs: - generate-fuzz-targets: - runs-on: ubuntu-latest - steps: - # - uses: actions/checkout@v4 +description: Generates Firedancer fuzz targets and .fc files - - name: Download artifacts from previous action - id: get-artifacts - uses: actions/download-artifact@v4 - with: - path: build - merge-multiple: true +inputs: + previous_action_outputs: + description: 'Outputs from the previous action' + required: true + type: string - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y zip unzip +runs: + using: "composite" + steps: + - name: Download artifacts from previous action + id: get-artifacts + uses: actions/download-artifact@v4 + with: + path: build + merge-multiple: true - - name: Generate fuzz targets and .fc files - run: | - chmod +x bundle-binaries-fc.sh - ./contrib/test/bundle-binaries-fc.sh ./build fc-fd-latest.zip + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y zip unzip + shell: bash - - name: Upload merged zipfile - uses: actions/upload-artifact@v4 - with: - name: fc-fd-latest.zip - path: fc-fd-latest.zip \ No newline at end of file + - name: Generate fuzz targets and .fc files + run: | + chmod +x bundle-binaries-fc.sh + ./contrib/test/bundle-binaries-fc.sh ./build fc-fd-latest.zip + shell: bash + + - name: Upload merged zipfile + uses: actions/upload-artifact@v4 + with: + name: fc-fd-latest.zip + path: fc-fd-latest.zip \ No newline at end of file From db8a5d3cd9bafe462d098a7e1445669afd93c03d Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 2 May 2024 16:50:06 +0000 Subject: [PATCH 10/26] please lort --- .github/actions/fc/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index 2435071210..56e6b838b9 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -25,7 +25,7 @@ runs: - name: Generate fuzz targets and .fc files run: | - chmod +x bundle-binaries-fc.sh + chmod +x ./contrib/test/bundle-binaries-fc.sh ./contrib/test/bundle-binaries-fc.sh ./build fc-fd-latest.zip shell: bash From 199ef7919b156dbe853ca1bd602e318261c420c9 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 2 May 2024 17:02:11 +0000 Subject: [PATCH 11/26] fc --- .github/actions/fc/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index 56e6b838b9..501bf951ab 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -10,6 +10,9 @@ inputs: runs: using: "composite" steps: + - name: checkout fd + uses: actions/checkout@v4 + - name: Download artifacts from previous action id: get-artifacts uses: actions/download-artifact@v4 From b51231e0e60f6660cf07d97a6c5695feb2f8e076 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 2 May 2024 17:39:34 +0000 Subject: [PATCH 12/26] so close --- .github/actions/fc/action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index 501bf951ab..df2ea76b94 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -12,7 +12,7 @@ runs: steps: - name: checkout fd uses: actions/checkout@v4 - + - name: Download artifacts from previous action id: get-artifacts uses: actions/download-artifact@v4 @@ -28,6 +28,9 @@ runs: - name: Generate fuzz targets and .fc files run: | + ls + ls contrib + ls ./* chmod +x ./contrib/test/bundle-binaries-fc.sh ./contrib/test/bundle-binaries-fc.sh ./build fc-fd-latest.zip shell: bash From bc15bd85a36d080a2ddc75e327c40041dc94cef3 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 2 May 2024 17:48:23 +0000 Subject: [PATCH 13/26] derp --- .github/actions/fc/action.yml | 4 +-- contrib/test/bundle-binaries-fc.sh | 43 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 contrib/test/bundle-binaries-fc.sh diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index df2ea76b94..ed4c398ea4 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -28,9 +28,7 @@ runs: - name: Generate fuzz targets and .fc files run: | - ls - ls contrib - ls ./* + ls contrib/test || ls contrib/* chmod +x ./contrib/test/bundle-binaries-fc.sh ./contrib/test/bundle-binaries-fc.sh ./build fc-fd-latest.zip shell: bash diff --git a/contrib/test/bundle-binaries-fc.sh b/contrib/test/bundle-binaries-fc.sh new file mode 100644 index 0000000000..1ee98d5f32 --- /dev/null +++ b/contrib/test/bundle-binaries-fc.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# This script will take the path to build/ as an argument $1 and create the needed .fc +# files alongside the harnesses, then create the final zip at $2. +set -e + +build_dir="$1" +output_zip="$2" + +# Loop through each architecture, compiler, and fuzz target +for arch in haswell icelake; do + for compiler in clang; do #aflgcc later + for target_dir in "$build_dir/linux/$compiler/$arch/fuzz-test"/*; do + target=$(basename "$target_dir") + + # Create the target directory structure + target_path="targets/$arch/$compiler/$target" + mkdir -p "$target_path" + + # Copy the fuzz target binary + cp "$target_dir/$target" "$target_path/" + + # Create the .fc file + fc_file="$target_path/$target.fc" + cat > "$fc_file" < Date: Thu, 2 May 2024 18:15:44 +0000 Subject: [PATCH 14/26] change back to use FD's CI --- .github/actions/deps/action.yml | 2 +- .github/actions/fc/action.yml | 6 +++--- .github/workflows/clusterfuzz.yml | 13 +++++++------ contrib/test/bundle-binaries-fc.sh | 4 ++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/actions/deps/action.yml b/.github/actions/deps/action.yml index 9a895a68a2..576d094316 100644 --- a/.github/actions/deps/action.yml +++ b/.github/actions/deps/action.yml @@ -24,7 +24,7 @@ runs: - name: apt-get update shell: bash - run: sudo apt-get update && sudo apt-get install -y cmake + run: sudo apt-get update if: env.HAS_APT_GET == '1' - id: deps-sh-hash diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index ed4c398ea4..13cc9e7a7f 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -30,11 +30,11 @@ runs: run: | ls contrib/test || ls contrib/* chmod +x ./contrib/test/bundle-binaries-fc.sh - ./contrib/test/bundle-binaries-fc.sh ./build fc-fd-latest.zip + ./contrib/test/bundle-binaries-fc.sh ./build shell: bash - + # TODO: Change this to FC AWS S3 - name: Upload merged zipfile uses: actions/upload-artifact@v4 with: name: fc-fd-latest.zip - path: fc-fd-latest.zip \ No newline at end of file + path: ./targets \ No newline at end of file diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index df55d2a1fe..1c617079fd 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -21,17 +21,17 @@ jobs: artifact_dir: build/linux/clang/icelake qualifier: highend runs-on: - ubuntu-latest + group: github-v1 env: MACHINE: ${{ matrix.machine }} EXTRAS: fuzz asan ubsan steps: - uses: actions/checkout@v4 - # - uses: ./.github/actions/hugepages + - uses: ./.github/actions/hugepages - uses: ./.github/actions/deps - - run: sudo apt update && sudo apt install -y zip patchelf + - run: sudo apt update && sudo apt install -y zip - uses: asymmetric-research/clusterfuzz-fuzzbot-builder@main name: Build fuzz tests @@ -58,13 +58,13 @@ jobs: FD_DIR=$(find /tmp/ | grep fdfuzz | head -n 1) echo "fd-dir=$FD_DIR" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@v4 - name: Upload harness binaries + name: Upload harness binaries to GHA Artifacts with: name: ${{ github.run_id }}-${{ matrix.machine }} path: ./build/ retention-days: 1 - uses: actions/upload-artifact@v4 - name: Upload .a dependency libraries + name: Upload .a dependency libraries to GHA Artifacts with: name: ${{ github.run_id }}-${{ matrix.machine }}-lib path: ${{ steps.find-fuzzdir.outputs.fd-dir }}/lib/*.a @@ -72,7 +72,8 @@ jobs: fc-publish: needs: clusterfuzz-publish - runs-on: ubuntu-latest + runs-on: + group: github-v1 steps: - uses: actions/checkout@v4 diff --git a/contrib/test/bundle-binaries-fc.sh b/contrib/test/bundle-binaries-fc.sh index 1ee98d5f32..3ef8efa45c 100644 --- a/contrib/test/bundle-binaries-fc.sh +++ b/contrib/test/bundle-binaries-fc.sh @@ -37,7 +37,7 @@ EOL done done -# Create the final zip archive -zip -r "$output_zip" targets/ +# Final Zip Archive creation is handled by the GitHub Action +# zip -r "$output_zip" targets/ echo "Fuzz targets and .fc files generated successfully!" \ No newline at end of file From 65bd9c7e85dd30889fc2be0b6775b91d9b1a8fd5 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 13:53:34 +0000 Subject: [PATCH 15/26] lets rock --- .github/actions/deps/action.yml | 2 +- .github/actions/fc/action.yml | 19 +++++++++++-------- .github/workflows/clusterfuzz.yml | 8 ++++---- contrib/test/bundle-binaries-fc.sh | 6 +++--- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/actions/deps/action.yml b/.github/actions/deps/action.yml index 576d094316..3d662cd790 100644 --- a/.github/actions/deps/action.yml +++ b/.github/actions/deps/action.yml @@ -22,7 +22,7 @@ runs: echo "HAS_APT_GET=0" >> $GITHUB_ENV fi - - name: apt-get update + - name: apt-get update && sudo apt-get install -y cmake shell: bash run: sudo apt-get update if: env.HAS_APT_GET == '1' diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index 13cc9e7a7f..a75359fe2e 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -26,15 +26,18 @@ runs: sudo apt install -y zip unzip shell: bash - - name: Generate fuzz targets and .fc files + - name: Generate fuzz targets and .fc files, md5 zipfile + id: fileops run: | ls contrib/test || ls contrib/* chmod +x ./contrib/test/bundle-binaries-fc.sh - ./contrib/test/bundle-binaries-fc.sh ./build + ./contrib/test/bundle-binaries-fc.sh ./build ./fc-fd-latest.zip + FCBUNDLEMD5=$(md5sum ./fc-fd-latest.zip | cut -d' ' -f1) + echo "bundle-md5=$FCBUNDLEMD5" >> "$GITHUB_OUTPUT" shell: bash - # TODO: Change this to FC AWS S3 - - name: Upload merged zipfile - uses: actions/upload-artifact@v4 - with: - name: fc-fd-latest.zip - path: ./targets \ No newline at end of file + + - name: Upload to FC S3 + run: | + aws s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip + + \ No newline at end of file diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index 1c617079fd..af14f7e5a0 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -21,17 +21,17 @@ jobs: artifact_dir: build/linux/clang/icelake qualifier: highend runs-on: - group: github-v1 + ubuntu-latest env: MACHINE: ${{ matrix.machine }} EXTRAS: fuzz asan ubsan steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/hugepages + # - uses: ./.github/actions/hugepages - uses: ./.github/actions/deps - - run: sudo apt update && sudo apt install -y zip + - run: sudo apt update && sudo apt install -y zip patchelf - uses: asymmetric-research/clusterfuzz-fuzzbot-builder@main name: Build fuzz tests @@ -73,7 +73,7 @@ jobs: fc-publish: needs: clusterfuzz-publish runs-on: - group: github-v1 + ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/contrib/test/bundle-binaries-fc.sh b/contrib/test/bundle-binaries-fc.sh index 3ef8efa45c..7705c7f957 100644 --- a/contrib/test/bundle-binaries-fc.sh +++ b/contrib/test/bundle-binaries-fc.sh @@ -16,7 +16,8 @@ for arch in haswell icelake; do target_path="targets/$arch/$compiler/$target" mkdir -p "$target_path" - # Copy the fuzz target binary + # Chmod and copy the fuzz target binary + chmod +x "$target_dir/$target" cp "$target_dir/$target" "$target_path/" # Create the .fc file @@ -37,7 +38,6 @@ EOL done done -# Final Zip Archive creation is handled by the GitHub Action -# zip -r "$output_zip" targets/ +zip -r "$output_zip" targets/ echo "Fuzz targets and .fc files generated successfully!" \ No newline at end of file From 5b8b9954b0eb46edc8b03f72b73eb17eff6ea864 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 14:03:23 +0000 Subject: [PATCH 16/26] shell: --- .github/actions/fc/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index a75359fe2e..57d5118c44 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -35,9 +35,9 @@ runs: FCBUNDLEMD5=$(md5sum ./fc-fd-latest.zip | cut -d' ' -f1) echo "bundle-md5=$FCBUNDLEMD5" >> "$GITHUB_OUTPUT" shell: bash - + - name: Upload to FC S3 run: | aws s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip - + shell: bash \ No newline at end of file From 7ff3688462dc32c316b2708739f5a29e6427e7ce Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 14:15:35 +0000 Subject: [PATCH 17/26] aws config --- .github/actions/fc/action.yml | 2 +- .github/workflows/clusterfuzz.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index 57d5118c44..7b33fd1791 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -38,6 +38,6 @@ runs: - name: Upload to FC S3 run: | - aws s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip + AWS_ACCESS_KEY_ID=${{ secrets.AWS_AKID }} AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SKID }} s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip shell: bash \ No newline at end of file diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index af14f7e5a0..c0c7aac131 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -70,7 +70,7 @@ jobs: path: ${{ steps.find-fuzzdir.outputs.fd-dir }}/lib/*.a retention-days: 1 - fc-publish: + fuzzcorp-publish: needs: clusterfuzz-publish runs-on: ubuntu-latest From 354f370a395698a951ed9adc6609e447a9866723 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 14:27:23 +0000 Subject: [PATCH 18/26] set env --- .github/actions/fc/action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index 7b33fd1791..5952cb4c15 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -37,7 +37,10 @@ runs: shell: bash - name: Upload to FC S3 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AKID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SKID }} run: | - AWS_ACCESS_KEY_ID=${{ secrets.AWS_AKID }} AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SKID }} s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip + s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip shell: bash \ No newline at end of file From 3b9e3766a854f31b3f761f7e40bf398b26536fc1 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 14:35:05 +0000 Subject: [PATCH 19/26] derp --- .github/actions/fc/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index 5952cb4c15..febdc2d216 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -37,9 +37,9 @@ runs: shell: bash - name: Upload to FC S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AKID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SKID }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AKID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SKID }} run: | s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip shell: bash From d5cfa33d750bfdf98a7a87e7bf8b3a584590af77 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 14:50:13 +0000 Subject: [PATCH 20/26] cant access secrets from composite action --- .github/actions/fc/action.yml | 12 ++++++++++-- .github/workflows/clusterfuzz.yml | 2 ++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index febdc2d216..ebbdca2f31 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -2,6 +2,14 @@ name: Generate Firedancer Fuzz Targets description: Generates Firedancer fuzz targets and .fc files inputs: + SKID: + description: "AWS Secret key id" + required: true + type: string + AKID: + description: "AWS access key" + required: true + type: string previous_action_outputs: description: 'Outputs from the previous action' required: true @@ -38,8 +46,8 @@ runs: - name: Upload to FC S3 env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AKID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SKID }} + AWS_ACCESS_KEY_ID: ${{ inputs.AKID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.SKID }} run: | s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip shell: bash diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index c0c7aac131..8568302cd9 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -79,6 +79,8 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/fc with: + SKID: ${{ secrets.AWS_SKID }} + AKID: ${{ secrets.AWS_AKID }} previous_action_outputs: | ${{ github.run_id }}-linux_clang_haswell ${{ github.run_id }}-linux_clang_icelake From 696708e8d101ddda79c7e830858daba66252ff78 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 14:57:46 +0000 Subject: [PATCH 21/26] cmon bruh --- .github/actions/fc/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index ebbdca2f31..e56a6a646a 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -49,6 +49,6 @@ runs: AWS_ACCESS_KEY_ID: ${{ inputs.AKID }} AWS_SECRET_ACCESS_KEY: ${{ inputs.SKID }} run: | - s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip + aws s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip shell: bash \ No newline at end of file From e167af545f2f7f8619d0aa8cd13421ca9e529248 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 15:21:38 +0000 Subject: [PATCH 22/26] wtf aws --- .github/actions/fc/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index e56a6a646a..f63bb18f63 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -49,6 +49,6 @@ runs: AWS_ACCESS_KEY_ID: ${{ inputs.AKID }} AWS_SECRET_ACCESS_KEY: ${{ inputs.SKID }} run: | - aws s3 cp ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip + aws s3 cp --debug ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip shell: bash \ No newline at end of file From 71214546aeae8289e0bf9243fb10c1e3173c95bb Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 15:32:03 +0000 Subject: [PATCH 23/26] region --- .github/actions/fc/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/fc/action.yml b/.github/actions/fc/action.yml index f63bb18f63..a6bb64e849 100644 --- a/.github/actions/fc/action.yml +++ b/.github/actions/fc/action.yml @@ -48,6 +48,7 @@ runs: env: AWS_ACCESS_KEY_ID: ${{ inputs.AKID }} AWS_SECRET_ACCESS_KEY: ${{ inputs.SKID }} + AWS_DEFAULT_REGION: us-east-2 run: | aws s3 cp --debug ./fc-fd-latest.zip s3://fuzzcorp-bundle-dropbox-975049986498-86c13d2/org_rvfruw8l/prj_rvfruw8lvGQ/${{ steps.fileops.outputs.bundle-md5 }}.zip shell: bash From 202d617a7f2fbf801b691e39a3514d3b4113a491 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 19:00:53 +0000 Subject: [PATCH 24/26] .so upload --- .github/workflows/clusterfuzz.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index 8568302cd9..ba8a2e1862 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -64,10 +64,10 @@ jobs: path: ./build/ retention-days: 1 - uses: actions/upload-artifact@v4 - name: Upload .a dependency libraries to GHA Artifacts + name: Upload .so dependency libraries to GHA Artifacts with: name: ${{ github.run_id }}-${{ matrix.machine }}-lib - path: ${{ steps.find-fuzzdir.outputs.fd-dir }}/lib/*.a + path: ${{ steps.find-fuzzdir.outputs.fd-dir }}/lib/*.so* retention-days: 1 fuzzcorp-publish: From d20762ac81ef0b7aa9f7351c0c777c90209f0475 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 19:13:58 +0000 Subject: [PATCH 25/26] libs --- contrib/test/bundle-binaries-fc.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/contrib/test/bundle-binaries-fc.sh b/contrib/test/bundle-binaries-fc.sh index 7705c7f957..494b14b09e 100644 --- a/contrib/test/bundle-binaries-fc.sh +++ b/contrib/test/bundle-binaries-fc.sh @@ -34,9 +34,15 @@ for arch in haswell icelake; do } } EOL - done - done -done + done # end targs + + LIB_DIR=targets/$arch/$compiler/lib + mkdir -p $LIB_DIR + cp $build_dir/lib/*.so* $LIB_DIR + + done # end compilers + +done # end arch zip -r "$output_zip" targets/ From 0a99c8efa5623a54d19fb7827b1552a2587ddb8e Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Fri, 3 May 2024 19:38:29 +0000 Subject: [PATCH 26/26] okie --- .github/workflows/clusterfuzz.yml | 17 ++++++----------- contrib/test/bundle-binaries-fc.sh | 3 ++- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/clusterfuzz.yml b/.github/workflows/clusterfuzz.yml index ba8a2e1862..b95d969c91 100644 --- a/.github/workflows/clusterfuzz.yml +++ b/.github/workflows/clusterfuzz.yml @@ -52,23 +52,19 @@ jobs: qualifier: ${{ matrix.qualifier }} service-account-credentials: ${{ secrets.FUZZ_SERVICE_ACCT_JSON_BUNDLE }} - - name: Find the tmpdir containing our binaries and rewritten libs + - name: Find the tmpdir containing our binaries and rewritten-rpath libs id: find-fuzzdir run: | FD_DIR=$(find /tmp/ | grep fdfuzz | head -n 1) - echo "fd-dir=$FD_DIR" >> "$GITHUB_OUTPUT" + mkdir -p ./build/lib + cp $FD_DIR/lib/*.so* ./build/lib/ - uses: actions/upload-artifact@v4 - name: Upload harness binaries to GHA Artifacts + name: Upload harness binaries and libs to GHA Artifacts with: name: ${{ github.run_id }}-${{ matrix.machine }} path: ./build/ retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload .so dependency libraries to GHA Artifacts - with: - name: ${{ github.run_id }}-${{ matrix.machine }}-lib - path: ${{ steps.find-fuzzdir.outputs.fd-dir }}/lib/*.so* - retention-days: 1 + fuzzcorp-publish: needs: clusterfuzz-publish @@ -84,5 +80,4 @@ jobs: previous_action_outputs: | ${{ github.run_id }}-linux_clang_haswell ${{ github.run_id }}-linux_clang_icelake - ${{ github.run_id }}-linux_clang_haswell-lib - ${{ github.run_id }}-linux_clang_icelake-lib + diff --git a/contrib/test/bundle-binaries-fc.sh b/contrib/test/bundle-binaries-fc.sh index 494b14b09e..607a8ece09 100644 --- a/contrib/test/bundle-binaries-fc.sh +++ b/contrib/test/bundle-binaries-fc.sh @@ -38,8 +38,9 @@ EOL LIB_DIR=targets/$arch/$compiler/lib mkdir -p $LIB_DIR + ls $build_dir/*lib || ls $build_dir cp $build_dir/lib/*.so* $LIB_DIR - + done # end compilers done # end arch