From ee1cb15cdc930548f24dbefbbef1be464a798c94 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 2 Sep 2022 15:38:16 +0100 Subject: [PATCH] fuzz: Fix upstream build The upstream build is failing with: /usr/local/bin/compile_native_go_fuzzer: line 35: addimport: command not found The changes uses the upstream approach to install the missing commands. Note that one of the reasons of the unilateral failure upstream is due to the base image changed in: https://github.com/google/oss-fuzz/commit/e71e320709b7249348a2115a18481dd623b27a40 Signed-off-by: Paulo Gomes --- tests/fuzz/Dockerfile.builder | 5 -- tests/fuzz/compile_native_go_fuzzer.sh | 102 ------------------------- tests/fuzz/oss_fuzz_build.sh | 46 ++++++++++- 3 files changed, 42 insertions(+), 111 deletions(-) delete mode 100755 tests/fuzz/compile_native_go_fuzzer.sh diff --git a/tests/fuzz/Dockerfile.builder b/tests/fuzz/Dockerfile.builder index 43acfe5c5..0e8cbaf3e 100644 --- a/tests/fuzz/Dockerfile.builder +++ b/tests/fuzz/Dockerfile.builder @@ -3,9 +3,4 @@ FROM gcr.io/oss-fuzz-base/base-builder-go COPY ./ $GOPATH/src/github.com/fluxcd/helm-controller/ COPY ./tests/fuzz/oss_fuzz_build.sh $SRC/build.sh -# Temporarily overrides compile_native_go_fuzzer. -# Pending upstream merge: https://github.com/google/oss-fuzz/pull/8285 -COPY tests/fuzz/compile_native_go_fuzzer.sh /usr/local/bin/compile_native_go_fuzzer -RUN go install golang.org/x/tools/cmd/goimports@latest - WORKDIR $SRC diff --git a/tests/fuzz/compile_native_go_fuzzer.sh b/tests/fuzz/compile_native_go_fuzzer.sh deleted file mode 100755 index 8983d9dfd..000000000 --- a/tests/fuzz/compile_native_go_fuzzer.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -eu -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -# Rewrites a copy of the fuzzer to allow for -# libFuzzer instrumentation. -function rewrite_go_fuzz_harness() { - fuzzer_filename=$1 - fuzz_function=$2 - - # Create a copy of the fuzzer to not modify the existing fuzzer. - cp $fuzzer_filename "${fuzzer_filename}"_fuzz_.go - mv $fuzzer_filename /tmp/ - fuzzer_fn="${fuzzer_filename}"_fuzz_.go - - # Remove the body of go testing funcs that may be co-located. - echo "removing *testing.T" - sed -i -e '/testing.T) {$/ {:r;/\n}/!{N;br}; s/\n.*\n/\n/}' "${fuzzer_fn}" - # After removing the body of the go testing funcs, consolidate the imports. - if command -v goimports; then - goimports -w "${fuzzer_fn}" - fi - - # Replace *testing.F with *go118fuzzbuildutils.F. - echo "replacing *testing.F" - sed -i "s/func $fuzz_function(\([a-zA-Z0-9]*\) \*testing\.F)/func $fuzz_function(\1 \*go118fuzzbuildutils\.F)/g" "${fuzzer_fn}" - - # Import https://github.com/AdamKorcz/go-118-fuzz-build. - # This changes the line numbers from the original fuzzer. - addimport -path "${fuzzer_fn}" -} - -function build_native_go_fuzzer() { - fuzzer=$1 - function=$2 - path=$3 - tags="-tags gofuzz" - - if [[ $SANITIZER = *coverage* ]]; then - echo "here we perform coverage build" - fuzzed_package=`go list $tags -f '{{.Name}}' $path` - abspath=`go list $tags -f {{.Dir}} $path` - cd $abspath - cp $GOPATH/native_ossfuzz_coverage_runner.go ./"${function,,}"_test.go - sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go - sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go - sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go - - # The repo is the module path/name, which is already created above - # in case it doesn't exist, but not always the same as the module - # path. This is necessary to handle SIV properly. - fuzzed_repo=$(go list $tags -f {{.Module}} "$path") - abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo` - # give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir - echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath - gotip test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path - - rm ./"${function,,}"_test.go - else - go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir - $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer - fi -} - - -path=$1 -function=$2 -fuzzer=$3 -tags="-tags gofuzz" - -# Get absolute path. -abs_file_dir=$(go list $tags -f {{.Dir}} $path) - -# TODO(adamkorcz): Get rid of "-r" flag here. -fuzzer_filename=$(grep -r -l --include='**.go' -s "$function" "${abs_file_dir}") - -# Test if file contains a line with "func $function" and "testing.F". -if [ $(grep -r "func $function" $fuzzer_filename | grep "testing.F" | wc -l) -eq 1 ] -then - - rewrite_go_fuzz_harness $fuzzer_filename $function - build_native_go_fuzzer $fuzzer $function $abs_file_dir - - # Clean up. - rm "${fuzzer_filename}_fuzz_.go" - mv /tmp/$(basename $fuzzer_filename) $fuzzer_filename -else - echo "Could not find the function: func ${function}(f *testing.F)" -fi diff --git a/tests/fuzz/oss_fuzz_build.sh b/tests/fuzz/oss_fuzz_build.sh index 9e4a36f2f..1b8351934 100755 --- a/tests/fuzz/oss_fuzz_build.sh +++ b/tests/fuzz/oss_fuzz_build.sh @@ -19,23 +19,61 @@ set -euxo pipefail GOPATH="${GOPATH:-/root/go}" GO_SRC="${GOPATH}/src" PROJECT_PATH="github.com/fluxcd/helm-controller" +TMP_DIR=$(mktemp -d /tmp/oss_fuzz-XXXXXX) + +cleanup(){ + rm -rf "${TMP_DIR}" +} +trap cleanup EXIT + +install_deps(){ + if ! command -v go-118-fuzz-build &> /dev/null || ! command -v addimport &> /dev/null; then + mkdir -p "${TMP_DIR}/go-118-fuzz-build" + + git clone https://github.com/AdamKorcz/go-118-fuzz-build "${TMP_DIR}/go-118-fuzz-build" + cd "${TMP_DIR}/go-118-fuzz-build" + go build -o "${GOPATH}/bin/go-118-fuzz-build" + + cd addimport + go build -o "${GOPATH}/bin/addimport" + fi + + if ! command -v goimports &> /dev/null; then + go install golang.org/x/tools/cmd/goimports@latest + fi +} + +# Removes the content of test funcs which could cause the Fuzz +# tests to break. +remove_test_funcs(){ + filename=$1 + + echo "removing co-located *testing.T" + sed -i -e '/func Test.*testing.T) {$/ {:r;/\n}/!{N;br}; s/\n.*\n/\n/}' "${filename}" + + # After removing the body of the go testing funcs, consolidate the imports. + goimports -w "${filename}" +} + +install_deps cd "${GO_SRC}/${PROJECT_PATH}" -go install github.com/AdamKorcz/go-118-fuzz-build@latest go get github.com/AdamKorcz/go-118-fuzz-build/utils # Iterate through all Go Fuzz targets, compiling each into a fuzzer. test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' .) for file in ${test_files} do + remove_test_funcs "${file}" + targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}") for target_name in ${targets} do - fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]') - target_dir=$(dirname "${file}") + fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]') + target_dir=$(dirname "${file}") echo "Building ${file}.${target_name} into ${fuzzer_name}" - compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}" fuzz + compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}" done done