build: Improve fuzz tests' reliability

controllers/controllers_fuzzer_test.go

@@ -1,5 +1,5 @@
-//go:build gofuzz
-// +build gofuzz
+//go:build gofuzz_libfuzzer
+// +build gofuzz_libfuzzer
 
 /*
 Copyright 2021 The Flux authors

tests/fuzz/README.md

@@ -6,8 +6,6 @@ open source projects.
 The long running fuzzing execution is configured in the [oss-fuzz repository].
 Shorter executions are done on a per-PR basis, configured as a [github workflow].
 
-For fuzzers to be called, they must be compiled within [oss_fuzz_build.sh](./oss_fuzz_build.sh).
-
 ### Testing locally
 
 Build fuzzers:
@@ -19,12 +17,12 @@ All fuzzers will be built into `./build/fuzz/out`.
 
 Smoke test fuzzers:
 
+All the fuzzers will be built and executed once, to ensure they are fully functional.
+
 ```bash
 make fuzz-smoketest
 ```
 
-The smoke test runs each fuzzer once to ensure they are fully functional.
-
 Run fuzzer locally:
 ```bash
 ./build/fuzz/out/fuzz_conditions_match
@@ -39,7 +37,46 @@ Run fuzzer inside a container:
 		/out/fuzz_conditions_match
 ```
 
+### Caveats of creating oss-fuzz compatible tests
+
+#### Segregate fuzz tests
+
+OSS-Fuzz does not properly support mixed `*_test.go` files, in which there is a combination
+of fuzz and non-fuzz tests. To mitigate this problem, ensure your fuzz tests are not in the
+same file as other Go tests. As a pattern, call your fuzz test files `*_fuzz_test.go`.
+
+#### Build tags to avoid conflicts when running Go tests
+
+Due to the issue above, code duplication will occur when creating fuzz tests that rely on
+helper functions that are shared with other tests. To avoid build issues, add a conditional
+build tag at the top of the `*_fuzz_test.go` file:
+```go
+//go:build gofuzz_libfuzzer
+// +build gofuzz_libfuzzer
+```
+
+The build tag above is set at [go-118-fuzz-build].
+At this point in time we can't pass on specific tags from [compile_native_go_fuzzer].
+
+### Running oss-fuzz locally
+
+The `make fuzz-smoketest` is meant to be an easy way to reproduce errors that may occur
+upstream. If our checks ever run out of sync with upstream, the upstream tests can be
+executed locally with:
+
+```
+git clone --depth 1 https://github.com/google/oss-fuzz
+cd oss-fuzz
+python infra/helper.py build_image fluxcd
+python infra/helper.py build_fuzzers --sanitizer address --architecture x86_64 fluxcd
+python infra/helper.py check_build --sanitizer address --architecture x86_64 fluxcd
+```
+
+For latest info on testing oss-fuzz locally, refer to the [upstream guide].
 
 [oss fuzz]: https://github.com/google/oss-fuzz
 [oss-fuzz repository]: https://github.com/google/oss-fuzz/tree/master/projects/fluxcd
 [github workflow]: .github/workflows/cifuzz.yaml
+[upstream guide]: https://google.github.io/oss-fuzz/getting-started/new-project-guide/#testing-locally
+[go-118-fuzz-build]: https://github.com/AdamKorcz/go-118-fuzz-build/blob/b2031950a318d4f2dcf3ec3e128f904d5cf84623/main.go#L40
+[compile_native_go_fuzzer]: https://github.com/google/oss-fuzz/blob/c2d827cb78529fdc757c9b0b4fea0f1238a54814/infra/base-images/base-builder/compile_native_go_fuzzer#L32

tests/fuzz/oss_fuzz_build.sh

@@ -16,69 +16,65 @@
 
 set -euxo pipefail
 
+# This file aims for:
+# - Dynamically discover and build all fuzz tests within the repository.
+# - Work for both local make fuzz-smoketest and the upstream oss-fuzz.
+
 GOPATH="${GOPATH:-/root/go}"
 GO_SRC="${GOPATH}/src"
 PROJECT_PATH="github.com/fluxcd/image-reflector-controller"
-TMP_DIR=$(mktemp -d /tmp/oss_fuzz-XXXXXX)
-
-cleanup(){
-	rm -rf "${TMP_DIR}"
-}
-trap cleanup EXIT
 
+# install_deps installs all dependencies needed for upstream oss-fuzz.
+# Unfortunately we can't pin versions here, as we want to always
+# have the latest, so that we can reproduce errors occuring upstream.
 install_deps(){
-	if ! command -v go-118-fuzz-build &> /dev/null || ! command -v addimport &> /dev/null; then
-		mkdir -p "${TMP_DIR}/go-118-fuzz-build"
-
-		git clone https://github.com/AdamKorcz/go-118-fuzz-build "${TMP_DIR}/go-118-fuzz-build"
-		cd "${TMP_DIR}/go-118-fuzz-build"
-		go build -o "${GOPATH}/bin/go-118-fuzz-build"
-
-		cd addimport
-		go build -o "${GOPATH}/bin/addimport"
-	fi
-
-	if ! command -v goimports &> /dev/null; then
-		go install golang.org/x/tools/cmd/goimports@latest
+	if ! command -v go-118-fuzz-build &> /dev/null; then
+		go install github.com/AdamKorcz/go-118-fuzz-build@latest
 	fi
 }
 
-# Removes the content of test funcs which could cause the Fuzz
-# tests to break.
-remove_test_funcs(){
-	filename=$1
-
-	echo "removing co-located *testing.T"
-	sed -i -e '/func Test.*testing.T) {$/ {:r;/\n}/!{N;br}; s/\n.*\n/\n/}' "${filename}"
-	# Remove gomega reference as it is not used by Fuzz tests.
-	sed -i 's;. "github.com/onsi/gomega";;g' "${filename}"
-
-	# After removing the body of the go testing funcs, consolidate the imports.
-	goimports -w "${filename}"
-}
-
 install_deps
 
 cd "${GO_SRC}/${PROJECT_PATH}"
 
-go get github.com/AdamKorcz/go-118-fuzz-build/utils
+# Ensure any project-specific requirements are catered for ahead of
+# the generic build process.
+if [ -f "tests/fuzz/oss_fuzz_prebuild.sh" ]; then
+	. tests/fuzz/oss_fuzz_prebuild.sh
+fi
+
+modules=$(find . -mindepth 1 -maxdepth 4 -type f -name 'go.mod' | cut -c 3- | sed 's|/[^/]*$$||' | sort -u | sed 's;/go.mod;;g' | sed 's;go.mod;.;g')
 
-mkdir -p controllers/testdata/crd
-cp config/crd/bases/*.yaml controllers/testdata/crd
+for module in ${modules}; do
 
-# Iterate through all Go Fuzz targets, compiling each into a fuzzer.
-test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' .)
-for file in ${test_files}
-do
-	remove_test_funcs "${file}"
+	cd "${GO_SRC}/${PROJECT_PATH}/${module}"
 
-	targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}")
-	for target_name in ${targets}
-	do
-		fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]')
-		target_dir=$(dirname "${file}")
+	test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' . || echo "")
+	if [ -z "${test_files}" ]; then
+		continue
+	fi
 
-		echo "Building ${file}.${target_name} into ${fuzzer_name}"
-		compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}"
+	go get github.com/AdamKorcz/go-118-fuzz-build/testing
+
+	# Iterate through all Go Fuzz targets, compiling each into a fuzzer.
+	for file in ${test_files}; do
+		# If the subdir is a module, skip this file, as it will be handled
+		# at the next iteration of the outer loop. 
+		if [ -f "$(dirname "${file}")/go.mod" ]; then
+			continue
+		fi
+
+		targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}")
+		for target_name in ${targets}; do
+			# Transform module path into module name (e.g. git/libgit2 to git_libgit2).
+			module_name=$(echo ${module} | tr / _)
+			# Compose fuzzer name based on the lowercase version of the func names.
+			# The module name is added after the fuzz prefix, for better discoverability.
+			fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]' | sed "s;fuzz_;fuzz_${module_name}_;g")
+			target_dir=$(dirname "${file}")
+
+			echo "Building ${file}.${target_name} into ${fuzzer_name}"
+			compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}"
+		done
 	done
 done

tests/fuzz/oss_fuzz_prebuild.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Flux authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euxo pipefail
+
+# This file is executed by upstream oss-fuzz for any requirements that
+# are specific for building this project.
+
+# Some tests requires embedded resources. Embedding does not allow
+# for traversing into ascending dirs, therefore we copy those contents here:
+mkdir -p controllers/testdata/crd
+cp config/crd/bases/*.yaml controllers/testdata/crd