go-git: Fix build issue and auto-discover targets

Main changes are:
- Dynamically find and build all fuzzing targets.
- Fix current build issues.
- Cache go dependencies when cloning repositories.
- Clean up test files by removing structs, global values and comments which were leading to some errors.
- Expand scope to go-git/go-billy.

The logic for dynamic finding and building fuzzing
targets is largely the same as for the fluxcd project.

Signed-off-by: Paulo Gomes <pjbgf@linux.com>

projects/go-git/Dockerfile

@@ -15,7 +15,26 @@
 ################################################################################
 
 FROM gcr.io/oss-fuzz-base/base-builder-go
-RUN git clone --depth 1 https://github.com/go-git/go-git.git
-RUN go install github.com/AdamKorcz/go-118-fuzz-build@latest
-RUN cp $SRC/go-git/oss-fuzz.sh $SRC/build.sh
-WORKDIR $SRC/go-git
+
+ENV GOPATH="${GOPATH:-/root/go}"
+ENV ORG_ROOT="${GOPATH}/src/github.com/go-git"
+
+RUN mkdir -p "${ORG_ROOT}"
+
+# The fuzzed components are scattered around multiple repositories.
+# Here we clone all of them and cache their go dependencies.
+# The build process happens as build.sh iterate over each one of them.
+ARG REPOSITORIES="go-git go-billy"
+RUN for repo in ${REPOSITORIES}; do \
+    git clone --depth 1 "https://github.com/go-git/${repo}" "${ORG_ROOT}/${repo}"; \
+    cd "${ORG_ROOT}/${repo}"; \
+    go mod download; \
+    cd -; \
+    done
+
+# Install go imports as the import section needs to reflect some of the changes
+# made by build.sh.
+RUN go install golang.org/x/tools/cmd/goimports@latest
+
+COPY build.sh $SRC/
+WORKDIR $SRC

projects/go-git/build.sh

@@ -0,0 +1,161 @@
+#!/bin/bash -eu
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# This code improves the use of Go Native by:
+# - Dynamically discovering and building all fuzz tests within the project root path.
+# - Supporting single (during PR checks) or multiple repositories (oss-fuzz).
+# - Enabling execution via CI builds and Makefile targets for each repo.
+
+GOPATH="${GOPATH:-/root/go}"
+ORG_ROOT="${ORG_ROOT:-${GOPATH}/src/github.com/go-git}"
+PREBUILD_SCRIPT_PATH="${PREBUILD_SCRIPT_PATH:-tests/fuzz/oss_fuzz_prebuild.sh}"
+POSTBUILD_SCRIPT_PATH="${POSTBUILD_SCRIPT_PATH:-tests/fuzz/oss_fuzz_postbuild.sh}"
+
+# source_prebuild_script sources the prebuild script, which executes project-specific
+# code and exposes environment variables that are needed during the generic build process.
+#
+# Examples of usage may be organising directory structure for embedding
+# files, downloading artifacts or setting environment variables.
+function source_prebuild_script(){
+	if [ -f "${PREBUILD_SCRIPT_PATH}" ]; then
+		# shellcheck source=/dev/null
+		. "${PREBUILD_SCRIPT_PATH}"
+	fi
+}
+
+# source_postbuild_script sources the postbuild script, which executes project-specific
+# code and unset environment variables that may break follow-up processes.
+function source_postbuild_script(){
+	if [ -f "${POSTBUILD_SCRIPT_PATH}" ]; then
+		# shellcheck source=/dev/null
+		. "${POSTBUILD_SCRIPT_PATH}"
+	fi
+}
+
+# go_native_build_all_fuzzers builds all Go Native fuzz tests defined in modules within
+# the given project dir.
+#
+# Args:
+# 	project_dir
+function go_native_build_all_fuzzers(){
+	local project_path="$1"
+
+	cd "${project_path}"
+
+	source_prebuild_script
+
+	modules=$(find . -mindepth 1 -maxdepth 4 -type f -name 'go.mod' | cut -c 3- | sed 's|/[^/]*$$||' | sort -u | sed 's;/go.mod;;g' | sed 's;go.mod;.;g')
+	for module in ${modules}; do
+		cd "${project_path}/${module}"
+
+		local test_files
+		test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' . || echo "")
+		if [ -z "${test_files}" ]; then
+			continue
+		fi
+
+		# go-118-fuzz-build is required for each module.
+		go get -u github.com/AdamKorcz/go-118-fuzz-build/testing
+
+		# The go get command above can affect transient dependencies, may lead
+		# to the go.sym to become out of sync, which would cause build to break.
+		# go mod tidy will only work if the current module has a reference
+		# to the above dependency, so we create one.
+		local pkgName
+		pkgName="$(grep -h '^package ' -- *.go | head -n 1)"
+		if [ -z "${test_files}" ]; then
+			pkgName="package fuzz"
+		fi
+
+		cat <<EOF > dep-placeholder.go
+${pkgName}
+
+import _ "github.com/AdamKorcz/go-118-fuzz-build/testing"
+EOF
+		# With the reference above, this updates go.sum.
+		go mod tidy
+
+		# Iterate through all Go Fuzz targets, compiling each into a fuzzer.
+		for file in ${test_files}; do
+			# If the subdir is a module, skip this file, as it will be handled
+			# at the next iteration of the outer loop. 
+			if [ -f "$(dirname "${file}")/go.mod" ]; then
+				continue
+			fi
+
+			# Remove all funcs that are not Fuzz tests. Some were resulting
+			# in errors due to referring to other test files.
+			#
+			# Suite based tests funcs were also failing with the error:
+			#   cannot use t (variable of type ...) as *"testing".T value in argument to suite.Run
+			#
+			# As those are not needed for fuzzing purposes they are removed.
+			sed -i '/^func Fuzz/!{/^func /,/^}/d}' "${file}"
+			# Remove global declarations and any comments.
+			sed -i '/^\(var\|const\)\b/d; /^\s*\/\//d; /^\s*\/\*/,/\*\//d' "${file}"
+			# Remove structs which may be referring to subtypes from other test files.
+			sed -i '/^type .* struct {/,/^}/d' "${file}"
+
+			# Ensure the file is still properly formatted.
+			go fmt "${file}"
+			goimports -w "${file}"
+
+			targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}")
+			for target_name in ${targets}; do
+				local module_name
+				local fuzzer_name
+				local target_dir
+
+				# Transform module path into module name (e.g. git/libgit2 to git_libgit2).
+				module_name="${module/\//_}_"
+				# If module equal '._', use empty string instead.
+				module_name="${module/#%._}"
+
+				# Compose fuzzer name based on the lowercase version of the func names.
+				fuzzer_name="${target_name,,}"
+				# The module name is added after the fuzz prefix, for better discoverability.
+				fuzzer_name="${target_name/fuzz_/fuzz_${module_name}}"
+				target_dir=$(dirname "${file}")
+
+				echo "Building ${file}.${target_name} into ${fuzzer_name}"
+				cat "${file}"
+				compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}"
+			done
+		done
+	done
+}
+
+function loop_through_org_repositories(){
+	local repos=""
+	repos="$(find "${ORG_ROOT}" -type d -mindepth 1 -maxdepth 1)"
+	for repo in ${repos}; do
+		go_native_build_all_fuzzers "${repo}"
+	done
+}
+
+function main(){
+	if grep -h '^module github.com/go-git/' "${SRC}/go.mod"; then
+		echo "Building Go Native fuzzers for ${SRC}"
+		go_native_build_all_fuzzers "${SRC}"
+		exit $?
+	fi
+
+	echo "Going through all repositories in ${ORG_ROOT}"
+	loop_through_org_repositories 
+}
+
+main