fluxcd: Add auto discovery for Go Native fuzz targets (#9064)

Add support for auto discovery and build for all Go Native fuzz targets
that exist within the range of Flux org repos defined.

Support for faster CI checks by only targeting the specific project when
running against PRs.

(cc: @stefanprodan @hiddeco)

---
Would there be interest on having `loop_through_org_repositories` as
part of infra?
This would enable Go Native projects to have auto discovery of fuzzers
without the need of calling `compile_native_go_fuzzer` for each fuzzer
they add into their repos.

Signed-off-by: Paulo Gomes <pjbgf@linux.com>

projects/fluxcd/Dockerfile

@@ -16,14 +16,20 @@
 
 FROM gcr.io/oss-fuzz-base/base-builder-go
 
-ENV PROJECT_ROOT="${GOPATH:-/root/go}/src/github.com/fluxcd"
+# cmake and pkg-config are only needed whilst libgit2 is in use.
+# The project aims to deprecate libgit2 at some point in Q1 2023,
+# by which point this can be removed.
+RUN apt-get update && apt-get install -y cmake pkg-config
 
-RUN mkdir -p "${PROJECT_ROOT}"
+ENV GOPATH="${GOPATH:-/root/go}"
+ENV ORG_ROOT="${ORG_ROOT:-${GOPATH}/src/github.com/fluxcd}"
+
+RUN mkdir -p "${ORG_ROOT}"
 
 # Flux has its components scattered around multiple repositories due to its architecture. 
 # Here we clone all of them. The build process happens as build.sh iterate over each one of them.
 ARG REPOSITORIES="pkg notification-controller kustomize-controller helm-controller image-reflector-controller source-controller image-automation-controller"
-RUN for repo in ${REPOSITORIES}; do git clone --depth 1 "https://github.com/fluxcd/${repo}" "${PROJECT_ROOT}/${repo}"; done
+RUN for repo in ${REPOSITORIES}; do git clone --depth 1 "https://github.com/fluxcd/${repo}" "${ORG_ROOT}/${repo}"; done
 
 COPY build.sh $SRC/
 WORKDIR $SRC

projects/fluxcd/build.sh

@@ -15,7 +15,111 @@
 #
 ################################################################################
 
-PROJECT_ROOT="${GOPATH:-/root/go}/src/github.com/fluxcd"
+# This code improves the use of Go Native by:
+# - Dynamically discovering and building all fuzz tests within the project root path.
+# - Supporting single (during PR checks) or multiple repositories (oss-fuzz).
+# - Enabling execution via CI builds and Makefile targets for each repo.
 
-# Build the fuzzers for all cloned sub-projects 
-find "${PROJECT_ROOT}/" -type f -name oss_fuzz_build.sh | bash -e
+GOPATH="${GOPATH:-/root/go}"
+ORG_ROOT="${ORG_ROOT:-${GOPATH}/src/github.com/fluxcd}"
+PREBUILD_SCRIPT_PATH="${PREBUILD_SCRIPT_PATH:-tests/fuzz/oss_fuzz_prebuild.sh}"
+POSTBUILD_SCRIPT_PATH="${POSTBUILD_SCRIPT_PATH:-tests/fuzz/oss_fuzz_postbuild.sh}"
+FLUX_CI="${FLUX_CI:-false}"
+
+# source_prebuild_script sources the prebuild script, which executes project-specific
+# code and exposes environment variables that are needed during the generic build process.
+#
+# Examples of usage may be organising directory structure for embedding
+# files, downloading artifacts or setting environment variables.
+function source_prebuild_script(){
+	if [ -f "${PREBUILD_SCRIPT_PATH}" ]; then
+		# shellcheck source=/dev/null
+		. "${PREBUILD_SCRIPT_PATH}"
+	fi
+}
+
+# source_postbuild_script sources the postbuild script, which executes project-specific
+# code and unset environment variables that may break follow-up processes.
+function source_postbuild_script(){
+	if [ -f "${POSTBUILD_SCRIPT_PATH}" ]; then
+		# shellcheck source=/dev/null
+		. "${POSTBUILD_SCRIPT_PATH}"
+	fi
+}
+
+# go_native_build_all_fuzzers builds all Go Native fuzz tests defined in modules within
+# the given project dir.
+#
+# Args:
+# 	project_dir
+function go_native_build_all_fuzzers(){
+	local project_path="$1"
+
+	cd "${project_path}"
+
+	source_prebuild_script
+
+	modules=$(find . -mindepth 1 -maxdepth 4 -type f -name 'go.mod' | cut -c 3- | sed 's|/[^/]*$$||' | sort -u | sed 's;/go.mod;;g' | sed 's;go.mod;.;g')
+	for module in ${modules}; do
+		cd "${project_path}/${module}"
+
+		local test_files
+		test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' . || echo "")
+		if [ -z "${test_files}" ]; then
+			continue
+		fi
+
+		# go-118-fuzz-build is required for each module.
+		go get github.com/AdamKorcz/go-118-fuzz-build/testing
+
+		# Iterate through all Go Fuzz targets, compiling each into a fuzzer.
+		for file in ${test_files}; do
+			# If the subdir is a module, skip this file, as it will be handled
+			# at the next iteration of the outer loop. 
+			if [ -f "$(dirname "${file}")/go.mod" ]; then
+				continue
+			fi
+
+			targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}")
+			for target_name in ${targets}; do
+				local module_name
+				local fuzzer_name
+				local target_dir
+
+				# Transform module path into module name (e.g. git/libgit2 to git_libgit2).
+				module_name="${module/\//_}_"
+				# If module equal '._', use empty string instead.
+				module_name="${module/#%._}"
+
+				# Compose fuzzer name based on the lowercase version of the func names.
+				fuzzer_name="${target_name,,}"
+				# The module name is added after the fuzz prefix, for better discoverability.
+				fuzzer_name="${target_name/fuzz_/fuzz_${module_name}}"
+				target_dir=$(dirname "${file}")
+
+				echo "Building ${file}.${target_name} into ${fuzzer_name}"
+				compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}"
+			done
+		done
+	done
+}
+
+function loop_through_org_repositories(){
+	local repos=""
+	repos="$(find "${ORG_ROOT}" -type d -mindepth 1 -maxdepth 1)"
+	for repo in ${repos}; do
+		go_native_build_all_fuzzers "${repo}"
+	done
+}
+
+function main(){
+	if [[ "${FLUX_CI}" == "true" ]]; then
+		echo "Building Go Native fuzzers for Flux CI"
+		go_native_build_all_fuzzers "${SRC}"
+	else
+		echo "Going through all repositories in ${ORG_ROOT}"
+		loop_through_org_repositories 
+	fi
+}
+
+main