From d62e27e5fd344f90ef95c593feba849ecaab28cf Mon Sep 17 00:00:00 2001 From: Maciej Mionskowski Date: Thu, 23 Nov 2023 17:20:29 +0100 Subject: [PATCH 1/9] Initial Commit --- .github/workflows/lint.yml | 22 ++ .github/workflows/release.yml | 29 +++ .github/workflows/test-github-action.yml | 50 +++++ .github/workflows/test.yml | 16 ++ .gitignore | 4 + .goreleaser.yaml | 44 ++++ LICENSE | 201 ++++++++++++++++++ README.md | 125 ++++++++++- ci/github-actions/fuzz/action.yml | 95 +++++++++ cmd/fuzz.go | 138 ++++++++++++ cmd/root.go | 34 +++ fuzz/copy.go | 61 ++++++ fuzz/corpus.go | 90 ++++++++ fuzz/corpus_test.go | 122 +++++++++++ fuzz/fuzz.go | 116 ++++++++++ fuzz/fuzz_test.go | 73 +++++++ fuzz/project.go | 6 + fuzz/targets.go | 140 ++++++++++++ fuzz/targets_test.go | 69 ++++++ fuzz/testdata/corpus/multiple/go.mod | 3 + fuzz/testdata/corpus/multiple/main_test.go | 12 ++ .../corpus/multiple/nocorpus/main_test.go | 13 ++ .../testdata/corpus/multiple/sub/main_test.go | 12 ++ ...93d0189a07e81603fbdf64f2ca44738aa27159acef | 2 + ...93d0189a07e81603fbdf64f2ca44738aa27159acef | 2 + ...93d0189a07e81603fbdf64f2ca44738aa27159acef | 2 + fuzz/testdata/discover/go.mod | 3 + fuzz/testdata/discover/main_test.go | 12 ++ fuzz/testdata/discover/submain/main_test.go | 13 ++ .../testdata/discover/subpackage/main_test.go | 13 ++ ...93d0189a07e81603fbdf64f2ca44738aa27159acef | 2 + fuzz/testdata/discovermain/go.mod | 3 + fuzz/testdata/discovermain/main_test.go | 19 ++ fuzz/testdata/fuzzing/new/.gitignore | 1 + fuzz/testdata/fuzzing/new/go.mod | 3 + fuzz/testdata/fuzzing/new/main_test.go | 12 ++ fuzz/testdata/fuzzing/nofindings/.gitignore | 1 + fuzz/testdata/fuzzing/nofindings/go.mod | 3 + fuzz/testdata/fuzzing/nofindings/main_test.go | 9 + fuzz/testdata/fuzzing/seed/go.mod | 3 + fuzz/testdata/fuzzing/seed/main_test.go | 12 ++ fuzz/testdata/fuzzing/seedfile/go.mod | 3 + fuzz/testdata/fuzzing/seedfile/main_test.go | 12 ++ ...93d0189a07e81603fbdf64f2ca44738aa27159acef | 2 + go.mod | 16 ++ go.sum | 18 ++ main.go | 7 + 47 files changed, 1646 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test-github-action.yml create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 .goreleaser.yaml create mode 100644 LICENSE create mode 100644 ci/github-actions/fuzz/action.yml create mode 100644 cmd/fuzz.go create mode 100644 cmd/root.go create mode 100644 fuzz/copy.go create mode 100644 fuzz/corpus.go create mode 100644 fuzz/corpus_test.go create mode 100644 fuzz/fuzz.go create mode 100644 fuzz/fuzz_test.go create mode 100644 fuzz/project.go create mode 100644 fuzz/targets.go create mode 100644 fuzz/targets_test.go create mode 100644 fuzz/testdata/corpus/multiple/go.mod create mode 100644 fuzz/testdata/corpus/multiple/main_test.go create mode 100644 fuzz/testdata/corpus/multiple/nocorpus/main_test.go create mode 100644 fuzz/testdata/corpus/multiple/sub/main_test.go create mode 100644 fuzz/testdata/corpus/multiple/sub/testdata/fuzz/FuzzNonExistingTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef create mode 100644 fuzz/testdata/corpus/multiple/sub/testdata/fuzz/FuzzSubTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef create mode 100644 fuzz/testdata/corpus/multiple/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef create mode 100644 fuzz/testdata/discover/go.mod create mode 100644 fuzz/testdata/discover/main_test.go create mode 100644 fuzz/testdata/discover/submain/main_test.go create mode 100644 fuzz/testdata/discover/subpackage/main_test.go create mode 100644 fuzz/testdata/discover/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef create mode 100644 fuzz/testdata/discovermain/go.mod create mode 100644 fuzz/testdata/discovermain/main_test.go create mode 100644 fuzz/testdata/fuzzing/new/.gitignore create mode 100644 fuzz/testdata/fuzzing/new/go.mod create mode 100644 fuzz/testdata/fuzzing/new/main_test.go create mode 100644 fuzz/testdata/fuzzing/nofindings/.gitignore create mode 100644 fuzz/testdata/fuzzing/nofindings/go.mod create mode 100644 fuzz/testdata/fuzzing/nofindings/main_test.go create mode 100644 fuzz/testdata/fuzzing/seed/go.mod create mode 100644 fuzz/testdata/fuzzing/seed/main_test.go create mode 100644 fuzz/testdata/fuzzing/seedfile/go.mod create mode 100644 fuzz/testdata/fuzzing/seedfile/main_test.go create mode 100644 fuzz/testdata/fuzzing/seedfile/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..bc1b7df --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,22 @@ +name: golangci-lint +on: + push: + pull_request: + +permissions: + contents: read + pull-requests: read + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + with: + go-version: stable + - name: golangci-lint + uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0 + with: + args: --timeout=5m \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4193d32 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,29 @@ +name: goreleaser + +on: + push: + tags: + - "v*.*.*" + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + - run: git fetch --force --tags + - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + with: + go-version: stable + cache: false + - uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0 + with: + distribution: goreleaser + version: latest + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/test-github-action.yml b/.github/workflows/test-github-action.yml new file mode 100644 index 0000000..550f30b --- /dev/null +++ b/.github/workflows/test-github-action.yml @@ -0,0 +1,50 @@ +name: test-github-action + +on: [push] + +permissions: + contents: read + +jobs: + fuzz-no-failure: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + with: + go-version: stable + - name: Run fuzzers + uses: ./ci/github-actions/fuzz + with: + fuzz-time: 15s + fail-fast: true + source-path: fuzz/testdata/fuzzing/nofindings + version: latest + fuzz-failure: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + with: + go-version: stable + - name: Run fuzzers + id: fuzz + uses: ./ci/github-actions/fuzz + continue-on-error: true + with: + fuzz-time: 2m + fail-fast: true + source-path: fuzz/testdata/fuzzing/new + version: latest + artifact-name: "failing-inputs-${{ matrix.os }}" + - name: Verify fuzzing failed + # https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context + # When a continue-on-error step fails, the outcome is failure, but the final conclusion is success. + if: steps.fuzz.outcome != 'failure' + run: exit 1 \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..d331d0a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,16 @@ +name: test + +on: + push: + pull_request: + +jobs: + test: + name: Test + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + with: + go-version-file: 'go.mod' + - run: go test -v ./... \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9a08f80 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/go-ci-fuzz +.idea/ + +dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..9b3ce1c --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,44 @@ +# This is an example .goreleaser.yml file with some sensible defaults. +# Make sure to check the documentation at https://goreleaser.com + +# The lines below are called `modelines`. See `:help modeline` +# Feel free to remove those if you don't want/need to use them. +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj + +version: 1 + +before: + hooks: + # You may remove this if you don't use go modules. + - go mod tidy + +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of `uname`. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip + +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 1c177dc..ef8808d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,123 @@ -# go-ci-fuzz -A tool for running Native Go Fuzz tests in CI pipelines +# Go CI Fuzz + +`go-ci-fuzz` is a CLI and a set of GitHub Actions that help you run [Native Go Fuzz Tests](https://go.dev/security/fuzz/) as part of your CI. + +It's a light wrapper around `go test -fuzz=` that supports multiple test targets. + +## Motivation + +`go-ci-fuzz` was created to achieve a developer friendly and lightweight way of running Native Go Fuzz Tests in Continuous Integration pipelines. +Current alternatives (ClusterFuzzLite, go-fuzz) don't support Native Go Fuzzing or support Native Go fuzzing inadequately through wrappers. + +## Typical Workflow + +```mermaid +flowchart LR + engineer[Engineer] -- 1. Writes Fuzz Tests --> GitHub + GitHub -- 2. Schedules a run --> go-ci-fuzz[Go CI Fuzz] + go-ci-fuzz -- 3. Reports failing inputs --> engineer + engineer -- 4. Commits failing inputs & fixes the issue --> GitHub +``` + + +## Running locally + +Although this tool is mostly meant to be used in CI pipelines it's still useful in local development. +If your project has many fuzz tests you can run all of them using `go-ci-fuzz`. + +```bash +go install github.com/form3tech-oss/go-ci-fuzz@{version} +go-ci-fuzz fuzz --fuzz-time 10m [--out /tmp/failures] +``` + +## Run as part of GitHub Actions + +This repo contains reusable actions located in [./ci/github-actions](ci/github-actions). +You can reference these actions from your workflows. + +All findings will be uploaded as artifacts to the workflow run. + +Below you'll find example workflows to incorporate in your CI pipeline. +Feel free to adjust the fuzz-time according to your appetite. + +The Github Action accepts the following properties: + +```yaml +inputs: + version: + description: "Version of go-ci-fuzz, e.g. latest or 0.1.0" + required: false + default: "0.1.0" + source-path: + description: "Path to the project's source code, current directory by default." + required: false + default: "." + fail-fast: + description: "Whether to continue fuzzing other targets if failing input was found." + required: false + default: "false" + fuzz-time: + description: "Cumulative time FuzzTests will run, in Go time.Duration format." + required: false + default: "5m" + artifact-name: + description: "Name of the artifact" + required: false + default: "failing-inputs" +``` + +### Fuzz incoming Pull Requests + +```yaml +# .github/workflows/gocifuzz_pr.yml +name: Go CI Fuzz - Pull Requests +on: + pull_request: + +permissions: + contents: read + +jobs: + Fuzz: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + with: + go-version: stable + - name: Run fuzzers + id: build + uses: form3tech-oss/go-ci-fuzz/ci/github-actions/fuzz@v0.1.0 + with: + fuzz-time: 5m + fail-fast: true +``` + +### Fuzz on fixed schedule + +```yaml +# .github/workflows/gocifuzz_schedule.yml +name: Go CI Fuzz - Scheduled +on: + workflow_dispatch: {} + schedule: + - cron: '0 2 * * *' + +permissions: + contents: read + +jobs: + Fuzz: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + with: + go-version: stable + - name: Run fuzzers + id: build + uses: form3tech-oss/go-ci-fuzz/ci/github-actions/fuzz@v0.1.0 + with: + fuzz-time: 30m + fail-fast: false +``` diff --git a/ci/github-actions/fuzz/action.yml b/ci/github-actions/fuzz/action.yml new file mode 100644 index 0000000..cdabb05 --- /dev/null +++ b/ci/github-actions/fuzz/action.yml @@ -0,0 +1,95 @@ +name: "fuzz" +description: "Runs fuzzing targets using go-ci-fuzz" +inputs: + version: + description: "Version of go-ci-fuzz, e.g. latest or 0.1.0" + required: false + default: "0.1.0" + source-path: + description: "Path to the project's source code, current directory by default." + required: false + default: "." + fail-fast: + description: "Whether to continue fuzzing other targets if failing input was found." + required: false + default: "false" + fuzz-time: + description: "Cumulative time FuzzTests will run, in Go time.Duration format." + required: false + default: "5m" + artifact-name: + description: "Name of the artifact" + required: false + default: "failing-inputs" +runs: + using: "composite" + steps: + - name: Compute Archive for Runner Environment + id: archive-info + shell: bash + run: | + set -eu + # https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + # The operating system of the runner executing the job. Possible values are Linux, Windows, or macOS. For example, Windows + EXE_EXT="" + if [ "$RUNNER_OS" == "Linux" ]; then + ARCHIVE_OS=Linux + ARCHIVE_EXT=tar.gz + elif [ "$RUNNER_OS" == "Windows" ]; then + ARCHIVE_OS=Windows + ARCHIVE_EXT=zip + EXE_EXT=".exe" + elif [ "$RUNNER_OS" == "macOS" ]; then + ARCHIVE_OS=Darwin + ARCHIVE_EXT=tar.gz + else + echo "$RUNNER_OS not supported" + exit 1 + fi + + # The architecture of the runner executing the job. Possible values are X86, X64, ARM, or ARM64. + if [ "$RUNNER_ARCH" == "X86" ]; then + ARCHIVE_ARCH="i386" + elif [ "$RUNNER_ARCH" == "X64" ]; then + ARCHIVE_ARCH="x86_64" + elif [ "$RUNNER_ARCH" == "ARM64" ]; then + ARCHIVE_ARCH="arm64" + else + echo "$RUNNER_ARCH not supported" + exit 1 + fi + + #go-ci-fuzz_Darwin_x86_64.tar.gz + echo "ARCHIVE_PATH=go-ci-fuzz_${ARCHIVE_OS}_${ARCHIVE_ARCH}.${ARCHIVE_EXT}" >> "$GITHUB_OUTPUT" + echo "EXE_NAME=go-ci-fuzz${EXE_EXT}" >> "$GITHUB_OUTPUT" + - id: fetch-asset + name: Get Release Asset + uses: dsaltares/fetch-gh-release-asset@3942ce82f1192754cd487a86f03eef6eeb89b5da # 1.1.0 + with: + repo: 'form3tech-oss/go-ci-fuzz' + version: ${{ inputs.version == 'latest' && 'latest' || format('tags/v%s', inputs.version) }} + file: ${{ steps.archive-info.outputs.ARCHIVE_PATH }} + - id: extract + name: Extract go-ci-fuzz + shell: bash + run: | + if [ "$RUNNER_OS" == "Windows" ]; then + unzip -u ${{ steps.archive-info.outputs.ARCHIVE_PATH }} go-ci-fuzz.exe -d "$RUNNER_TEMP" + else + tar -zxvf ${{ steps.archive-info.outputs.ARCHIVE_PATH }} --directory "$RUNNER_TEMP" go-ci-fuzz + fi + - id: fuzz + name: "Fuzz" + shell: bash + working-directory: "${{ inputs.source-path }}" + run: | + TEMP_DIR="$RUNNER_TEMP/failing-inputs" + mkdir -p $TEMP_DIR + echo "FAILING_INPUTS_DIR=${TEMP_DIR}" >> "$GITHUB_OUTPUT" + ${RUNNER_TEMP}/${{ steps.archive-info.outputs.EXE_NAME }} fuzz ./... --fuzz-time "${{ inputs.fuzz-time }}" --fail-fast="${{ inputs.fail-fast }}" --out="${TEMP_DIR}/" + - uses: actions/upload-artifact@v3 + if: 'failure()' + with: + name: ${{ inputs.artifact-name }} + path: ${{ steps.fuzz.outputs.FAILING_INPUTS_DIR }}/ + if-no-files-found: ignore diff --git a/cmd/fuzz.go b/cmd/fuzz.go new file mode 100644 index 0000000..4940d34 --- /dev/null +++ b/cmd/fuzz.go @@ -0,0 +1,138 @@ +package cmd + +import ( + "context" + "github.com/form3tech-oss/go-ci-fuzz/fuzz" + "github.com/spf13/cobra" + "os" + "path/filepath" + "time" +) + +const ( + flagFuzzTime = "fuzz-time" + flagFailFast = "fail-fast" + flagOut = "out" +) + +var fuzzCmd = &cobra.Command{ + Use: "fuzz [packages...]", + Short: "Runs all fuzz targets of packages", + Long: `Runs all fuzz targets in in current directory for the duration of --fuzz-time / N where N is the number of fuzz targets. +Continues to the next fuzz target on failure unless --fail-fast is defined. + +Failing outputs are written to --out directory if specified. The structure is identical to how corpora is stored locally. +e.g. +out-dir +└── testdata + └── fuzz + └── FuzzTarget + └── 0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef +`, + Run: fuzzRun, + SilenceUsage: true, +} + +func init() { + fuzzCmd.Flags().StringP(flagOut, "o", "", "directory to write failing outputs to") + fuzzCmd.Flags().Duration(flagFuzzTime, 10*time.Minute, "fuzzing duration for the whole suite") + fuzzCmd.Flags().Bool(flagFailFast, false, "exit once failing input is discovered") +} + +func fuzzRun(cmd *cobra.Command, args []string) { + ctx := context.Background() + + quiet, err := cmd.Flags().GetBool(flagQuiet) + if err != nil { + cmd.PrintErrln(err) + os.Exit(1) + } + + fuzzTime, err := cmd.Flags().GetDuration(flagFuzzTime) + if err != nil { + cmd.PrintErrln(err) + os.Exit(1) + } + + out, err := cmd.Flags().GetString(flagOut) + if err != nil { + cmd.PrintErrln(err) + os.Exit(1) + } + + failFast, err := cmd.Flags().GetBool(flagFailFast) + if err != nil { + cmd.PrintErrln(err) + os.Exit(1) + } + + wd, err := os.Getwd() + if err != nil { + cmd.PrintErrln(err) + os.Exit(1) + } + + proj := &fuzz.Project{ + Directory: wd, + Quiet: quiet, + } + + packages := []string{"."} + if len(args) > 0 { + packages = args + } + + targets, err := proj.ListFuzzTargets(ctx, packages...) + if err != nil { + cmd.PrintErrln(err) + os.Exit(1) + } + + if len(targets) == 0 { + cmd.Println("No fuzz tests found") + os.Exit(0) + } + + timePerTarget := time.Duration(fuzzTime.Milliseconds()/int64(len(targets))) * time.Millisecond + + hasFailures := false + + cmd.Printf("go-ci-fuzz: discovered %d targets, each of them will be fuzzed for %s\n", len(targets), timePerTarget) + for _, target := range targets { + cmd.Printf("go-ci-fuzz: fuzzing %s for %s\n", target, timePerTarget) + if err := proj.Fuzz(target, timePerTarget); err != nil { + hasFailures = true + if inputErr, ok := err.(fuzz.FailingInputError); ok { + if inputErr.File != "" && out != "" { + srcFile := filepath.Join(proj.Directory, inputErr.File) + destFile := filepath.Join(out, inputErr.File) + destFileFolder := filepath.Dir(destFile) + + if err := os.MkdirAll(destFileFolder, 0755); err != nil { + cmd.PrintErrf("error creating %s directory when copying a failing input from %s: %s\n", destFileFolder, inputErr.File, err) + os.Exit(1) + } + + if err := fuzz.CopyFile(destFile, srcFile, 0644); err != nil { + cmd.PrintErrf("copying a failing input from %s to %s: %s\n", srcFile, destFile, err) + os.Exit(1) + } + + cmd.Printf("Found failing input, saving to %s\n", destFile) + } else { + cmd.Printf("Found %s, not saving\n", inputErr) + } + } else { + cmd.PrintErrln(err) + os.Exit(1) + } + if failFast { + os.Exit(2) + } + } + } + + if hasFailures { + os.Exit(2) + } +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..d482545 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +const ( + flagQuiet = "quiet" +) + +var rootCmd = &cobra.Command{ + Use: "go-ci-fuzz", + Short: "Run Go Fuzz targets in CI systems", + Long: `go-ci-fuzz implements missing functionality in 'go test -fuzz' such as +- running multiple test targets in a single command +- extracting failed outputs +- corpus management +`, + Example: `go-ci-fuzz fuzz ./... --fuzz-time 10m --out /tmp/failing-inputs`, +} + +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + rootCmd.AddCommand(fuzzCmd) + rootCmd.PersistentFlags().Bool(flagQuiet, false, "silences underlying Go CLI StdOut") +} diff --git a/fuzz/copy.go b/fuzz/copy.go new file mode 100644 index 0000000..a7ffe46 --- /dev/null +++ b/fuzz/copy.go @@ -0,0 +1,61 @@ +package fuzz + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" +) + +func copyDirectory(dest string, src string) error { + err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + + destPath := filepath.Join(dest, relPath) + + if d.IsDir() { + if err := os.MkdirAll(destPath, 0755); err != nil { + return err + } + return nil + } + + if !d.Type().IsRegular() { + return nil + } + + return CopyFile(destPath, path, 0666) + }) + + if err != nil { + return err + } + return nil +} + +func CopyFile(dest, src string, perm os.FileMode) (err error) { + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.OpenFile(dest, os.O_CREATE|os.O_RDWR|os.O_TRUNC, perm) + if err != nil { + return fmt.Errorf("cannot create destination file") + } + defer destination.Close() + + if _, err := io.Copy(destination, source); err != nil { + return err + } + return nil +} diff --git a/fuzz/corpus.go b/fuzz/corpus.go new file mode 100644 index 0000000..70a8329 --- /dev/null +++ b/fuzz/corpus.go @@ -0,0 +1,90 @@ +package fuzz + +import ( + "context" + "fmt" + "os" + "path/filepath" +) + +func (p *Project) CorpusExtract(ctx context.Context, destination string, packages ...string) error { + targets, err := p.ListFuzzTargets(ctx, packages...) + if err != nil { + return err + } + + if len(targets) == 0 { + return nil + } + + if err := os.MkdirAll(destination, 0700); err != nil { + return fmt.Errorf("cannot create destination corpus directory: %w", err) + } + + for _, target := range targets { + corpusDir := p.relCorpusDir(target) + + srcCorpusDir := filepath.Join(p.Directory, corpusDir) + if _, err := os.Stat(srcCorpusDir); os.IsNotExist(err) { + continue + } + + destCorpusDir := filepath.Join(destination, corpusDir) + if err := os.MkdirAll(destCorpusDir, 0700); err != nil { + return fmt.Errorf("cannot create target destination corpus for %s: %w", destCorpusDir, err) + } + + if err := copyDirectory(destCorpusDir, srcCorpusDir); err != nil { + return fmt.Errorf("copying %q to %q failed: %w", srcCorpusDir, destCorpusDir, err) + } + } + + return nil +} + +func (p *Project) CorpusDelete(ctx context.Context, packages ...string) error { + targets, err := p.ListFuzzTargets(ctx, packages...) + if err != nil { + return err + } + + for _, target := range targets { + relDir := p.relCorpusDir(target) + + currentCorpusDir := filepath.Join(p.Directory, relDir) + err := os.RemoveAll(currentCorpusDir) + if err != nil { + return fmt.Errorf("error deleting corpus entries for %q located at %s", target, relDir) + } + } + + return nil +} + +func (p *Project) CorpusReplace(ctx context.Context, external string, packages ...string) error { + err := p.CorpusDelete(ctx, packages...) + if err != nil { + return err + } + return p.CorpusMerge(ctx, external, packages...) +} + +func (p *Project) CorpusMerge(ctx context.Context, external string, packages ...string) error { + targets, err := p.ListFuzzTargets(ctx, packages...) + if err != nil { + return err + } + + for _, target := range targets { + corpusDir := p.relCorpusDir(target) + + currentCorpusDir := filepath.Join(p.Directory, corpusDir) + externalCorpusDir := filepath.Join(external, corpusDir) + + if err := copyDirectory(currentCorpusDir, externalCorpusDir); err != nil { + return fmt.Errorf("copying %q to %q failed: %w", externalCorpusDir, currentCorpusDir, err) + } + } + + return nil +} diff --git a/fuzz/corpus_test.go b/fuzz/corpus_test.go new file mode 100644 index 0000000..17adbad --- /dev/null +++ b/fuzz/corpus_test.go @@ -0,0 +1,122 @@ +package fuzz + +import ( + "context" + "github.com/stretchr/testify/assert" + "io/fs" + "os" + "path/filepath" + "testing" +) + +func listFilesRecursively(dir string) ([]string, error) { + var files []string + + if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + files = append(files, rel) + return nil + }); err != nil { + return nil, err + } + + return files, nil +} + +func TestCorpusExtract(t *testing.T) { + t.Run("extracts fuzzing targets", func(t *testing.T) { + project := Project{Directory: "./testdata/corpus/multiple"} + ctx := context.Background() + + tempDir, err := os.MkdirTemp("", "go-ci-fuzz-*") + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + if os.RemoveAll(tempDir) != nil { + t.Error(err) + } + }) + + err = project.CorpusExtract(ctx, tempDir, "...") + if !assert.NoError(t, err, "corpus copying should not fail") { + return + } + + files, err := listFilesRecursively(tempDir) + if !assert.NoError(t, err, "listing tempDir should not fail") { + return + } + + assert.ElementsMatch(t, files, []string{ + "testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef", + "sub/testdata/fuzz/FuzzSubTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef", + }) + }) +} + +func TestCorpusDelete(t *testing.T) { + t.Run("deletes corpora that have matching target", func(t *testing.T) { + tempDir, err := os.MkdirTemp("", "go-ci-fuzz-*") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if os.RemoveAll(tempDir) != nil { + t.Error(err) + } + }) + + if err := copyDirectory(tempDir, "./testdata/corpus/multiple"); err != nil { + t.Fatal(err) + } + + files, err := listFilesRecursively(tempDir) + if !assert.NoError(t, err, "listing tempDir should not fail") { + return + } + + if !assert.ElementsMatch(t, files, []string{ + "go.mod", + "main_test.go", + "nocorpus/main_test.go", + "sub/main_test.go", + "testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef", + "sub/testdata/fuzz/FuzzSubTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef", + "sub/testdata/fuzz/FuzzNonExistingTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef", + }) { + return + } + + project := Project{Directory: tempDir} + ctx := context.Background() + err = project.CorpusDelete(ctx, "...") + assert.NoError(t, err, "corpus deletion") + + files, err = listFilesRecursively(tempDir) + if !assert.NoError(t, err, "listing tempDir should not fail") { + return + } + + if !assert.ElementsMatch(t, files, []string{ + "go.mod", + "main_test.go", + "nocorpus/main_test.go", + "sub/main_test.go", + "sub/testdata/fuzz/FuzzNonExistingTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef", + }) { + return + } + + }) +} diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go new file mode 100644 index 0000000..0233fed --- /dev/null +++ b/fuzz/fuzz.go @@ -0,0 +1,116 @@ +package fuzz + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" +) + +var failingInputRegex = regexp.MustCompile(`^\s*go test -run=Fuzz([a-zA-Z0-9_]+)/([a-zA-Z0-9#]+)`) +var failingSeedInputRegex = regexp.MustCompile(`^\s*failure while testing seed corpus entry: Fuzz([a-zA-Z0-9_]+)/([a-zA-Z0-9#]+)`) + +type FailingInputError struct { + ID string + File string + Seed bool +} + +func (f FailingInputError) Error() string { + newOrSeed := "new" + if f.Seed { + newOrSeed = "seed" + } + + if f.File != "" { + return fmt.Sprintf("failing %s input, saved at %s", newOrSeed, f.File) + } + return fmt.Sprintf("failing %s input: %s", newOrSeed, f.ID) +} + +func (p *Project) relCorpusDir(target TestTarget) string { + // target.Package contains the root package as well + // we need to strip it because it refers to the current working directory . + pkg := strings.TrimPrefix(strings.TrimPrefix(target.Package, target.RootPackage), "/") + return filepath.Join(pkg, "testdata/fuzz", target.Name) +} + +func (p *Project) Fuzz(target TestTarget, d time.Duration) error { + args := []string{ + "test", + "-test.run=^$", + "-test.fuzz=^" + target.Name + "$", + "-test.fuzztime=" + d.String(), + target.Package, + } + + cmd := exec.Command("go", args...) + if p.Directory != "" { + cmd.Dir = p.Directory + } + + var stdout bytes.Buffer + if !p.Quiet { + cmd.Stdout = io.MultiWriter(os.Stdout, &stdout) + } else { + cmd.Stdout = &stdout + } + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err == nil { + return nil + } + + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + return fmt.Errorf("fuzzing failed with an unexpected error: %w", err) + } + + scanner := bufio.NewScanner(&stdout) + corpusDirectory := p.relCorpusDir(target) + for scanner.Scan() { + line := scanner.Text() + + // For newly discovered inputs the CLI outputs the following: + // > Failing input written to testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef + // > To re-run: + // > go test -run=FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef + // we match against the last line and extract the Test ID from it + if matches := failingInputRegex.FindStringSubmatch(line); matches != nil { + if len(matches) != 3 { + return fmt.Errorf("parsing fuzzing output failed, matched %q, but found %d submatches, expected 2", line, len(matches)) + } + + id := matches[2] + return FailingInputError{ID: id, File: filepath.Join(corpusDirectory, id)} + } + + // For inputs already in the corpus we get + // > failure while testing seed corpus entry: FuzzTarget/seed#0 + // for seed corpus entries added by f.Add() OR + // > failure while testing seed corpus entry: FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef + // for seed corpus stored in files in ./testdata directory + if matches := failingSeedInputRegex.FindStringSubmatch(line); matches != nil { + if len(matches) != 3 { + return fmt.Errorf("parsing seed corpus fuzzing output failed, matched %q, but found %d submatches, expected 2", line, len(matches)) + } + id := matches[2] + if strings.HasPrefix(id, "seed#") { + return FailingInputError{ID: id, Seed: true} + } else { + return FailingInputError{ID: id, File: filepath.Join(corpusDirectory, id), Seed: true} + } + } + + } + + return fmt.Errorf("fuzzing failed with an unexpected exit error: %w", err) +} diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go new file mode 100644 index 0000000..1740a19 --- /dev/null +++ b/fuzz/fuzz_test.go @@ -0,0 +1,73 @@ +package fuzz + +import ( + "github.com/stretchr/testify/assert" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestFuzz(t *testing.T) { + t.Run("Add() seed entry", func(t *testing.T) { + p := Project{Directory: "./testdata/fuzzing/seed", Quiet: true} + + err := p.Fuzz(TestTarget{ + Name: "FuzzTarget", + Package: "seed", + RootPackage: "seed", + }, 1*time.Minute) + + assert.ErrorIs(t, err, FailingInputError{ID: "seed#0", File: "", Seed: true}) + }) + + t.Run("file seed entry", func(t *testing.T) { + p := Project{Directory: "./testdata/fuzzing/seedfile", Quiet: true} + + err := p.Fuzz(TestTarget{ + Name: "FuzzTarget", + Package: "seedfile", + RootPackage: "seedfile", + }, 1*time.Minute) + + assert.ErrorIs(t, err, FailingInputError{ID: "0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef", File: "testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef", Seed: true}) + }) + + t.Run("new entry discovered", func(t *testing.T) { + p := Project{Directory: "./testdata/fuzzing/new", Quiet: true} + + removeTestData := func() { + err := os.RemoveAll(filepath.Join(p.Directory, "testdata")) + if err != nil { + t.Fatal("removing old testdata failed", err) + } + } + removeTestData() + t.Cleanup(removeTestData) + + err := p.Fuzz(TestTarget{ + Name: "FuzzTarget", + Package: "github.com/form3tech-oss/new", + RootPackage: "github.com/form3tech-oss/new", + }, 1*time.Minute) + + var inputErr FailingInputError + + assert.ErrorAs(t, err, &inputErr) + assert.True(t, strings.HasPrefix(inputErr.File, "testdata/fuzz/FuzzTarget/"), "error.File must begin with testdata/fuzz/FuzzTarget/ it's %s instead", inputErr.File) + assert.False(t, inputErr.Seed, "newly found failing input must not be marked as Seed") + }) + + t.Run("no findings", func(t *testing.T) { + p := Project{Directory: "./testdata/fuzzing/nofindings", Quiet: true} + + err := p.Fuzz(TestTarget{ + Name: "FuzzTarget", + Package: "nofindings", + RootPackage: "nofindings", + }, 5*time.Second) + + assert.NoError(t, err) + }) +} diff --git a/fuzz/project.go b/fuzz/project.go new file mode 100644 index 0000000..01c235f --- /dev/null +++ b/fuzz/project.go @@ -0,0 +1,6 @@ +package fuzz + +type Project struct { + Directory string + Quiet bool +} diff --git a/fuzz/targets.go b/fuzz/targets.go new file mode 100644 index 0000000..a904bee --- /dev/null +++ b/fuzz/targets.go @@ -0,0 +1,140 @@ +package fuzz + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "go/scanner" + "go/token" + "os" + "os/exec" + "path/filepath" + "regexp" +) + +type TestTarget struct { + Name string + Package string + RootPackage string +} + +func (tt TestTarget) String() string { + return fmt.Sprintf("%s#%s", tt.Package, tt.Name) +} + +func (p *Project) ListFuzzTargets(ctx context.Context, packages ...string) ([]TestTarget, error) { + relativePackages := make([]string, len(packages)) + for i, pkg := range packages { + // NOTE: Go will look in GOPATH for packages that cannot be found in the current module which is slow and breaks the tool + // TODO: find a better solution to allow calling go-ci-fuzz with fully qualified module name such as github.com///some/package + relativePackages[i] = fmt.Sprintf("./%s", pkg) + } + + targets, err := p.listTestTargets(ctx, "^Fuzz*", relativePackages...) + if err != nil { + return nil, fmt.Errorf("discovering fuzz targets failed: %s", err) + } + + return targets, nil +} + +type Module struct { + Path string + Main bool +} + +type Package struct { + Dir string + ImportPath string + Name string + TestGoFiles []string + XTestGoFiles []string + Module Module +} + +func (p *Project) listPackages(ctx context.Context, packages ...string) ([]Package, error) { + args := append([]string{ + "list", + "-find", + "-json", + }, packages...) + + cmd := exec.CommandContext(ctx, "go", args...) + if p.Directory != "" { + cmd.Dir = p.Directory + } + + pkgBytes, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("cannot get package list for: %w", err) + } + + var pkgs []Package + + decoder := json.NewDecoder(bytes.NewReader(pkgBytes)) + for decoder.More() { + var pkg Package + if err := decoder.Decode(&pkg); err != nil { + return nil, err + } + pkgs = append(pkgs, pkg) + } + + return pkgs, nil +} + +// We cannot use go test -list because of this bug: https://github.com/golang/go/issues/25339 +// So we list all packages and test files and look for test targets ourselves by running go.Scanner +func (p *Project) listTestTargets(ctx context.Context, pattern string, packages ...string) ([]TestTarget, error) { + pkgs, err := p.listPackages(ctx, packages...) + if err != nil { + return nil, fmt.Errorf("error listing packages: %w", err) + } + + pat, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("error compiling pattern") + } + + var targets []TestTarget + + for _, pkg := range pkgs { + var testFiles []string + testFiles = append(testFiles, pkg.TestGoFiles...) + testFiles = append(testFiles, pkg.XTestGoFiles...) + + for _, testFile := range testFiles { + path := filepath.Join(pkg.Dir, testFile) + content, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var scan scanner.Scanner + fs := token.NewFileSet() + f := fs.AddFile(testFile, fs.Base(), len(content)) + scan.Init(f, content, nil, 0) + + // TODO: we can improve this so that it checks if the first param of the function is testing.F + var previousToken token.Token + for { + _, tok, lit := scan.Scan() + if tok == token.EOF { + break + } + if previousToken == token.FUNC && tok.IsLiteral() { + if pat.MatchString(lit) { + targets = append(targets, TestTarget{ + Name: lit, + Package: pkg.ImportPath, + RootPackage: pkg.Module.Path, + }) + } + } + previousToken = tok + } + } + } + return targets, nil +} diff --git a/fuzz/targets_test.go b/fuzz/targets_test.go new file mode 100644 index 0000000..e2ef89a --- /dev/null +++ b/fuzz/targets_test.go @@ -0,0 +1,69 @@ +package fuzz + +import ( + "context" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestDiscoverTargets(t *testing.T) { + p := Project{Directory: "./testdata/discover", Quiet: true} + t.Run("single fuzz target", func(t *testing.T) { + ctx := context.Background() + targets, err := p.ListFuzzTargets(ctx, ".") + assert.NoError(t, err) + assert.EqualValues(t, []TestTarget{{ + Name: "FuzzTarget", + Package: "discover", + RootPackage: "discover", + }}, targets) + }) + + t.Run("all packages", func(t *testing.T) { + ctx := context.Background() + targets, err := p.ListFuzzTargets(ctx, "...") + assert.NoError(t, err) + assert.ElementsMatch(t, []TestTarget{{ + Name: "FuzzTarget", + Package: "discover", + RootPackage: "discover", + }, { + Name: "FuzzSubTarget", + Package: "discover/subpackage", + RootPackage: "discover", + }, { + Name: "FuzzMain", + Package: "discover/submain", + RootPackage: "discover", + }}, targets) + }) + + t.Run("non-existent package", func(t *testing.T) { + ctx := context.Background() + _, err := p.ListFuzzTargets(ctx, "doesnotexist") + assert.Error(t, err, "discovering non existent package should fail") + }) + + t.Run("subpackage", func(t *testing.T) { + ctx := context.Background() + targets, err := p.ListFuzzTargets(ctx, "subpackage") + assert.NoError(t, err) + assert.ElementsMatch(t, []TestTarget{{ + Name: "FuzzSubTarget", + Package: "discover/subpackage", + RootPackage: "discover", + }}, targets) + }) + + t.Run("main does not run", func(t *testing.T) { + p := Project{Directory: "./testdata/discovermain", Quiet: true} + ctx := context.Background() + targets, err := p.ListFuzzTargets(ctx, "...") + assert.NoError(t, err) + assert.ElementsMatch(t, []TestTarget{{ + Name: "FuzzTarget", + Package: "discovermain", + RootPackage: "discovermain", + }}, targets) + }) +} diff --git a/fuzz/testdata/corpus/multiple/go.mod b/fuzz/testdata/corpus/multiple/go.mod new file mode 100644 index 0000000..ebecc1a --- /dev/null +++ b/fuzz/testdata/corpus/multiple/go.mod @@ -0,0 +1,3 @@ +module multiple + +go 1.19 diff --git a/fuzz/testdata/corpus/multiple/main_test.go b/fuzz/testdata/corpus/multiple/main_test.go new file mode 100644 index 0000000..54378b3 --- /dev/null +++ b/fuzz/testdata/corpus/multiple/main_test.go @@ -0,0 +1,12 @@ +package multiple + +import "testing" + +func FuzzTarget(f *testing.F) { + f.Add("a") + f.Fuzz(func(t *testing.T, in string) { + if in == "z" { + t.Fail() + } + }) +} diff --git a/fuzz/testdata/corpus/multiple/nocorpus/main_test.go b/fuzz/testdata/corpus/multiple/nocorpus/main_test.go new file mode 100644 index 0000000..50e7c96 --- /dev/null +++ b/fuzz/testdata/corpus/multiple/nocorpus/main_test.go @@ -0,0 +1,13 @@ +package nocorpus + +import "testing" + +func FuzzSubTarget(f *testing.F) { + f.Add("a") + f.Add("z") + f.Fuzz(func(t *testing.T, in string) { + if in == "z" { + t.Fail() + } + }) +} diff --git a/fuzz/testdata/corpus/multiple/sub/main_test.go b/fuzz/testdata/corpus/multiple/sub/main_test.go new file mode 100644 index 0000000..f2a32a9 --- /dev/null +++ b/fuzz/testdata/corpus/multiple/sub/main_test.go @@ -0,0 +1,12 @@ +package sub + +import "testing" + +func FuzzSubTarget(f *testing.F) { + f.Add("a") + f.Fuzz(func(t *testing.T, in string) { + if in == "z" { + t.Fail() + } + }) +} diff --git a/fuzz/testdata/corpus/multiple/sub/testdata/fuzz/FuzzNonExistingTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef b/fuzz/testdata/corpus/multiple/sub/testdata/fuzz/FuzzNonExistingTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef new file mode 100644 index 0000000..5866254 --- /dev/null +++ b/fuzz/testdata/corpus/multiple/sub/testdata/fuzz/FuzzNonExistingTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef @@ -0,0 +1,2 @@ +go test fuzz v1 +string("z") diff --git a/fuzz/testdata/corpus/multiple/sub/testdata/fuzz/FuzzSubTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef b/fuzz/testdata/corpus/multiple/sub/testdata/fuzz/FuzzSubTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef new file mode 100644 index 0000000..5866254 --- /dev/null +++ b/fuzz/testdata/corpus/multiple/sub/testdata/fuzz/FuzzSubTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef @@ -0,0 +1,2 @@ +go test fuzz v1 +string("z") diff --git a/fuzz/testdata/corpus/multiple/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef b/fuzz/testdata/corpus/multiple/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef new file mode 100644 index 0000000..5866254 --- /dev/null +++ b/fuzz/testdata/corpus/multiple/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef @@ -0,0 +1,2 @@ +go test fuzz v1 +string("z") diff --git a/fuzz/testdata/discover/go.mod b/fuzz/testdata/discover/go.mod new file mode 100644 index 0000000..8f23bd0 --- /dev/null +++ b/fuzz/testdata/discover/go.mod @@ -0,0 +1,3 @@ +module discover + +go 1.19 diff --git a/fuzz/testdata/discover/main_test.go b/fuzz/testdata/discover/main_test.go new file mode 100644 index 0000000..ab3af15 --- /dev/null +++ b/fuzz/testdata/discover/main_test.go @@ -0,0 +1,12 @@ +package discover + +import "testing" + +func FuzzTarget(f *testing.F) { + f.Add("a") + f.Fuzz(func(t *testing.T, in string) { + if in == "z" { + t.Fail() + } + }) +} diff --git a/fuzz/testdata/discover/submain/main_test.go b/fuzz/testdata/discover/submain/main_test.go new file mode 100644 index 0000000..a59e645 --- /dev/null +++ b/fuzz/testdata/discover/submain/main_test.go @@ -0,0 +1,13 @@ +package main + +import "testing" + +func FuzzMain(f *testing.F) { + f.Add("a") + f.Add("z") + f.Fuzz(func(t *testing.T, in string) { + if in == "z" { + t.Fail() + } + }) +} diff --git a/fuzz/testdata/discover/subpackage/main_test.go b/fuzz/testdata/discover/subpackage/main_test.go new file mode 100644 index 0000000..1afb097 --- /dev/null +++ b/fuzz/testdata/discover/subpackage/main_test.go @@ -0,0 +1,13 @@ +package subpackage + +import "testing" + +func FuzzSubTarget(f *testing.F) { + f.Add("a") + f.Add("z") + f.Fuzz(func(t *testing.T, in string) { + if in == "z" { + t.Fail() + } + }) +} diff --git a/fuzz/testdata/discover/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef b/fuzz/testdata/discover/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef new file mode 100644 index 0000000..5866254 --- /dev/null +++ b/fuzz/testdata/discover/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef @@ -0,0 +1,2 @@ +go test fuzz v1 +string("z") diff --git a/fuzz/testdata/discovermain/go.mod b/fuzz/testdata/discovermain/go.mod new file mode 100644 index 0000000..e3484f4 --- /dev/null +++ b/fuzz/testdata/discovermain/go.mod @@ -0,0 +1,3 @@ +module discovermain + +go 1.19 diff --git a/fuzz/testdata/discovermain/main_test.go b/fuzz/testdata/discovermain/main_test.go new file mode 100644 index 0000000..47dff66 --- /dev/null +++ b/fuzz/testdata/discovermain/main_test.go @@ -0,0 +1,19 @@ +package discovermain_test + +import ( + "log" + "testing" +) + +func TestMain(m *testing.M) { + log.Fatalln("this shall not be called") +} + +func FuzzTarget(f *testing.F) { + f.Add("a") + f.Fuzz(func(t *testing.T, in string) { + if in == "z" { + t.Fail() + } + }) +} diff --git a/fuzz/testdata/fuzzing/new/.gitignore b/fuzz/testdata/fuzzing/new/.gitignore new file mode 100644 index 0000000..d07440c --- /dev/null +++ b/fuzz/testdata/fuzzing/new/.gitignore @@ -0,0 +1 @@ +./testdata \ No newline at end of file diff --git a/fuzz/testdata/fuzzing/new/go.mod b/fuzz/testdata/fuzzing/new/go.mod new file mode 100644 index 0000000..1b328b0 --- /dev/null +++ b/fuzz/testdata/fuzzing/new/go.mod @@ -0,0 +1,3 @@ +module github.com/form3tech-oss/new + +go 1.19 diff --git a/fuzz/testdata/fuzzing/new/main_test.go b/fuzz/testdata/fuzzing/new/main_test.go new file mode 100644 index 0000000..9827883 --- /dev/null +++ b/fuzz/testdata/fuzzing/new/main_test.go @@ -0,0 +1,12 @@ +package new + +import "testing" + +func FuzzTarget(f *testing.F) { + f.Add("a") + f.Fuzz(func(t *testing.T, in string) { + if in == "z" { + t.Fail() + } + }) +} diff --git a/fuzz/testdata/fuzzing/nofindings/.gitignore b/fuzz/testdata/fuzzing/nofindings/.gitignore new file mode 100644 index 0000000..d07440c --- /dev/null +++ b/fuzz/testdata/fuzzing/nofindings/.gitignore @@ -0,0 +1 @@ +./testdata \ No newline at end of file diff --git a/fuzz/testdata/fuzzing/nofindings/go.mod b/fuzz/testdata/fuzzing/nofindings/go.mod new file mode 100644 index 0000000..283acb8 --- /dev/null +++ b/fuzz/testdata/fuzzing/nofindings/go.mod @@ -0,0 +1,3 @@ +module nofindings + +go 1.19 diff --git a/fuzz/testdata/fuzzing/nofindings/main_test.go b/fuzz/testdata/fuzzing/nofindings/main_test.go new file mode 100644 index 0000000..1e7ba5e --- /dev/null +++ b/fuzz/testdata/fuzzing/nofindings/main_test.go @@ -0,0 +1,9 @@ +package nofindings + +import "testing" + +func FuzzTarget(f *testing.F) { + f.Add("a") + f.Fuzz(func(t *testing.T, in string) { + }) +} diff --git a/fuzz/testdata/fuzzing/seed/go.mod b/fuzz/testdata/fuzzing/seed/go.mod new file mode 100644 index 0000000..92615be --- /dev/null +++ b/fuzz/testdata/fuzzing/seed/go.mod @@ -0,0 +1,3 @@ +module seed + +go 1.19 diff --git a/fuzz/testdata/fuzzing/seed/main_test.go b/fuzz/testdata/fuzzing/seed/main_test.go new file mode 100644 index 0000000..ea37aa3 --- /dev/null +++ b/fuzz/testdata/fuzzing/seed/main_test.go @@ -0,0 +1,12 @@ +package seed + +import "testing" + +func FuzzTarget(f *testing.F) { + f.Add("z") + f.Fuzz(func(t *testing.T, in string) { + if in == "z" { + t.Fail() + } + }) +} diff --git a/fuzz/testdata/fuzzing/seedfile/go.mod b/fuzz/testdata/fuzzing/seedfile/go.mod new file mode 100644 index 0000000..b71590f --- /dev/null +++ b/fuzz/testdata/fuzzing/seedfile/go.mod @@ -0,0 +1,3 @@ +module seedfile + +go 1.19 diff --git a/fuzz/testdata/fuzzing/seedfile/main_test.go b/fuzz/testdata/fuzzing/seedfile/main_test.go new file mode 100644 index 0000000..59ed12f --- /dev/null +++ b/fuzz/testdata/fuzzing/seedfile/main_test.go @@ -0,0 +1,12 @@ +package seedfile + +import "testing" + +func FuzzTarget(f *testing.F) { + f.Add("a") + f.Fuzz(func(t *testing.T, in string) { + if in == "z" { + t.Fail() + } + }) +} diff --git a/fuzz/testdata/fuzzing/seedfile/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef b/fuzz/testdata/fuzzing/seedfile/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef new file mode 100644 index 0000000..5866254 --- /dev/null +++ b/fuzz/testdata/fuzzing/seedfile/testdata/fuzz/FuzzTarget/0a7e5e215d8c088d4b9c4993d0189a07e81603fbdf64f2ca44738aa27159acef @@ -0,0 +1,2 @@ +go test fuzz v1 +string("z") diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1c1788c --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module github.com/form3tech-oss/go-ci-fuzz + +go 1.21 + +require ( + github.com/spf13/cobra v1.8.0 + github.com/stretchr/testify v1.8.4 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8236a26 --- /dev/null +++ b/go.sum @@ -0,0 +1,18 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..426df16 --- /dev/null +++ b/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/form3tech-oss/go-ci-fuzz/cmd" + +func main() { + cmd.Execute() +} From dbd626f66bba5f921293937d8d1f81b188cb6a0d Mon Sep 17 00:00:00 2001 From: Maciej Mionskowski Date: Mon, 27 Nov 2023 09:44:30 +0100 Subject: [PATCH 2/9] chore: extract the context from cobra.Command, pass the ctx to Fuzz() func --- README.md | 1 - cmd/fuzz.go | 5 ++--- fuzz/fuzz.go | 5 +++-- fuzz/fuzz_test.go | 13 +++++++++---- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ef8808d..c71dc75 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ flowchart LR engineer -- 4. Commits failing inputs & fixes the issue --> GitHub ``` - ## Running locally Although this tool is mostly meant to be used in CI pipelines it's still useful in local development. diff --git a/cmd/fuzz.go b/cmd/fuzz.go index 4940d34..d2086c1 100644 --- a/cmd/fuzz.go +++ b/cmd/fuzz.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "github.com/form3tech-oss/go-ci-fuzz/fuzz" "github.com/spf13/cobra" "os" @@ -40,7 +39,7 @@ func init() { } func fuzzRun(cmd *cobra.Command, args []string) { - ctx := context.Background() + ctx := cmd.Context() quiet, err := cmd.Flags().GetBool(flagQuiet) if err != nil { @@ -100,7 +99,7 @@ func fuzzRun(cmd *cobra.Command, args []string) { cmd.Printf("go-ci-fuzz: discovered %d targets, each of them will be fuzzed for %s\n", len(targets), timePerTarget) for _, target := range targets { cmd.Printf("go-ci-fuzz: fuzzing %s for %s\n", target, timePerTarget) - if err := proj.Fuzz(target, timePerTarget); err != nil { + if err := proj.Fuzz(ctx, target, timePerTarget); err != nil { hasFailures = true if inputErr, ok := err.(fuzz.FailingInputError); ok { if inputErr.File != "" && out != "" { diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go index 0233fed..b0b264a 100644 --- a/fuzz/fuzz.go +++ b/fuzz/fuzz.go @@ -3,6 +3,7 @@ package fuzz import ( "bufio" "bytes" + "context" "errors" "fmt" "io" @@ -42,7 +43,7 @@ func (p *Project) relCorpusDir(target TestTarget) string { return filepath.Join(pkg, "testdata/fuzz", target.Name) } -func (p *Project) Fuzz(target TestTarget, d time.Duration) error { +func (p *Project) Fuzz(ctx context.Context, target TestTarget, d time.Duration) error { args := []string{ "test", "-test.run=^$", @@ -51,7 +52,7 @@ func (p *Project) Fuzz(target TestTarget, d time.Duration) error { target.Package, } - cmd := exec.Command("go", args...) + cmd := exec.CommandContext(ctx, "go", args...) if p.Directory != "" { cmd.Dir = p.Directory } diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go index 1740a19..720f166 100644 --- a/fuzz/fuzz_test.go +++ b/fuzz/fuzz_test.go @@ -1,6 +1,7 @@ package fuzz import ( + "context" "github.com/stretchr/testify/assert" "os" "path/filepath" @@ -11,9 +12,10 @@ import ( func TestFuzz(t *testing.T) { t.Run("Add() seed entry", func(t *testing.T) { + ctx := context.Background() p := Project{Directory: "./testdata/fuzzing/seed", Quiet: true} - err := p.Fuzz(TestTarget{ + err := p.Fuzz(ctx, TestTarget{ Name: "FuzzTarget", Package: "seed", RootPackage: "seed", @@ -23,9 +25,10 @@ func TestFuzz(t *testing.T) { }) t.Run("file seed entry", func(t *testing.T) { + ctx := context.Background() p := Project{Directory: "./testdata/fuzzing/seedfile", Quiet: true} - err := p.Fuzz(TestTarget{ + err := p.Fuzz(ctx, TestTarget{ Name: "FuzzTarget", Package: "seedfile", RootPackage: "seedfile", @@ -35,6 +38,7 @@ func TestFuzz(t *testing.T) { }) t.Run("new entry discovered", func(t *testing.T) { + ctx := context.Background() p := Project{Directory: "./testdata/fuzzing/new", Quiet: true} removeTestData := func() { @@ -46,7 +50,7 @@ func TestFuzz(t *testing.T) { removeTestData() t.Cleanup(removeTestData) - err := p.Fuzz(TestTarget{ + err := p.Fuzz(ctx, TestTarget{ Name: "FuzzTarget", Package: "github.com/form3tech-oss/new", RootPackage: "github.com/form3tech-oss/new", @@ -60,9 +64,10 @@ func TestFuzz(t *testing.T) { }) t.Run("no findings", func(t *testing.T) { + ctx := context.Background() p := Project{Directory: "./testdata/fuzzing/nofindings", Quiet: true} - err := p.Fuzz(TestTarget{ + err := p.Fuzz(ctx, TestTarget{ Name: "FuzzTarget", Package: "nofindings", RootPackage: "nofindings", From fab511eae6db12ebf984690cef74c13a859323d0 Mon Sep 17 00:00:00 2001 From: Maciej Mionskowski Date: Mon, 27 Nov 2023 09:46:24 +0100 Subject: [PATCH 3/9] chore: reword Long command description in the root command --- cmd/root.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index d482545..89810d9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,9 +13,9 @@ const ( var rootCmd = &cobra.Command{ Use: "go-ci-fuzz", Short: "Run Go Fuzz targets in CI systems", - Long: `go-ci-fuzz implements missing functionality in 'go test -fuzz' such as -- running multiple test targets in a single command -- extracting failed outputs + Long: `Implements missing functionalities in 'go test -fuzz' such as +- run multiple test targets in a single command +- extract failed outputs - corpus management `, Example: `go-ci-fuzz fuzz ./... --fuzz-time 10m --out /tmp/failing-inputs`, From 9f3bf15e2dda16d45bf535b6a9b40fcf5439b11b Mon Sep 17 00:00:00 2001 From: Maciej Mionskowski Date: Mon, 27 Nov 2023 09:47:56 +0100 Subject: [PATCH 4/9] chore: move Project struct to fuzz.go --- fuzz/fuzz.go | 11 +++++++++-- fuzz/project.go | 6 ------ 2 files changed, 9 insertions(+), 8 deletions(-) delete mode 100644 fuzz/project.go diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go index b0b264a..ea279d8 100644 --- a/fuzz/fuzz.go +++ b/fuzz/fuzz.go @@ -15,8 +15,15 @@ import ( "time" ) -var failingInputRegex = regexp.MustCompile(`^\s*go test -run=Fuzz([a-zA-Z0-9_]+)/([a-zA-Z0-9#]+)`) -var failingSeedInputRegex = regexp.MustCompile(`^\s*failure while testing seed corpus entry: Fuzz([a-zA-Z0-9_]+)/([a-zA-Z0-9#]+)`) +var ( + failingInputRegex = regexp.MustCompile(`^\s*go test -run=Fuzz([a-zA-Z0-9_]+)/([a-zA-Z0-9#]+)`) + failingSeedInputRegex = regexp.MustCompile(`^\s*failure while testing seed corpus entry: Fuzz([a-zA-Z0-9_]+)/([a-zA-Z0-9#]+)`) +) + +type Project struct { + Directory string + Quiet bool +} type FailingInputError struct { ID string diff --git a/fuzz/project.go b/fuzz/project.go deleted file mode 100644 index 01c235f..0000000 --- a/fuzz/project.go +++ /dev/null @@ -1,6 +0,0 @@ -package fuzz - -type Project struct { - Directory string - Quiet bool -} From 102a43636d99bab35fb7f6d36b0255c8a4fcc917 Mon Sep 17 00:00:00 2001 From: Maciej Mionskowski Date: Mon, 27 Nov 2023 09:57:00 +0100 Subject: [PATCH 5/9] chore: use filepath.Rel instead of strings.StripPrefix when constructing relative corpus dir --- fuzz/corpus.go | 17 +++++++++++++---- fuzz/fuzz.go | 16 +++++++++++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/fuzz/corpus.go b/fuzz/corpus.go index 70a8329..58d722d 100644 --- a/fuzz/corpus.go +++ b/fuzz/corpus.go @@ -22,7 +22,10 @@ func (p *Project) CorpusExtract(ctx context.Context, destination string, package } for _, target := range targets { - corpusDir := p.relCorpusDir(target) + corpusDir, err := p.relCorpusDir(target) + if err != nil { + return fmt.Errorf("cannot get corpus directory path: %w", err) + } srcCorpusDir := filepath.Join(p.Directory, corpusDir) if _, err := os.Stat(srcCorpusDir); os.IsNotExist(err) { @@ -49,10 +52,13 @@ func (p *Project) CorpusDelete(ctx context.Context, packages ...string) error { } for _, target := range targets { - relDir := p.relCorpusDir(target) + relDir, err := p.relCorpusDir(target) + if err != nil { + return fmt.Errorf("cannot get corpus directory path: %w", err) + } currentCorpusDir := filepath.Join(p.Directory, relDir) - err := os.RemoveAll(currentCorpusDir) + err = os.RemoveAll(currentCorpusDir) if err != nil { return fmt.Errorf("error deleting corpus entries for %q located at %s", target, relDir) } @@ -76,7 +82,10 @@ func (p *Project) CorpusMerge(ctx context.Context, external string, packages ... } for _, target := range targets { - corpusDir := p.relCorpusDir(target) + corpusDir, err := p.relCorpusDir(target) + if err != nil { + return fmt.Errorf("cannot get corpus directory path: %w", err) + } currentCorpusDir := filepath.Join(p.Directory, corpusDir) externalCorpusDir := filepath.Join(external, corpusDir) diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go index ea279d8..6358407 100644 --- a/fuzz/fuzz.go +++ b/fuzz/fuzz.go @@ -43,11 +43,14 @@ func (f FailingInputError) Error() string { return fmt.Sprintf("failing %s input: %s", newOrSeed, f.ID) } -func (p *Project) relCorpusDir(target TestTarget) string { +func (p *Project) relCorpusDir(target TestTarget) (string, error) { // target.Package contains the root package as well - // we need to strip it because it refers to the current working directory . - pkg := strings.TrimPrefix(strings.TrimPrefix(target.Package, target.RootPackage), "/") - return filepath.Join(pkg, "testdata/fuzz", target.Name) + // we need to strip it because it refers to the current working directory. + pkg, err := filepath.Rel(target.RootPackage, target.Package) + if err != nil { + return "", err + } + return filepath.Join(pkg, "testdata/fuzz", target.Name), nil } func (p *Project) Fuzz(ctx context.Context, target TestTarget, d time.Duration) error { @@ -83,7 +86,10 @@ func (p *Project) Fuzz(ctx context.Context, target TestTarget, d time.Duration) } scanner := bufio.NewScanner(&stdout) - corpusDirectory := p.relCorpusDir(target) + corpusDirectory, err := p.relCorpusDir(target) + if err != nil { + return fmt.Errorf("cannot locate relative corpus directory: %w", err) + } for scanner.Scan() { line := scanner.Text() From 9fabb1b5517f1d1d9a26177117c908b27e0a0bf3 Mon Sep 17 00:00:00 2001 From: Maciej Mionskowski Date: Mon, 27 Nov 2023 10:11:32 +0100 Subject: [PATCH 6/9] feat: lookup `go` in PATH before executing to be able to bail out early and provide useful information --- fuzz/fuzz.go | 8 ++++++-- fuzz/targets.go | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go index 6358407..f2db78b 100644 --- a/fuzz/fuzz.go +++ b/fuzz/fuzz.go @@ -62,7 +62,11 @@ func (p *Project) Fuzz(ctx context.Context, target TestTarget, d time.Duration) target.Package, } - cmd := exec.CommandContext(ctx, "go", args...) + goBin, err := exec.LookPath("go") + if err != nil { + return errors.New("go is not installed") + } + cmd := exec.CommandContext(ctx, goBin, args...) if p.Directory != "" { cmd.Dir = p.Directory } @@ -75,7 +79,7 @@ func (p *Project) Fuzz(ctx context.Context, target TestTarget, d time.Duration) } cmd.Stderr = os.Stderr - err := cmd.Run() + err = cmd.Run() if err == nil { return nil } diff --git a/fuzz/targets.go b/fuzz/targets.go index a904bee..afc6c24 100644 --- a/fuzz/targets.go +++ b/fuzz/targets.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "go/scanner" "go/token" @@ -60,7 +61,11 @@ func (p *Project) listPackages(ctx context.Context, packages ...string) ([]Packa "-json", }, packages...) - cmd := exec.CommandContext(ctx, "go", args...) + goBin, err := exec.LookPath("go") + if err != nil { + return nil, errors.New("go is not installed") + } + cmd := exec.CommandContext(ctx, goBin, args...) if p.Directory != "" { cmd.Dir = p.Directory } From ead4f4017af03e8ff5f7fc781edc234b5594a728 Mon Sep 17 00:00:00 2001 From: Maciej Mionskowski Date: Mon, 27 Nov 2023 10:15:46 +0100 Subject: [PATCH 7/9] chore: get rid of manual TempDir creation in favor of t.TempDir() --- fuzz/corpus_test.go | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/fuzz/corpus_test.go b/fuzz/corpus_test.go index 17adbad..297cb4a 100644 --- a/fuzz/corpus_test.go +++ b/fuzz/corpus_test.go @@ -4,7 +4,6 @@ import ( "context" "github.com/stretchr/testify/assert" "io/fs" - "os" "path/filepath" "testing" ) @@ -36,19 +35,9 @@ func TestCorpusExtract(t *testing.T) { t.Run("extracts fuzzing targets", func(t *testing.T) { project := Project{Directory: "./testdata/corpus/multiple"} ctx := context.Background() + tempDir := t.TempDir() - tempDir, err := os.MkdirTemp("", "go-ci-fuzz-*") - if err != nil { - t.Fatal(err) - } - - t.Cleanup(func() { - if os.RemoveAll(tempDir) != nil { - t.Error(err) - } - }) - - err = project.CorpusExtract(ctx, tempDir, "...") + err := project.CorpusExtract(ctx, tempDir, "...") if !assert.NoError(t, err, "corpus copying should not fail") { return } @@ -67,15 +56,7 @@ func TestCorpusExtract(t *testing.T) { func TestCorpusDelete(t *testing.T) { t.Run("deletes corpora that have matching target", func(t *testing.T) { - tempDir, err := os.MkdirTemp("", "go-ci-fuzz-*") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - if os.RemoveAll(tempDir) != nil { - t.Error(err) - } - }) + tempDir := t.TempDir() if err := copyDirectory(tempDir, "./testdata/corpus/multiple"); err != nil { t.Fatal(err) From 79c2a9d32190e79aece8b48fbd98b7e6b1d33a27 Mon Sep 17 00:00:00 2001 From: Maciej Mionskowski Date: Mon, 27 Nov 2023 10:18:10 +0100 Subject: [PATCH 8/9] chore: rename TestTarget to Target --- fuzz/fuzz.go | 4 ++-- fuzz/fuzz_test.go | 8 ++++---- fuzz/targets.go | 14 +++++++------- fuzz/targets_test.go | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go index f2db78b..10a6751 100644 --- a/fuzz/fuzz.go +++ b/fuzz/fuzz.go @@ -43,7 +43,7 @@ func (f FailingInputError) Error() string { return fmt.Sprintf("failing %s input: %s", newOrSeed, f.ID) } -func (p *Project) relCorpusDir(target TestTarget) (string, error) { +func (p *Project) relCorpusDir(target Target) (string, error) { // target.Package contains the root package as well // we need to strip it because it refers to the current working directory. pkg, err := filepath.Rel(target.RootPackage, target.Package) @@ -53,7 +53,7 @@ func (p *Project) relCorpusDir(target TestTarget) (string, error) { return filepath.Join(pkg, "testdata/fuzz", target.Name), nil } -func (p *Project) Fuzz(ctx context.Context, target TestTarget, d time.Duration) error { +func (p *Project) Fuzz(ctx context.Context, target Target, d time.Duration) error { args := []string{ "test", "-test.run=^$", diff --git a/fuzz/fuzz_test.go b/fuzz/fuzz_test.go index 720f166..c281846 100644 --- a/fuzz/fuzz_test.go +++ b/fuzz/fuzz_test.go @@ -15,7 +15,7 @@ func TestFuzz(t *testing.T) { ctx := context.Background() p := Project{Directory: "./testdata/fuzzing/seed", Quiet: true} - err := p.Fuzz(ctx, TestTarget{ + err := p.Fuzz(ctx, Target{ Name: "FuzzTarget", Package: "seed", RootPackage: "seed", @@ -28,7 +28,7 @@ func TestFuzz(t *testing.T) { ctx := context.Background() p := Project{Directory: "./testdata/fuzzing/seedfile", Quiet: true} - err := p.Fuzz(ctx, TestTarget{ + err := p.Fuzz(ctx, Target{ Name: "FuzzTarget", Package: "seedfile", RootPackage: "seedfile", @@ -50,7 +50,7 @@ func TestFuzz(t *testing.T) { removeTestData() t.Cleanup(removeTestData) - err := p.Fuzz(ctx, TestTarget{ + err := p.Fuzz(ctx, Target{ Name: "FuzzTarget", Package: "github.com/form3tech-oss/new", RootPackage: "github.com/form3tech-oss/new", @@ -67,7 +67,7 @@ func TestFuzz(t *testing.T) { ctx := context.Background() p := Project{Directory: "./testdata/fuzzing/nofindings", Quiet: true} - err := p.Fuzz(ctx, TestTarget{ + err := p.Fuzz(ctx, Target{ Name: "FuzzTarget", Package: "nofindings", RootPackage: "nofindings", diff --git a/fuzz/targets.go b/fuzz/targets.go index afc6c24..43f9d2b 100644 --- a/fuzz/targets.go +++ b/fuzz/targets.go @@ -14,17 +14,17 @@ import ( "regexp" ) -type TestTarget struct { +type Target struct { Name string Package string RootPackage string } -func (tt TestTarget) String() string { - return fmt.Sprintf("%s#%s", tt.Package, tt.Name) +func (t Target) String() string { + return fmt.Sprintf("%s#%s", t.Package, t.Name) } -func (p *Project) ListFuzzTargets(ctx context.Context, packages ...string) ([]TestTarget, error) { +func (p *Project) ListFuzzTargets(ctx context.Context, packages ...string) ([]Target, error) { relativePackages := make([]string, len(packages)) for i, pkg := range packages { // NOTE: Go will look in GOPATH for packages that cannot be found in the current module which is slow and breaks the tool @@ -91,7 +91,7 @@ func (p *Project) listPackages(ctx context.Context, packages ...string) ([]Packa // We cannot use go test -list because of this bug: https://github.com/golang/go/issues/25339 // So we list all packages and test files and look for test targets ourselves by running go.Scanner -func (p *Project) listTestTargets(ctx context.Context, pattern string, packages ...string) ([]TestTarget, error) { +func (p *Project) listTestTargets(ctx context.Context, pattern string, packages ...string) ([]Target, error) { pkgs, err := p.listPackages(ctx, packages...) if err != nil { return nil, fmt.Errorf("error listing packages: %w", err) @@ -102,7 +102,7 @@ func (p *Project) listTestTargets(ctx context.Context, pattern string, packages return nil, fmt.Errorf("error compiling pattern") } - var targets []TestTarget + var targets []Target for _, pkg := range pkgs { var testFiles []string @@ -130,7 +130,7 @@ func (p *Project) listTestTargets(ctx context.Context, pattern string, packages } if previousToken == token.FUNC && tok.IsLiteral() { if pat.MatchString(lit) { - targets = append(targets, TestTarget{ + targets = append(targets, Target{ Name: lit, Package: pkg.ImportPath, RootPackage: pkg.Module.Path, diff --git a/fuzz/targets_test.go b/fuzz/targets_test.go index e2ef89a..da0be46 100644 --- a/fuzz/targets_test.go +++ b/fuzz/targets_test.go @@ -12,7 +12,7 @@ func TestDiscoverTargets(t *testing.T) { ctx := context.Background() targets, err := p.ListFuzzTargets(ctx, ".") assert.NoError(t, err) - assert.EqualValues(t, []TestTarget{{ + assert.EqualValues(t, []Target{{ Name: "FuzzTarget", Package: "discover", RootPackage: "discover", @@ -23,7 +23,7 @@ func TestDiscoverTargets(t *testing.T) { ctx := context.Background() targets, err := p.ListFuzzTargets(ctx, "...") assert.NoError(t, err) - assert.ElementsMatch(t, []TestTarget{{ + assert.ElementsMatch(t, []Target{{ Name: "FuzzTarget", Package: "discover", RootPackage: "discover", @@ -48,7 +48,7 @@ func TestDiscoverTargets(t *testing.T) { ctx := context.Background() targets, err := p.ListFuzzTargets(ctx, "subpackage") assert.NoError(t, err) - assert.ElementsMatch(t, []TestTarget{{ + assert.ElementsMatch(t, []Target{{ Name: "FuzzSubTarget", Package: "discover/subpackage", RootPackage: "discover", @@ -60,7 +60,7 @@ func TestDiscoverTargets(t *testing.T) { ctx := context.Background() targets, err := p.ListFuzzTargets(ctx, "...") assert.NoError(t, err) - assert.ElementsMatch(t, []TestTarget{{ + assert.ElementsMatch(t, []Target{{ Name: "FuzzTarget", Package: "discovermain", RootPackage: "discovermain", From 77f9b0e59836f44d3c6bd5194393874eeecaff33 Mon Sep 17 00:00:00 2001 From: "simon.caplette" Date: Fri, 24 Nov 2023 09:46:53 +0100 Subject: [PATCH 9/9] chore: adjust README --- README.md | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c71dc75..6cef460 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,20 @@ # Go CI Fuzz -`go-ci-fuzz` is a CLI and a set of GitHub Actions that help you run [Native Go Fuzz Tests](https://go.dev/security/fuzz/) as part of your CI. +CLI and set of GitHub Actions to help you run [Native Go Fuzz Tests](https://go.dev/security/fuzz/) in CI. -It's a light wrapper around `go test -fuzz=` that supports multiple test targets. +It's a light wrapper around `go test -fuzz=` supporting multiple test targets. ## Motivation -`go-ci-fuzz` was created to achieve a developer friendly and lightweight way of running Native Go Fuzz Tests in Continuous Integration pipelines. -Current alternatives (ClusterFuzzLite, go-fuzz) don't support Native Go Fuzzing or support Native Go fuzzing inadequately through wrappers. +This project was created to achieve a developer friendly and lightweight way of running _Native Go Fuzz Tests_ in Continuous Integration pipelines. -## Typical Workflow +It implements missing functionalities in 'go test -fuzz' such as +- run multiple test targets in a single command +- extract failed outputs + +Current alternatives (ClusterFuzzLite, go-fuzz, etc.) don't support _Native Go Fuzzing_ or only inadequately through wrappers. + +## Workflow ```mermaid flowchart LR @@ -19,27 +24,26 @@ flowchart LR engineer -- 4. Commits failing inputs & fixes the issue --> GitHub ``` -## Running locally +## Run + +### Locally + +Although this tool is meant for CI pipelines, it's still useful in local development. -Although this tool is mostly meant to be used in CI pipelines it's still useful in local development. -If your project has many fuzz tests you can run all of them using `go-ci-fuzz`. +If your project has many fuzz tests you can run all of them with: -```bash +```shell go install github.com/form3tech-oss/go-ci-fuzz@{version} go-ci-fuzz fuzz --fuzz-time 10m [--out /tmp/failures] ``` -## Run as part of GitHub Actions +### As GitHub Action -This repo contains reusable actions located in [./ci/github-actions](ci/github-actions). -You can reference these actions from your workflows. +From your own workflow, you can reference our reusable Github actions located in [./ci/github-actions](ci/github-actions). -All findings will be uploaded as artifacts to the workflow run. +All fuzz findings are uploaded as artifacts to the workflow run. -Below you'll find example workflows to incorporate in your CI pipeline. -Feel free to adjust the fuzz-time according to your appetite. - -The Github Action accepts the following properties: +Here are the Github Action properties: ```yaml inputs: @@ -65,6 +69,10 @@ inputs: default: "failing-inputs" ``` +## Examples + +Here are some example workflows to incorporate in your CI pipelines. Feel free to adjust the fuzz-time according to your appetite! + ### Fuzz incoming Pull Requests ```yaml