Merge branch 'main' into package-private

* main: Update syft bootstrap tools to latest versions. (anchore#1171) Fix update-bootstrap-tools workflow (anchore#1170) workflow to create automated PRs to update bootstrap tools (anchore#1167) feat: add support for licenses in package-lock json v2 (anchore#1164) External sources configuration (anchore#1158) feat: add support for pnpm (anchore#1166) Prevent symlinks causing duplicate package-file relationships (anchore#1168) Associate node package licenses from node_modules (anchore#1152) Give the contributing guide a substantial rework (anchore#1155) Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>
scothis · Aug 24, 2022 · 01836fa · 01836fa
2 parents 6d6078a + b0fc955
commit 01836fa
Show file tree

Hide file tree

Showing 51 changed files with 1,083 additions and 196 deletions.
diff --git a/.github/workflows/update-bootstrap-tools.yml b/.github/workflows/update-bootstrap-tools.yml
@@ -0,0 +1,75 @@
+name: PR for latest versions of bootstrap tools
+on:
+  schedule:
+    - cron: "0 8 * * *" # 3 AM EST
+
+  workflow_dispatch:
+
+env:
+  GO_VERSION: "1.18.x"
+  GO_STABLE_VERSION: true
+
+jobs:
+  update-bootstrap-tools:
+    runs-on: ubuntu-latest
+    if: github.repository == 'anchore/syft' # only run for main repo
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          stable: ${{ env.GO_STABLE_VERSION }}
+
+      - run: |
+          GOLANGCILINT_LATEST_VERSION=$(go list -m -json github.com/golangci/golangci-lint@latest 2>/dev/null | jq -r '.Version')
+          BOUNCER_LATEST_VERSION=$(go list -m -json github.com/wagoodman/go-bouncer@latest 2>/dev/null | jq -r '.Version')
+          CHRONICLE_LATEST_VERSION=$(go list -m -json github.com/anchore/chronicle@latest 2>/dev/null | jq -r '.Version')
+          GORELEASER_LATEST_VERSION=$(go list -m -json github.com/goreleaser/goreleaser@latest 2>/dev/null | jq -r '.Version')
+          YAJSV_LATEST_VERSION=$(go list -m -json github.com/neilpa/yajsv@latest 2>/dev/null | jq -r '.Version')
+          COSIGN_LATEST_VERSION=$(go list -m -json github.com/sigstore/cosign@latest 2>/dev/null | jq -r '.Version')
+          
+          # update version variables in the Makefile
+          sed -r -i -e 's/^(GOLANGCILINT_VERSION = ).*/\1'${GOLANGCILINT_LATEST_VERSION}'/' Makefile
+          sed -r -i -e 's/^(BOUNCER_VERSION = ).*/\1'${BOUNCER_LATEST_VERSION}'/' Makefile
+          sed -r -i -e 's/^(CHRONICLE_VERSION = ).*/\1'${CHRONICLE_LATEST_VERSION}'/' Makefile
+          sed -r -i -e 's/^(GORELEASER_VERSION = ).*/\1'${GORELEASER_LATEST_VERSION}'/' Makefile
+          sed -r -i -e 's/^(YAJSV_VERSION = ).*/\1'${YAJSV_LATEST_VERSION}'/' Makefile
+          sed -r -i -e 's/^(COSIGN_VERSION = ).*/\1'${COSIGN_LATEST_VERSION}'/' Makefile
+          
+          # update cosign in go.mod as well
+          go get github.com/sigstore/cosign@$COSIGN_LATEST_VERSION
+          go mod tidy
+          
+          # export the versions for use with create-pull-request
+          echo "::set-output name=GOLANGCILINT::$GOLANGCILINT_LATEST_VERSION"
+          echo "::set-output name=BOUNCER::$BOUNCER_LATEST_VERSION"
+          echo "::set-output name=CHRONICLE::$CHRONICLE_LATEST_VERSION"
+          echo "::set-output name=GORELEASER::$GORELEASER_LATEST_VERSION"
+          echo "::set-output name=YAJSV::$YAJSV_LATEST_VERSION"
+          echo "::set-output name=COSIGN::$COSIGN_LATEST_VERSION"
+        id: latest-versions
+
+      - uses: tibdex/github-app-token@v1
+        id: generate-token
+        with:
+          app_id: ${{ secrets.TOKEN_APP_ID }}
+          private_key: ${{ secrets.TOKEN_APP_PRIVATE_KEY }}
+
+      - uses: peter-evans/create-pull-request@v4
+        with:
+          signoff: true
+          delete-branch: true
+          branch: auto/latest-bootstrap-tools
+          labels: dependencies
+          commit-message: 'Update syft bootstrap tools to latest versions.'
+          title: 'Update syft bootstrap tools to latest versions.'
+          body: |
+            - [golangci-lint ${{ steps.latest-versions.outputs.GOLANGCILINT }}](https://github.com/golangci/golangci-lint/releases/tag/${{ steps.latest-versions.outputs.GOLANGCILINT }})
+            - [bouncer ${{ steps.latest-versions.outputs.BOUNCER }}](https://github.com/wagoodman/go-bouncer/releases/tag/${{ steps.latest-versions.outputs.BOUNCER }})
+            - [chronicle ${{ steps.latest-versions.outputs.CHRONICLE }}](https://github.com/anchore/chronicle/releases/tag/${{ steps.latest-versions.outputs.CHRONICLE }})
+            - [goreleaser ${{ steps.latest-versions.outputs.GORELEASER }}](https://github.com/goreleaser/goreleaser/releases/tag/${{ steps.latest-versions.outputs.GORELEASER }})
+            - [yajsv ${{ steps.latest-versions.outputs.YAJSV }}](https://github.com/neilpa/yajsv/releases/tag/${{ steps.latest-versions.outputs.YAJSV }})
+            - [cosign ${{ steps.latest-versions.outputs.COSIGN }}](https://github.com/sigstore/cosign/releases/tag/${{ steps.latest-versions.outputs.COSIGN }})  
+            This is an auto-generated pull request to update all of the bootstrap tools to the latest versions.
+          token: ${{ steps.generate-token.outputs.token }}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -2,6 +2,57 @@
 
 If you are looking to contribute to this project and want to open a GitHub pull request ("PR"), there are a few guidelines of what we are looking for in patches. Make sure you go through this document and ensure that your code proposal is aligned.
 
+## Setting up your environment
+
+Before you can contribute to Syft, you need to configure your development environment.
+
+### Debian setup
+
+You will need to install Go. The version on https://go.dev works best, using the system golang doesn't always work the way you might expect.
+
+At the time of writing, Go 1.19 does not work correctly with Syft. Please use Go 1.18 for now.
+
+You will also need Docker. There's no reason the system packages shouldn't work, but we used the official Docker package. You can find instructions for installing Docker in Debian [here](https://docs.docker.com/engine/install/debian/).
+
+You also need to install some Debian packages
+
+```sh
+sudo apt-get install build-essential zip bc libxml2-utils git
+```
+
+## Configuring Git
+
+You will need to configure your git client with your name and email address. This is easily done from the command line.
+
+```text
+$ git config --global user.name "John Doe"
+$ git config --global user.email "john.doe@example.com"
+```
+
+This username and email address will matter later in this guide.
+
+## Fork the repo
+
+You should fork the Syft repo using the "Fork" button at the top right of the Syft GitHub [site](https://github.com/anchore/syft/). You will be doing your development in your fork, then submit a pull request to Syft. There are many resources how to use GitHub effectively, we will not cover those here.
+
+## Adding a feature or fix
+
+If you look at the Syft [Issue](https://github.com/anchore/syft/issues) there are plenty of bugs and feature requests. Maybe look at the [good first issue](https://github.com/anchore/syft/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) list if you're not sure where to start.
+
+## Commit guidelines
+
+In the Syft project we like commits and pull requests (PR) to be easy to understand and review. Open source thrives best when everything happening is over documented and small enough to be understood.
+
+### Granular commits
+
+Please try to make every commit as simple as possible, but no simpler. The idea is that each commit should be a logical unit of code. Try not to commit too many tiny changes, for example every line changed in a file as a separate commit. And also try not to make a commit enormous, for example committing all your work at the end of the day.
+
+Rather than try to follow a strict guide on what is or is not best, we try to be flexible and simple in this space. Do what makes the most sense for the changes you are trying to include.
+
+### Commit title and description
+
+Remember that the message you leave for a commit is for the reviewer in the present, and for someone (maybe you) changing something in the future. Please make sure the title and description used is easy to understand and explains what was done. Jokes and clever comments generally don't age well in commit messages. Just the facts please.
+
 ## Sign off your work
 
 The `sign-off` is an added line at the end of the explanation for the commit, certifying that you wrote it or otherwise have the right to submit it as an open-source patch. By submitting a contribution, you agree to be bound by the terms of the DCO Version 1.1 and Apache License Version 2.0.
@@ -44,13 +95,6 @@ When committing your change, you can add the required line manually so that it l
 Signed-off-by: John Doe <john.doe@example.com>
 ```
 
-Alternatively, configure your Git client with your name and email to use the `-s` flag when creating a commit:
-
-```text
-$ git config --global user.name "John Doe"
-$ git config --global user.email "john.doe@example.com"
-```
-
 Creating a signed-off commit is then possible with `-s` or `--signoff`:
 
 ```text
@@ -70,21 +114,9 @@ Date:   Mon Aug 1 11:27:13 2020 -0400
     Signed-off-by: John Doe <john.doe@example.com>
 ```
 
-
-[//]: # (TODO: Commit guidelines, granular commits)
-
-
-[//]: # (TODO: Commit guidelines, descriptive messages)
-
-
-[//]: # (TODO: Commit guidelines, commit title, extra body description)
-
-
-[//]: # (TODO: PR title and description)
-
 ## Test your changes
 
-This project has a `Makefile` which includes many helpers running both unit and integration tests. Although PRs will have automatic checks for these, it is useful to run them locally, ensuring they pass before submitting changes. Ensure you've bootstrapped once before running tests:
+This project has a `Makefile` which includes many helpers running both unit and integration tests. You can run `make help` to see all the options. Although PRs will have automatic checks for these, it is useful to run them locally, ensuring they pass before submitting changes. Ensure you've bootstrapped once before running tests:
 
 ```text
 $ make bootstrap
@@ -97,6 +129,26 @@ $ make unit
 $ make integration
 ```
 
+You can also run `make all` to run a more extensive test suite, but there is additional configuration that will be needed for those tests to run correctly. We will not cover the extra steps here.
+
+## Pull Request
+
+If you made it this far and all the tests are passing, it's time to submit a Pull Request (PR) for Syft. Submitting a PR is always a scary moment as what happens next can be an unknown. The Syft project strives to be easy to work with, we appreciate all contributions. Nobody is going to yell at you or try to make you feel bad. We love contributions and know how scary that first PR can be.
+
+### PR Title and Description
+
+Just like the commit title and description mentioned above, the PR title and description is very important for letting others know what's happening. Please include any details you think a reviewer will need to more properly review your PR.
+
+A PR that is very large or poorly described has a higher likelihood of being pushed to the end of the list. Reviewers like PRs they can understand and quickly review.
+
+### What to expect next
+
+Please be patient with the project. We try to review PRs in a timely manner, but this is highly dependent on all the other tasks we have going on. It's OK to ask for a status update every week or two, it's not OK to ask for a status update every day.
+
+It's very likely the reviewer will have questions and suggestions for changes to your PR. If your changes don't match the current style and flow of the other code, expect a request to change what you've done.
+
 ## Document your changes
 
-When proposed changes are modifying user-facing functionality or output, it is expected the PR will include updates to the documentation as well.
+And lastly, when proposed changes are modifying user-facing functionality or output, it is expected the PR will include updates to the documentation as well. Syft is not a project that is heavy on documentation. This will mostly be updating the README and help for the tool.
+
+If nobody knows new features exist, they can't use them!
diff --git a/Makefile b/Makefile
@@ -9,6 +9,12 @@ SNAPSHOT_CMD=$(RELEASE_CMD) --skip-publish --snapshot
 VERSION=$(shell git describe --dirty --always --tags)
 COMPARE_TEST_IMAGE = centos:8.2.2004
 COMPARE_DIR = ./test/compare
+GOLANGCILINT_VERSION = v1.48.0
+BOUNCER_VERSION = v0.4.0
+CHRONICLE_VERSION = v0.4.1
+GORELEASER_VERSION = v1.10.3
+YAJSV_VERSION = v1.4.0
+COSIGN_VERSION = v1.11.0
 
 # formatting variables
 BOLD := $(shell tput -T linux bold)
@@ -106,12 +112,12 @@ $(TEMPDIR):
 .PHONY: bootstrap-tools
 bootstrap-tools: $(TEMPDIR)
 	GO111MODULE=off GOBIN=$(shell realpath $(TEMPDIR)) go get -u golang.org/x/perf/cmd/benchstat
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.47.2
-	curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ v0.4.0
-	curl -sSfL https://raw.githubusercontent.com/anchore/chronicle/main/install.sh | sh -s -- -b $(TEMPDIR)/ v0.3.0
-	.github/scripts/goreleaser-install.sh -d -b $(TEMPDIR)/ v1.10.3
-	GOBIN="$(shell realpath $(TEMPDIR))" go install github.com/neilpa/yajsv@v1.4.0
-	GOBIN="$(shell realpath $(TEMPDIR))" go install github.com/sigstore/cosign/cmd/cosign@v1.10.0
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ $(GOLANGCILINT_VERSION)
+	curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ $(BOUNCER_VERSION)
+	curl -sSfL https://raw.githubusercontent.com/anchore/chronicle/main/install.sh | sh -s -- -b $(TEMPDIR)/ $(CHRONICLE_VERSION)
+	.github/scripts/goreleaser-install.sh -d -b $(TEMPDIR)/ $(GORELEASER_VERSION)
+	GOBIN="$(shell realpath $(TEMPDIR))" go install github.com/neilpa/yajsv@$(YAJSV_VERSION)
+	GOBIN="$(shell realpath $(TEMPDIR))" go install github.com/sigstore/cosign/cmd/cosign@$(COSIGN_VERSION)
 
 .PHONY: bootstrap-go
 bootstrap-go:

diff --git a/cmd/syft/cli/commands.go b/cmd/syft/cli/commands.go
@@ -31,7 +31,8 @@ const indent = "  "
 // at this level. Values from the config should only be used after `app.LoadAllValues` has been called.
 // Cobra does not have knowledge of the user provided flags until the `RunE` block of each command.
 // `RunE` is the earliest that the complete application configuration can be loaded.
-// nolint:funlen
+//
+//nolint:funlen
 func New() (*cobra.Command, error) {
 	app := &config.Application{}
 

diff --git a/cmd/syft/cli/eventloop/event_loop.go b/cmd/syft/cli/eventloop/event_loop.go
@@ -14,7 +14,8 @@ import (
 // eventLoop listens to worker errors (from execution path), worker events (from a partybus subscription), and
 // signal interrupts. Is responsible for handling each event relative to a given UI an to coordinate eventing until
 // an eventual graceful exit.
-// nolint:funlen
+//
+//nolint:funlen
 func EventLoop(workerErrs <-chan error, signals <-chan os.Signal, subscription *partybus.Subscription, cleanupFn func(), uxs ...ui.UI) error {
 	defer cleanupFn()
 	events := subscription.Events()

diff --git a/cmd/syft/cli/options/packages.go b/cmd/syft/cli/options/packages.go
@@ -26,6 +26,7 @@ type PackagesOptions struct {
 	OverwriteExistingImage bool
 	ImportTimeout          uint
 	Catalogers             []string
+	ExternalSourcesEnabled bool
 }
 
 var _ Interface = (*PackagesOptions)(nil)
@@ -70,9 +71,13 @@ func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error {
 	cmd.Flags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30,
 		"set a timeout duration (in seconds) for the upload to Anchore Enterprise")
 
+	cmd.Flags().BoolVarP(&o.ExternalSourcesEnabled, "external-sources-enabled", "", false,
+		"shut off any use of external sources during sbom generation (default false")
+
 	return bindPackageConfigOptions(cmd.Flags(), v)
 }
 
+//nolint:funlen
 func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 	// Formatting & Input options //////////////////////////////////////////////
 
@@ -104,6 +109,10 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 		return err
 	}
 
+	if err := v.BindPFlag("external_sources.external-sources-enabled", flags.Lookup("external-sources-enabled")); err != nil {
+		return err
+	}
+
 	// Upload options //////////////////////////////////////////////////////////
 
 	if err := v.BindPFlag("anchore.host", flags.Lookup("host")); err != nil {