Add additional catalog indexes for performance (anchore#154)

* add additional catalog indexes for performance Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * [wip] link resolution Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add leaf link resolution on tree responses (defer ancestor link resolution) Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add filetree search context Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add tests for new search context object Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * remove unused tar header fields from file.Metadata struct Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * use singular file type definitions Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add logging for filetree searches Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add limited support for glob classes and alternatives Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add failing test to show that index shortcircuits correct behavior Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add link resolution via filetree search context Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * allow index symlink resolution to function through cycles Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add tests for filetree.Index Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add search by parent basename and fix requirements filtering Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * sort search results Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * change file.Type to int + fix layer 0 squashed search context Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * more cleanup Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * switch to generic set implementation Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update linter Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * replace generic set implemetation with plain set (unstable in go1.19) Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * introduce filtree builter and foster usage of reader interfaces Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * rename content helper functions Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update docs with background Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix get_xid for cross compilation Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * upgrade CI validations workflow Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix snapshot builds Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add tests for file.Index.GetByFileType Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * rename file.Type and file.Resolution Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * ensure that glob results match search facade Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * replace stringset implementation + move resolution tests Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add note about podman dependency for testing Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * address PR comments Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * remove extra whitespace Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * constrain OS build support Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update/remove TODO comments Signed-off-by: Alex Goodman <alex.goodman@anchore.com> --------- Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
deepfence · Jun 15, 2023 · 61a4559 · 61a4559
1 parent 178bc5c
commit 61a4559
Show file tree

Hide file tree

Showing 72 changed files with 8,047 additions and 1,142 deletions.
diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml
@@ -0,0 +1,80 @@
+name: "Bootstrap"
+description: "Bootstrap all tools and dependencies"
+inputs:
+  go-version:
+    description: "Go version to install"
+    required: true
+    default: "1.19.x"
+  use-go-cache:
+    description: "Restore go cache"
+    required: true
+    default: "true"
+  cache-key-prefix:
+    description: "Prefix all cache keys with this value"
+    required: true
+    default: "831180ac25"
+  build-cache-key-prefix:
+    description: "Prefix build cache key with this value"
+    required: true
+    default: "f8b6d31dea"
+  bootstrap-apt-packages:
+    description: "Space delimited list of tools to install via apt"
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-go@v3
+      with:
+        go-version: ${{ inputs.go-version }}
+
+    - name: Restore tool cache
+      id: tool-cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ github.workspace }}/.tmp
+        key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-tool-${{ hashFiles('Makefile') }}
+
+    # note: we need to keep restoring the go mod cache before bootstrapping tools since `go install` is used in
+    # some installations of project tools.
+    - name: Restore go module cache
+      id: go-mod-cache
+      if: inputs.use-go-cache == 'true'
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/go/pkg/mod
+        key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ inputs.cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}-
+
+    - name: (cache-miss) Bootstrap project tools
+      shell: bash
+      if: steps.tool-cache.outputs.cache-hit != 'true'
+      run: make bootstrap-tools
+
+    - name: Restore go build cache
+      id: go-cache
+      if: inputs.use-go-cache == 'true'
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/go-build
+        key: ${{ inputs.cache-key-prefix }}-${{ inputs.build-cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ inputs.cache-key-prefix }}-${{ inputs.build-cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}-
+
+    - name: (cache-miss) Bootstrap go dependencies
+      shell: bash
+      if: steps.go-mod-cache.outputs.cache-hit != 'true' && inputs.use-go-cache == 'true'
+      run: make bootstrap-go
+
+    - name: Bootstrap CI dependencies
+      shell: bash
+      run: make ci-bootstrap
+
+    - name: Install apt packages
+      if: inputs.bootstrap-apt-packages != ''
+      shell: bash
+      run: |
+        DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y ${{ inputs.bootstrap-apt-packages }}
diff --git a/.github/scripts/build.sh b/.github/scripts/build.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+SNAPSHOT_DIR=$1
+
+# Based on https://gist.github.com/eduncan911/68775dba9d3c028181e4 and https://gist.github.com/makeworld-the-better-one/e1bb127979ae4195f43aaa3ad46b1097
+# but improved to use the `go` command so it never goes out of date.
+
+type setopt >/dev/null 2>&1
+
+contains() {
+    # Source: https://stackoverflow.com/a/8063398/7361270
+    [[ $1 =~ (^|[[:space:]])$2($|[[:space:]]) ]]
+}
+
+mkdir -p "${SNAPSHOT_DIR}"
+
+BUILD_TARGET=./examples
+OUTPUT=${SNAPSHOT_DIR}/stereoscope-example
+FAILURES=""
+
+# You can set your own flags on the command line
+FLAGS=${FLAGS:-"-ldflags=\"-s -w\""}
+
+# A list of OSes and architectures to not build for, space-separated
+# It can be set from the command line when the script is called.
+NOT_ALLOWED_OS=${NOT_ALLOWED_OS:-"js android ios solaris illumos aix dragonfly plan9 freebsd openbsd netbsd"}
+NOT_ALLOWED_ARCH=${NOT_ALLOWED_ARCH:-"riscv64 mips mips64 mips64le ppc64 ppc64le s390x wasm"}
+
+
+# Get all targets
+while IFS= read -r target; do
+    GOOS=${target%/*}
+    GOARCH=${target#*/}
+    BIN_FILENAME="${OUTPUT}-${GOOS}-${GOARCH}"
+
+    if contains "$NOT_ALLOWED_OS" "$GOOS" ; then
+        continue
+    fi
+
+    if contains "$NOT_ALLOWED_ARCH" "$GOARCH" ; then
+        continue
+    fi
+
+    # Check for arm and set arm version
+    if [[ $GOARCH == "arm" ]]; then
+        # Set what arm versions each platform supports
+        if [[ $GOOS == "darwin" ]]; then
+            arms="7"
+        elif [[ $GOOS == "windows" ]]; then
+             # This is a guess, it's not clear what Windows supports from the docs
+             # But I was able to build all these on my machine
+            arms="5 6 7"
+        elif [[ $GOOS == *"bsd"  ]]; then
+            arms="6 7"
+        else
+            # Linux goes here
+            arms="5 6 7"
+        fi
+
+        # Now do the arm build
+        for GOARM in $arms; do
+            BIN_FILENAME="${OUTPUT}-${GOOS}-${GOARCH}${GOARM}"
+            if [[ "${GOOS}" == "windows" ]]; then BIN_FILENAME="${BIN_FILENAME}.exe"; fi
+            CMD="GOARM=${GOARM} GOOS=${GOOS} GOARCH=${GOARCH} go build $FLAGS -o ${BIN_FILENAME} ${BUILD_TARGET}"
+            echo "${CMD}"
+            eval "${CMD}" || FAILURES="${FAILURES} ${GOOS}/${GOARCH}${GOARM}"
+        done
+    else
+        # Build non-arm here
+        if [[ "${GOOS}" == "windows" ]]; then BIN_FILENAME="${BIN_FILENAME}.exe"; fi
+        CMD="GOOS=${GOOS} GOARCH=${GOARCH} go build $FLAGS -o ${BIN_FILENAME} ${BUILD_TARGET}"
+        echo "${CMD}"
+        eval "${CMD}" || FAILURES="${FAILURES} ${GOOS}/${GOARCH}"
+    fi
+done <<< "$(go tool dist list)"
+
+if [[ "${FAILURES}" != "" ]]; then
+    echo ""
+    echo "build failed for: ${FAILURES}"
+    exit 1
+fi
diff --git a/.github/scripts/coverage.py b/.github/scripts/coverage.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+import shlex
+
+
+class bcolors:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKCYAN = '\033[96m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+
+if len(sys.argv) < 3:
+    print("Usage: coverage.py [threshold] [go-coverage-report]")
+    sys.exit(1)
+
+
+threshold = float(sys.argv[1])
+report = sys.argv[2]
+
+
+args = shlex.split(f"go tool cover -func {report}")
+p = subprocess.run(args, capture_output=True, text=True)
+
+percent_coverage = float(p.stdout.splitlines()[-1].split()[-1].replace("%", ""))
+print(f"{bcolors.BOLD}Coverage: {percent_coverage}%{bcolors.ENDC}")
+
+if percent_coverage < threshold:
+    print(f"{bcolors.BOLD}{bcolors.FAIL}Coverage below threshold of {threshold}%{bcolors.ENDC}")
+    sys.exit(1)
diff --git a/.github/scripts/go-mod-tidy-check.sh b/.github/scripts/go-mod-tidy-check.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -eu
+
+ORIGINAL_STATE_DIR=$(mktemp -d "TEMP-original-state-XXXXXXXXX")
+TIDY_STATE_DIR=$(mktemp -d "TEMP-tidy-state-XXXXXXXXX")
+
+trap "cp -p ${ORIGINAL_STATE_DIR}/* ./ && git update-index -q --refresh && rm -fR ${ORIGINAL_STATE_DIR} ${TIDY_STATE_DIR}" EXIT
+
+# capturing original state of files...
+cp go.mod go.sum "${ORIGINAL_STATE_DIR}"
+
+# capturing state of go.mod and go.sum after running go mod tidy...
+go mod tidy
+cp go.mod go.sum "${TIDY_STATE_DIR}"
+
+set +e
+
+# detect difference between the git HEAD state and the go mod tidy state
+DIFF_MOD=$(diff -u "${ORIGINAL_STATE_DIR}/go.mod" "${TIDY_STATE_DIR}/go.mod")
+DIFF_SUM=$(diff -u "${ORIGINAL_STATE_DIR}/go.sum" "${TIDY_STATE_DIR}/go.sum")
+
+if [[ -n "${DIFF_MOD}" || -n "${DIFF_SUM}" ]]; then
+    echo "go.mod diff:"
+    echo "${DIFF_MOD}"
+    echo "go.sum diff:"
+    echo "${DIFF_SUM}"
+    echo ""
+    printf "FAILED! go.mod and/or go.sum are NOT tidy; please run 'go mod tidy'.\n\n"
+    exit 1
+fi
diff --git a/.github/workflows/benchmark-testing.yaml b/.github/workflows/benchmark-testing.yaml
@@ -0,0 +1,58 @@
+name: "Benchmark testing"
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+
+  Benchmark-Test:
+    name: "Benchmark tests"
+    runs-on: ubuntu-20.04
+    # note: we want benchmarks to run on pull_request events in order to publish results to a sticky comment, and
+    # we also want to run on push such that merges to main are recorded to the cache. For this reason we don't filter
+    # the job by event.
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Bootstrap environment
+        uses: ./.github/actions/bootstrap
+
+      - name: Restore base benchmark result
+        uses: actions/cache@v3
+        with:
+          path: test/results/benchmark-main.txt
+          # use base sha for PR or new commit hash for main push in benchmark result key
+          key: ${{ runner.os }}-bench-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }}
+
+      - name: Run benchmark tests
+        id: benchmark
+        run: |
+          REF_NAME=${GITHUB_REF##*/} make benchmark
+          OUTPUT=$(make show-benchstat)
+          OUTPUT="${OUTPUT//'%'/'%25'}"   # URL encode all '%' characters
+          OUTPUT="${OUTPUT//$'\n'/'%0A'}" # URL encode all '\n' characters
+          OUTPUT="${OUTPUT//$'\r'/'%0D'}" # URL encode all '\r' characters
+          echo "::set-output name=result::$OUTPUT"
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: benchmark-test-results
+          path: test/results/**/*
+
+      - name: Update PR benchmark results comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        continue-on-error: true
+        with:
+          header: benchmark
+          message: |
+            ### Benchmark Test Results
+
+            <details>
+              <summary>Benchmark results from the latest changes vs base branch</summary>
+
+            ```
+            ${{ steps.benchmark.outputs.result }}
+            ```
+
+            </details>