From a8c1df29ddea4b09aeacd6596c5eeda6ff7ac70a Mon Sep 17 00:00:00 2001 From: Justin Law Date: Wed, 25 Sep 2024 16:34:24 -0400 Subject: [PATCH 01/41] WIP badge verification workflow --- .../workflows/made-for-uds-verification.yaml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/made-for-uds-verification.yaml diff --git a/.github/workflows/made-for-uds-verification.yaml b/.github/workflows/made-for-uds-verification.yaml new file mode 100644 index 000000000..af116287c --- /dev/null +++ b/.github/workflows/made-for-uds-verification.yaml @@ -0,0 +1,32 @@ +name: Made for UDS Badge Verification + +on: + pull_request: + branches: + - "main" + paths: + - "packages/" + - ".github/workflows/made-for-uds-verification.yaml" + +concurrency: + group: uds-badge-verification-${{ github.ref }} + cancel-in-progress: true + +jobs: + uds-badge-verification: + runs-on: ubuntu-latest + name: UDS Badge Verification + + permissions: + contents: read + + steps: + - name: Checkout Repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Verify API + uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + with: + chartPath: "" + groupName: "" + commonZarf: "" From 4ce0070a6e08bdf6ae2c3e335d8a37ff48e445b2 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Wed, 25 Sep 2024 16:50:37 -0400 Subject: [PATCH 02/41] add all packages --- .../workflows/made-for-uds-verification.yaml | 32 ------ .github/workflows/uds-verification.yaml | 97 +++++++++++++++++++ 2 files changed, 97 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/made-for-uds-verification.yaml create mode 100644 .github/workflows/uds-verification.yaml diff --git a/.github/workflows/made-for-uds-verification.yaml b/.github/workflows/made-for-uds-verification.yaml deleted file mode 100644 index af116287c..000000000 --- a/.github/workflows/made-for-uds-verification.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Made for UDS Badge Verification - -on: - pull_request: - branches: - - "main" - paths: - - "packages/" - - ".github/workflows/made-for-uds-verification.yaml" - -concurrency: - group: uds-badge-verification-${{ github.ref }} - cancel-in-progress: true - -jobs: - uds-badge-verification: - runs-on: ubuntu-latest - name: UDS Badge Verification - - permissions: - contents: read - - steps: - - name: Checkout Repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Verify API - uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - with: - chartPath: "" - groupName: "" - commonZarf: "" diff --git a/.github/workflows/uds-verification.yaml b/.github/workflows/uds-verification.yaml new file mode 100644 index 000000000..36bf4292f --- /dev/null +++ b/.github/workflows/uds-verification.yaml @@ -0,0 +1,97 @@ +name: Made for UDS Badge Verification + +on: + pull_request: + branches: + - "main" + paths: + - "packages/" + - ".github/workflows/uds-verification.yaml" + +concurrency: + group: uds-verification-${{ github.ref }} + cancel-in-progress: true + +jobs: + uds-verification: + runs-on: ubuntu-latest + name: Made for UDS Badge Verification + + permissions: + contents: read + + steps: + - name: Checkout Repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Setup UDS Environment + uses: defenseunicorns/uds-common/.github/actions/setup@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + with: + registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} + registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} + ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} + + - name: Verify API + continue-on-error: true + uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + with: + chartPath: "packages/api/chart" + groupName: "lfai" + commonZarf: "true" + + - name: Verify UI + continue-on-error: true + uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + with: + chartPath: "packages/ui/chart" + groupName: "lfai" + commonZarf: "false" + + - name: Verify vLLM + continue-on-error: true + uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + with: + chartPath: "packages/vllm/chart" + groupName: "lfai" + commonZarf: "false" + + - name: Verify LLaMA-CPP-Python + continue-on-error: true + uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + with: + chartPath: "packages/llama-cpp-python/chart" + groupName: "lfai" + commonZarf: "false" + + - name: Verify Whisper + continue-on-error: true + uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + with: + chartPath: "packages/whisper/chart" + groupName: "lfai" + commonZarf: "false" + + - name: Verify Text-Embeddings + continue-on-error: true + uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + with: + chartPath: "packages/text-embeddings/chart" + groupName: "lfai" + commonZarf: "false" + + - name: Verify Repeater + continue-on-error: true + uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + with: + chartPath: "packages/repeater/chart" + groupName: "lfai" + commonZarf: "false" + + - name: Verify Supabase + continue-on-error: true + uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 + with: + chartPath: "packages/supabase/chart" + groupName: "lfai" + commonZarf: "false" From f3afa074a87306fb8908a128d0e0e2ba67478448 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Wed, 25 Sep 2024 16:54:13 -0400 Subject: [PATCH 03/41] only download uds --- .github/workflows/uds-verification.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/uds-verification.yaml b/.github/workflows/uds-verification.yaml index 36bf4292f..09f16eddf 100644 --- a/.github/workflows/uds-verification.yaml +++ b/.github/workflows/uds-verification.yaml @@ -26,11 +26,6 @@ jobs: - name: Setup UDS Environment uses: defenseunicorns/uds-common/.github/actions/setup@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - with: - registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} - registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} - ghToken: ${{ secrets.GITHUB_TOKEN }} - chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - name: Verify API continue-on-error: true From 4702bddc95c35f04b6fb4bdbaea32904d7f8122b Mon Sep 17 00:00:00 2001 From: Justin Law Date: Wed, 25 Sep 2024 17:01:43 -0400 Subject: [PATCH 04/41] uds tasks added --- .github/workflows/uds-lint.yaml | 8 ++++++++ .pre-commit-config.yaml | 23 +++++++++++++++++++++++ tasks.yaml | 7 +++++++ 3 files changed, 38 insertions(+) create mode 100644 tasks.yaml diff --git a/.github/workflows/uds-lint.yaml b/.github/workflows/uds-lint.yaml index 8f2e6834c..a4c911b73 100644 --- a/.github/workflows/uds-lint.yaml +++ b/.github/workflows/uds-lint.yaml @@ -46,3 +46,11 @@ jobs: run: | check-jsonschema bundles/latest/gpu/uds-bundle.yaml --schemafile uds.schema.json check-jsonschema bundles/latest/cpu/uds-bundle.yaml --schemafile uds.schema.json + + - name: Download UDS Tasks Schema + run: curl -o uds.schema.json https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json + + - name: Validate tasks.yaml + if: always() + run: | + check-jsonschema tasks.yaml --schemafile tasks.schema.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6caadd6c8..c9883c5e9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -137,3 +137,26 @@ repos: files: "uds-bundle.yaml" types: [yaml] args: ["--schemafile", "uds-v0.14.0.schema.json"] + + # UDS TASKS CHECK + - repo: local + hooks: + - id: download-schema + name: "Download UDS Tasks Schema" + entry: | + bash -c 'FILE="tasks-v0.14.0.schema.json" + if [ -f "$(git rev-parse --show-toplevel)/$FILE" ]; then + echo "$FILE already exists in the root of the git project, skipping download." + else + curl -o "$FILE" https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json + fi' + language: system + + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.14.0 + hooks: + - id: check-jsonschema + name: "Validate UDS Bundles Against Schema" + files: "uds-bundle.yaml" + types: [yaml] + args: ["--schemafile", "tasks-v0.14.0.schema.json"] diff --git a/tasks.yaml b/tasks.yaml new file mode 100644 index 000000000..2f394c668 --- /dev/null +++ b/tasks.yaml @@ -0,0 +1,7 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.10.4/tasks.schema.json + +includes: + - badge: https://raw.githubusercontent.com/defenseunicorns/uds-common/refs/heads/main/tasks/badge.yaml + +tasks: + - name: default From 01f88b363a26d8656ad0bd6d00c6665e53b42d6e Mon Sep 17 00:00:00 2001 From: Justin Law Date: Wed, 25 Sep 2024 17:03:05 -0400 Subject: [PATCH 05/41] typo in uds lint --- .github/workflows/uds-lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/uds-lint.yaml b/.github/workflows/uds-lint.yaml index a4c911b73..cf7050cb8 100644 --- a/.github/workflows/uds-lint.yaml +++ b/.github/workflows/uds-lint.yaml @@ -48,7 +48,7 @@ jobs: check-jsonschema bundles/latest/cpu/uds-bundle.yaml --schemafile uds.schema.json - name: Download UDS Tasks Schema - run: curl -o uds.schema.json https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json + run: curl -o tasks.schema.json https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json - name: Validate tasks.yaml if: always() From 0f58501766752648020bc1dc569a6cd0fd43ad32 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Wed, 25 Sep 2024 17:05:52 -0400 Subject: [PATCH 06/41] unify UDS CLI version to 0.14.0 --- tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.yaml b/tasks.yaml index 2f394c668..f160d0e6b 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.10.4/tasks.schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json includes: - badge: https://raw.githubusercontent.com/defenseunicorns/uds-common/refs/heads/main/tasks/badge.yaml From 5bdc8399a1cb8d15248f082746e2bd58694e0bfd Mon Sep 17 00:00:00 2001 From: Justin Law Date: Wed, 25 Sep 2024 17:57:34 -0400 Subject: [PATCH 07/41] temp custom badge verification task --- .github/workflows/uds-verification.yaml | 92 ------ tasks.yaml | 13 +- tasks/badge.yaml | 379 ++++++++++++++++++++++++ 3 files changed, 390 insertions(+), 94 deletions(-) delete mode 100644 .github/workflows/uds-verification.yaml create mode 100644 tasks/badge.yaml diff --git a/.github/workflows/uds-verification.yaml b/.github/workflows/uds-verification.yaml deleted file mode 100644 index 09f16eddf..000000000 --- a/.github/workflows/uds-verification.yaml +++ /dev/null @@ -1,92 +0,0 @@ -name: Made for UDS Badge Verification - -on: - pull_request: - branches: - - "main" - paths: - - "packages/" - - ".github/workflows/uds-verification.yaml" - -concurrency: - group: uds-verification-${{ github.ref }} - cancel-in-progress: true - -jobs: - uds-verification: - runs-on: ubuntu-latest - name: Made for UDS Badge Verification - - permissions: - contents: read - - steps: - - name: Checkout Repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Setup UDS Environment - uses: defenseunicorns/uds-common/.github/actions/setup@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - - - name: Verify API - continue-on-error: true - uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - with: - chartPath: "packages/api/chart" - groupName: "lfai" - commonZarf: "true" - - - name: Verify UI - continue-on-error: true - uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - with: - chartPath: "packages/ui/chart" - groupName: "lfai" - commonZarf: "false" - - - name: Verify vLLM - continue-on-error: true - uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - with: - chartPath: "packages/vllm/chart" - groupName: "lfai" - commonZarf: "false" - - - name: Verify LLaMA-CPP-Python - continue-on-error: true - uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - with: - chartPath: "packages/llama-cpp-python/chart" - groupName: "lfai" - commonZarf: "false" - - - name: Verify Whisper - continue-on-error: true - uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - with: - chartPath: "packages/whisper/chart" - groupName: "lfai" - commonZarf: "false" - - - name: Verify Text-Embeddings - continue-on-error: true - uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - with: - chartPath: "packages/text-embeddings/chart" - groupName: "lfai" - commonZarf: "false" - - - name: Verify Repeater - continue-on-error: true - uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - with: - chartPath: "packages/repeater/chart" - groupName: "lfai" - commonZarf: "false" - - - name: Verify Supabase - continue-on-error: true - uses: defenseunicorns/uds-common/.github/actions/verify-badge@e3008473beab00b12a94f9fcc7340124338d5c08 # v0.13.1 - with: - chartPath: "packages/supabase/chart" - groupName: "lfai" - commonZarf: "false" diff --git a/tasks.yaml b/tasks.yaml index f160d0e6b..522754135 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -1,7 +1,16 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json includes: - - badge: https://raw.githubusercontent.com/defenseunicorns/uds-common/refs/heads/main/tasks/badge.yaml + - badge: ./tasks/badge.yaml tasks: - - name: default + - name: verify-uds-badge + actions: + - description: Verify API + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="true" \ + --set PACKAGE_DIR="packages/api/" \ + --no-progress diff --git a/tasks/badge.yaml b/tasks/badge.yaml new file mode 100644 index 000000000..dea977d43 --- /dev/null +++ b/tasks/badge.yaml @@ -0,0 +1,379 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/uds-cli/main/tasks.schema.json + +variables: + - name: CHART_PATH + description: Relative path to the directory with the uds-config chart + default: chart/ + - name: GROUP_NAME + description: The name of the niche/group that the package fits into. E.g. package,swf,lfai + default: package + - name: COMMON_ZARF + description: Whether or not there is a common zarf.yaml file + default: "true" + - name: PACKAGE_DIR + description: Base directory where packages are located, relative to the root of the repository + default: ./ + +tasks: + - name: verify-badge + actions: + - description: Verify that the package meets uds badging standards + shell: + linux: bash + darwin: bash + cmd: | + # Functions + + WARNINGS=0 + ERRORS=0 + + gh_notice() { + if [[ "${CI}" == "true" ]]; then + printf "::notice::" + fi + echo "$1" + } + + gh_warning() { + if [[ "${CI}" == "true" ]]; then + printf "::warning::" + fi + WARNINGS=$((WARNINGS + 1)) + echo "$1" + } + + gh_error() { + if [[ "${CI}" == "true" ]]; then + printf "::error::" + fi + ERRORS=$((ERRORS + 1)) + echo "$1" + } + + gh_group() { + if [[ "${CI}" == "true" ]]; then + echo "::group::$1" + else + echo "$1" + fi + } + + gh_endgroup() { + if [[ "${CI}" == "true" ]]; then + echo "::endgroup::" + else + echo + fi + } + + gh_group "šŸ“ƒ Setup" + + # Default values for vars, maru-runner issue workaround + CHART_PATH=${CHART_PATH:-chart/} + gh_notice " ā„¹ļø Chart Path: $CHART_PATH" + GROUP_NAME=${GROUP_NAME:-package} + gh_notice " ā„¹ļø Group Name: $GROUP_NAME" + COMMON_ZARF=${COMMON_ZARF:-true} + gh_notice " ā„¹ļø Common Zarf: $COMMON_ZARF" + PACKAGE_DIR=${PACKAGE_DIR:-"./"} + gh_notice " ā„¹ļø Package Directory: $PACKAGE_DIR" + + # Cover when GOBIN isn't set + export GOBIN=/tmp/gobin + + gh_notice " ā„¹ļø Installing kubectl-validate..." + if go install sigs.k8s.io/kubectl-validate@latest; then + gh_notice " āœ… kubectl-validate installed" + else + gh_error " āŒ kubectl-validate failed to install" + fi + + if [[ "${COMMON_ZARF}" == "true" ]]; then + NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}common/zarf.yaml" | uniq) + else + gh_warning " āš ļø There is no common zarf.yaml file" + NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}zarf.yaml" | uniq) + fi + gh_notice " ā„¹ļø Namespace: $NAMESPACE" + + UDS_PACKAGE_JSON=$(kubectl get Packages -n "${NAMESPACE}" -o jsonpath='{.items[]}') + gh_notice " ā„¹ļø Retrieved UDS Package JSON" + + UDS_PACKAGE_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.metadata.name') + gh_notice " ā„¹ļø Package Name: $UDS_PACKAGE_NAME" + + gh_endgroup + + gh_group "šŸ” Config Chart Validation" + + # Validate UDS Package/Config chart is valid + + gh_notice " ā„¹ļø Validating uds-config chart..." + printf "\033[A" + if ${GOBIN}/kubectl-validate <(helm template "${PACKAGE_DIR}${CHART_PATH}" | grep -v '^ *#' | perl -p0e 's/---[\s---]*---/---/sg') &> /tmp/validate-output; then + gh_notice " āœ… uds-config chart is valid" + else + gh_error " āŒ uds-config chart is invalid" + cat /tmp/validate-output | sed 's/^/ /' + fi + + gh_endgroup + + gh_group "šŸ” Istio Validation" + + ## Must define any external interfaces under the expose key. + + ENDPOINTS="$(echo "$UDS_PACKAGE_JSON" | jq -r '.status.endpoints[]')" + gh_notice " ā„¹ļø Endpoints: $ENDPOINTS" + + for ENDPOINT in $ENDPOINTS; do + # Curl endpoint and check that status isn't 404 or 5XX + STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://$ENDPOINT") + if [ "$STATUS" -eq 404 ] || [[ $STATUS == 5* ]]; then + gh_error " āŒ Endpoint $ENDPOINT is returning $STATUS" + else + gh_notice " āœ… Endpoint $ENDPOINT was successfully curl'd" + fi + done + + echo + + ## Must deploy and operate successfully with Istio injection enabled in the namespace. + + POD_COUNT=$(kubectl get pods -n "${NAMESPACE}" --no-headers | wc -l) + POD_SIDECAR_COUNT=$(kubectl get pods -n "${NAMESPACE}" -o json | jq '.items[].spec.containers[] | select(.name=="istio-proxy") | length' | wc -l) + + if [ "$POD_COUNT" -ne "$POD_SIDECAR_COUNT" ]; then + gh_error " āŒ Not all pods have the istio sidecar" + else + gh_notice " āœ… All pods have the istio sidecar" + fi + + echo + + ## Should avoid workarounds such as disabling strict mTLS peer authentication. + + PEERAUTH=$(kubectl get Peerauthentication -n "${NAMESPACE}" -o=json | jq -r '.items[].spec.mtls.mode') + + if [ "$PEERAUTH" != "STRICT" ] && [ "$PEERAUTH" != "" ]; then + gh_warning " āš ļø Peerauth is not strict or inherited, review needed" + else + gh_notice " āœ… Peerauthentication is set to strict" + fi + + echo + + # Network Policies + + ## Must define network policies under the allow key as required. + + NETPOL_AMOUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow | length') + + if [ "$NETPOL_AMOUNT" -eq 0 ]; then + gh_warning " āš ļø No network policies defined, review needed" + else + gh_notice " āœ… Network policies are defined" + fi + + ## Should minimize network policies to specific selectors needed for Ingress/Egress traffic. + + NETPOL_EXT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | length' | wc -l) + gh_notice " ā„¹ļø Non-IntraNamespace network policies: $NETPOL_EXT_COUNT" + + NETPOL_EXT_SELECTOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | select(.selector != null) | length' | wc -l) + gh_notice " ā„¹ļø Non-IntraNamespace network policies with selectors: $NETPOL_EXT_SELECTOR_COUNT" + + if [ "$NETPOL_EXT_COUNT" -ne "$NETPOL_EXT_SELECTOR_COUNT" ]; then + gh_error " āŒ Not all applicable network policies are using selectors" + else + gh_notice " āœ… All applicable network policies are using selectors" + fi + + NETPOL_EXT_NOTKUBE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | select(.remoteGenerated != "KubeAPI") | length' | wc -l) + gh_notice " ā„¹ļø Non-IntraNamespace, non-KubeAPI network policies: $NETPOL_EXT_NOTKUBE_COUNT" + + NETPOL_EXT_NOTKUBE_PORT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | select(.remoteGenerated != "KubeAPI") | select(.ports != null or .port != null) | length' | wc -l) + gh_notice " ā„¹ļø Non-IntraNamespace, non-KubeAPI network policies with ports: $NETPOL_EXT_NOTKUBE_PORT_COUNT" + + if [ "$NETPOL_EXT_NOTKUBE_COUNT" -ne "$NETPOL_EXT_NOTKUBE_PORT_COUNT" ]; then + gh_error " āŒ Not all applicable network policies are using ports" + else + gh_notice " āœ… All applicable network policies are using ports" + fi + + NETPOL_ANYWHERE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated == "Anywhere") | length' | wc -l) + if [ "$NETPOL_ANYWHERE_COUNT" -gt 0 ]; then + gh_warning " āš ļø Network policies with 'remoteGenerated: Anywhere' are present, review needed" + else + gh_notice " āœ… No network policies with 'remoteGenerated: Anywhere' are present" + fi + + gh_endgroup + + gh_group "šŸ” Keycloak/SSO Validation" + + SSO_CLIENT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso | length') + + ## Must use and create a Keycloak client through the sso key if the application provides a user login. + if [ "$SSO_CLIENT_COUNT" -gt 0 ]; then + gh_notice " ā„¹ļø There are $SSO_CLIENT_COUNT SSO clients defined" + + # Should consider security options during implementation to provide the most secure default possible (i.e. SAML w/SCIM vs OIDC). + SSO_CLIENT_PROTOCOL=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].protocol') + if [ "$SSO_CLIENT_PROTOCOL" != "saml" ]; then + gh_warning " āš ļø SAML is not the default protocol, review needed" + else + gh_notice " āœ… Default protocol is SAML" + fi + + # Should name the client Login (i.e. Mattermost Login) to provide login UX consistency. + SSO_CLIENT_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].name') + if [ "$SSO_CLIENT_NAME" != "${UDS_PACKAGE_NAME^} Login" ]; then + gh_warning " āš ļø SSO client name not in preferred format, review needed" + else + gh_notice " āœ… SSO client name is correct" + fi + + # Should clearly mark the client id with the group and app name uds-- (i.e. uds-swf-mattermost) to provide consistency in the Keycloak UI. + SSO_CLIENT_ID=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].clientId') + if [ "$SSO_CLIENT_ID" != "uds-${GROUP_NAME})-${UDS_PACKAGE_NAME}-${SSO_CLIENT_PROTOCOL}" ]; then + gh_warning " āš ļø SSO client id not in the format uds---protocol, review needed" + else + gh_notice " āœ… SSO client id is correct" + fi + + ## May end any generated secrets with -sso to easily locate them when querying the cluster. + SSO_SECRET_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].secretName') + if [ "$SSO_SECRET_NAME" != "" ]; then + gh_notice " ā„¹ļø SSO secret name is defined" + if [[ "$SSO_SECRET_NAME" != *-sso ]]; then + gh_warning " āš ļø SSO secret name not in the format -sso, review needed" + else + gh_notice " āœ… SSO secret name is in the correct format" + fi + fi + else + gh_warning " āš ļø No SSO configuration found, review needed" + fi + + gh_endgroup + + gh_group "šŸ” Monitoring Validation" + + ## Must implement monitors for each application metrics endpoint using it's built-in chart monitors, the Package CR monitor key, or manual monitors in the config chart. + + PKG_MONITOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.monitor | length') + + if [ "$PKG_MONITOR_COUNT" -eq 0 ]; then + gh_notice " ā„¹ļø No monitors defined in the package, checking for ServiceMonitors" + + # Check for built-in serviceMonitors + SERVICE_MONITOR_COUNT=$(kubectl get ServiceMonitor -n "${NAMESPACE}" -o json | jq -r '.items | length') + if [ "$SERVICE_MONITOR_COUNT" -eq 0 ]; then + gh_notice " ā„¹ļø No ServiceMonitors defined, checking for PodMonitors" + + # Check for built-in podMonitors + POD_MONITOR_COUNT=$(kubectl get PodMonitor -n "${NAMESPACE}" -o json | jq -r '.items | length') + if [ "$POD_MONITOR_COUNT" -eq 0 ]; then + gh_error " āŒ No monitors defined" + else + gh_notice " āœ… PodMonitor found" + gh_warning " āš ļø There may be more monitors to implement, review needed" + fi + else + gh_notice " āœ… ServiceMonitor found" + gh_warning " āš ļø There may be more monitors to implement, review needed" + fi + else + gh_notice " āœ… Monitor defined in UDS Package" + gh_warning " āš ļø There may be more monitors to implement, review needed" + fi + + gh_endgroup + + gh_group "šŸ” Exemptions Validation" + + ## Must minimize the scope and number of the exemptions to only what is absolutely required by the application + + EXEMPTION_COUNT=$(kubectl get Exemptions -n "${NAMESPACE}" -o json | jq -r '.items | length') + if [ "$EXEMPTION_COUNT" -gt 0 ]; then + gh_warning " āš ļø Exemptions are present, review needed" + else + gh_notice " āœ… No exemptions present" + fi + + gh_endgroup + + gh_group "šŸ” Package Structure Validation" + + ## Should expose all configuration (uds.dev CRs, additional Secrets/ConfigMaps, etc) through a Helm chart (ideally in a chart or charts directory). + + if [ "${COMMON_ZARF}" == "true" ]; then + COMMON_ZARF_MANIFEST_COUNT=$(cat "${PACKAGE_DIR}common/zarf.yaml" | uds zarf tools yq '.components[] | select(.manifests != null) | length' | wc -l) + if [ "$COMMON_ZARF_MANIFEST_COUNT" -gt 0 ]; then + gh_error " āŒ Manifests present in common/zarf.yaml" + else + gh_notice " āœ… No manifests present in common/zarf.yaml" + fi + fi + + MAIN_ZARF_MANIFEST_COUNT=$(cat "${PACKAGE_DIR}zarf.yaml" | uds zarf tools yq '.components[] | select(.manifests != null) | length' | wc -l) + if [ "$MAIN_ZARF_MANIFEST_COUNT" -gt 0 ]; then + gh_error " āŒ Manifests present in zarf.yaml" + else + gh_notice " āœ… No manifests present in zarf.yaml" + fi + + ## Should implement or allow for multiple flavors (ideally with common definitions in a common directory) + + ZARF_COMPONENTS_FLAVOR_COUNT=$(cat "${PACKAGE_DIR}zarf.yaml" | uds zarf tools yq '.components[] | select(.only.flavor != null) | length' | wc -l) + if [ "$ZARF_COMPONENTS_FLAVOR_COUNT" -eq 0 ]; then + gh_error " āŒ No flavors defined in zarf.yaml" + else + gh_notice " āœ… At least one flavor defined in zarf.yaml" + fi + + gh_endgroup + + gh_group "šŸ” Package Testing Validation" + + ## Must implement Journey Testing to cover the basic user flows and features of the application, especially where an application interacts with an external service / interface. + + if [ -d "${PACKAGE_DIR}tests" ]; then + if [ "$(ls -A ${PACKAGE_DIR}tests)" ]; then + gh_notice " āœ… Tests folder exists and has files" + else + gh_error " āŒ Tests folder exists but is empty" + fi + else + gh_error " āŒ Tests folder does not exist" + fi + + gh_endgroup + + gh_group "šŸ” Versioning Validation" + + # Must be consistently versioned across flavors - this can take many forms but flavors should differ in image bases/builds not application versions. + + ZARF_PKG_VERSION=$(uds zarf tools yq '.metadata.version' "${PACKAGE_DIR}zarf.yaml" | cut -d '-' -f1) + IMAGE_WITH_ZARF_VERSION_COUNT=$(cat "${PACKAGE_DIR}zarf.yaml" | uds zarf tools yq '.components[].images' | grep -o "${ZARF_PKG_VERSION}" | wc -l) + + if [ "$IMAGE_WITH_ZARF_VERSION_COUNT" -ge "$ZARF_COMPONENTS_FLAVOR_COUNT" ]; then + gh_notice " āœ… Version is consistent across flavors and package" + else + gh_warning " āš ļø Version is not consistent across flavors and package" + fi + + gh_endgroup + + # Summary + + if [ "$ERRORS" -gt 0 ]; then + gh_error "āŒ $ERRORS errors found" + fi + if [ "$WARNINGS" -gt 0 ]; then + gh_warning "āš ļø $WARNINGS warnings found" + fi + if [ "$ERRORS" -eq 0 ] && [ "$WARNINGS" -eq 0 ]; then + gh_notice "āœ… No errors or warnings found" + fi From 9d705604997b55d5bdd266c0888734e0795daf93 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Wed, 25 Sep 2024 18:07:55 -0400 Subject: [PATCH 08/41] individual package verify task --- tasks.yaml | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/tasks.yaml b/tasks.yaml index 522754135..12d370565 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -4,7 +4,7 @@ includes: - badge: ./tasks/badge.yaml tasks: - - name: verify-uds-badge + - name: verify-uds-badge-api actions: - description: Verify API cmd: | @@ -14,3 +14,80 @@ tasks: --set COMMON_ZARF="true" \ --set PACKAGE_DIR="packages/api/" \ --no-progress + + - name: verify-uds-badge-ui + actions: + - description: Verify UI + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/ui/" \ + --no-progress + + - name: verify-uds-badge-llama-cpp-python + actions: + - description: Verify LLaMA-CPP-Python + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/llama-cpp-python/" \ + --no-progress + + - name: verify-uds-badge-vllm + actions: + - description: Verify vLLM + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/vllm/" \ + --no-progress + + - name: verify-uds-badge-text-embeddings + actions: + - description: Verify Text-Embeddings + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/text-embeddings/" \ + --no-progress + + - name: verify-uds-badge-whisper + actions: + - description: Verify Whisper + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/whisper/" \ + --no-progress + + - name: verify-uds-badge-repeater + actions: + - description: Verify Repeater + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/repeater/" \ + --no-progress + + - name: verify-uds-badge-supabase + actions: + - description: Verify Supabase + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/supabase/" \ + --no-progress From dec511916903499e89c9a1e96e7f1b3a65fc0d89 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Wed, 25 Sep 2024 18:11:23 -0400 Subject: [PATCH 09/41] relative directory slash moved downstream --- tasks.yaml | 16 ++++++++-------- tasks/badge.yaml | 24 ++++++++++++------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tasks.yaml b/tasks.yaml index 12d370565..c349b6015 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -12,7 +12,7 @@ tasks: --set CHART_PATH="chart" \ --set GROUP_NAME="package" \ --set COMMON_ZARF="true" \ - --set PACKAGE_DIR="packages/api/" \ + --set PACKAGE_DIR="packages/api" \ --no-progress - name: verify-uds-badge-ui @@ -23,7 +23,7 @@ tasks: --set CHART_PATH="chart" \ --set GROUP_NAME="package" \ --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/ui/" \ + --set PACKAGE_DIR="packages/ui" \ --no-progress - name: verify-uds-badge-llama-cpp-python @@ -34,7 +34,7 @@ tasks: --set CHART_PATH="chart" \ --set GROUP_NAME="package" \ --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/llama-cpp-python/" \ + --set PACKAGE_DIR="packages/llama-cpp-python" \ --no-progress - name: verify-uds-badge-vllm @@ -45,7 +45,7 @@ tasks: --set CHART_PATH="chart" \ --set GROUP_NAME="package" \ --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/vllm/" \ + --set PACKAGE_DIR="packages/vllm" \ --no-progress - name: verify-uds-badge-text-embeddings @@ -56,7 +56,7 @@ tasks: --set CHART_PATH="chart" \ --set GROUP_NAME="package" \ --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/text-embeddings/" \ + --set PACKAGE_DIR="packages/text-embeddings" \ --no-progress - name: verify-uds-badge-whisper @@ -67,7 +67,7 @@ tasks: --set CHART_PATH="chart" \ --set GROUP_NAME="package" \ --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/whisper/" \ + --set PACKAGE_DIR="packages/whisper" \ --no-progress - name: verify-uds-badge-repeater @@ -78,7 +78,7 @@ tasks: --set CHART_PATH="chart" \ --set GROUP_NAME="package" \ --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/repeater/" \ + --set PACKAGE_DIR="packages/repeater" \ --no-progress - name: verify-uds-badge-supabase @@ -89,5 +89,5 @@ tasks: --set CHART_PATH="chart" \ --set GROUP_NAME="package" \ --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/supabase/" \ + --set PACKAGE_DIR="packages/supabase" \ --no-progress diff --git a/tasks/badge.yaml b/tasks/badge.yaml index dea977d43..634230c59 100644 --- a/tasks/badge.yaml +++ b/tasks/badge.yaml @@ -12,7 +12,7 @@ variables: default: "true" - name: PACKAGE_DIR description: Base directory where packages are located, relative to the root of the repository - default: ./ + default: . tasks: - name: verify-badge @@ -75,7 +75,7 @@ tasks: gh_notice " ā„¹ļø Group Name: $GROUP_NAME" COMMON_ZARF=${COMMON_ZARF:-true} gh_notice " ā„¹ļø Common Zarf: $COMMON_ZARF" - PACKAGE_DIR=${PACKAGE_DIR:-"./"} + PACKAGE_DIR=${PACKAGE_DIR:-"."} gh_notice " ā„¹ļø Package Directory: $PACKAGE_DIR" # Cover when GOBIN isn't set @@ -89,10 +89,10 @@ tasks: fi if [[ "${COMMON_ZARF}" == "true" ]]; then - NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}common/zarf.yaml" | uniq) + NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}/common/zarf.yaml" | uniq) else gh_warning " āš ļø There is no common zarf.yaml file" - NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}zarf.yaml" | uniq) + NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}/zarf.yaml" | uniq) fi gh_notice " ā„¹ļø Namespace: $NAMESPACE" @@ -110,7 +110,7 @@ tasks: gh_notice " ā„¹ļø Validating uds-config chart..." printf "\033[A" - if ${GOBIN}/kubectl-validate <(helm template "${PACKAGE_DIR}${CHART_PATH}" | grep -v '^ *#' | perl -p0e 's/---[\s---]*---/---/sg') &> /tmp/validate-output; then + if ${GOBIN}/kubectl-validate <(helm template "${PACKAGE_DIR}/${CHART_PATH}" | grep -v '^ *#' | perl -p0e 's/---[\s---]*---/---/sg') &> /tmp/validate-output; then gh_notice " āœ… uds-config chart is valid" else gh_error " āŒ uds-config chart is invalid" @@ -309,7 +309,7 @@ tasks: ## Should expose all configuration (uds.dev CRs, additional Secrets/ConfigMaps, etc) through a Helm chart (ideally in a chart or charts directory). if [ "${COMMON_ZARF}" == "true" ]; then - COMMON_ZARF_MANIFEST_COUNT=$(cat "${PACKAGE_DIR}common/zarf.yaml" | uds zarf tools yq '.components[] | select(.manifests != null) | length' | wc -l) + COMMON_ZARF_MANIFEST_COUNT=$(cat "${PACKAGE_DIR}/common/zarf.yaml" | uds zarf tools yq '.components[] | select(.manifests != null) | length' | wc -l) if [ "$COMMON_ZARF_MANIFEST_COUNT" -gt 0 ]; then gh_error " āŒ Manifests present in common/zarf.yaml" else @@ -317,7 +317,7 @@ tasks: fi fi - MAIN_ZARF_MANIFEST_COUNT=$(cat "${PACKAGE_DIR}zarf.yaml" | uds zarf tools yq '.components[] | select(.manifests != null) | length' | wc -l) + MAIN_ZARF_MANIFEST_COUNT=$(cat "${PACKAGE_DIR}/zarf.yaml" | uds zarf tools yq '.components[] | select(.manifests != null) | length' | wc -l) if [ "$MAIN_ZARF_MANIFEST_COUNT" -gt 0 ]; then gh_error " āŒ Manifests present in zarf.yaml" else @@ -326,7 +326,7 @@ tasks: ## Should implement or allow for multiple flavors (ideally with common definitions in a common directory) - ZARF_COMPONENTS_FLAVOR_COUNT=$(cat "${PACKAGE_DIR}zarf.yaml" | uds zarf tools yq '.components[] | select(.only.flavor != null) | length' | wc -l) + ZARF_COMPONENTS_FLAVOR_COUNT=$(cat "${PACKAGE_DIR}/zarf.yaml" | uds zarf tools yq '.components[] | select(.only.flavor != null) | length' | wc -l) if [ "$ZARF_COMPONENTS_FLAVOR_COUNT" -eq 0 ]; then gh_error " āŒ No flavors defined in zarf.yaml" else @@ -339,8 +339,8 @@ tasks: ## Must implement Journey Testing to cover the basic user flows and features of the application, especially where an application interacts with an external service / interface. - if [ -d "${PACKAGE_DIR}tests" ]; then - if [ "$(ls -A ${PACKAGE_DIR}tests)" ]; then + if [ -d "${PACKAGE_DIR}/tests" ]; then + if [ "$(ls -A ${PACKAGE_DIR}/tests)" ]; then gh_notice " āœ… Tests folder exists and has files" else gh_error " āŒ Tests folder exists but is empty" @@ -355,8 +355,8 @@ tasks: # Must be consistently versioned across flavors - this can take many forms but flavors should differ in image bases/builds not application versions. - ZARF_PKG_VERSION=$(uds zarf tools yq '.metadata.version' "${PACKAGE_DIR}zarf.yaml" | cut -d '-' -f1) - IMAGE_WITH_ZARF_VERSION_COUNT=$(cat "${PACKAGE_DIR}zarf.yaml" | uds zarf tools yq '.components[].images' | grep -o "${ZARF_PKG_VERSION}" | wc -l) + ZARF_PKG_VERSION=$(uds zarf tools yq '.metadata.version' "${PACKAGE_DIR}/zarf.yaml" | cut -d '-' -f1) + IMAGE_WITH_ZARF_VERSION_COUNT=$(cat "${PACKAGE_DIR}/zarf.yaml" | uds zarf tools yq '.components[].images' | grep -o "${ZARF_PKG_VERSION}" | wc -l) if [ "$IMAGE_WITH_ZARF_VERSION_COUNT" -ge "$ZARF_COMPONENTS_FLAVOR_COUNT" ]; then gh_notice " āœ… Version is consistent across flavors and package" From 7fb7d2123269dec054ee1f3729868e5750ef65a5 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Thu, 26 Sep 2024 11:05:15 -0400 Subject: [PATCH 10/41] workflow version of badge tasks --- .../nightly-uds-badge-verification.yaml | 77 ++++++++++++ tasks.yaml | 111 ++++-------------- tasks/badge.yaml | 105 ++++++++++++++++- 3 files changed, 201 insertions(+), 92 deletions(-) create mode 100644 .github/workflows/nightly-uds-badge-verification.yaml diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml new file mode 100644 index 000000000..18c022e31 --- /dev/null +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -0,0 +1,77 @@ +name: nightly-uds-badge-verification + +on: + schedule: + - cron: "0 8 * * *" # Runs daily at 12 AM PST + workflow_dispatch: # trigger manually as needed + pull_request: + types: + - opened # default trigger + - reopened # default trigger + - synchronize # default trigger + - ready_for_review # don't run on draft PRs + - milestoned # allows us to trigger on bot PRs + paths: + - .github/workflows/nightly-uds-badge-verification.yaml + +concurrency: + group: nightly-uds-badge-verification-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + SNAPSHOT_VERSION: snapshot-latest + +permissions: + contents: read + packages: read + id-token: write # This is needed for OIDC federation. + +jobs: + uds-badge-verification: + runs-on: ai-ubuntu-big-boy-8-core + name: nightly_uds_badge_verification + if: ${{ !github.event.pull_request.draft }} + + steps: + - name: Checkout Repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: main + + - name: Setup UDS Cluster + uses: ./.github/actions/uds-cluster + with: + registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} + registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} + ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} + + - name: Print the Commit SHA + run: | + COMMIT_SHA=$(git rev-parse HEAD) + echo "The latest commit on the main branch is: $COMMIT_SHA" + + # Set UDS CPU bundle refs and repositories to snapshot-latest + - name: Mutation of the UDS Bundle + run: | + uds zarf tools yq -i '.metadata.version = "${{ env.SNAPSHOT_VERSION }}"' bundles/latest/cpu/uds-bundle.yaml + + uds zarf tools yq -i '.packages[].ref |= sub("^[^ ]+-upstream$", "${{ env.SNAPSHOT_VERSION }}-upstream")' bundles/latest/cpu/uds-bundle.yaml + + uds zarf tools yq -i '.packages[].repository |= sub("/uds/", "/uds/snapshots/")' bundles/latest/cpu/uds-bundle.yaml + + - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }}) + run: | + cd bundles/latest/cpu + uds create . --confirm && \ + uds deploy uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst --confirm --no-progress && \ + rm -rf uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst && \ + docker system prune -af + + - name: Run UDS Badge Verification Task + run: | + uds run verify-uds-badge-cpu diff --git a/tasks.yaml b/tasks.yaml index c349b6015..756d09aab 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -4,90 +4,27 @@ includes: - badge: ./tasks/badge.yaml tasks: - - name: verify-uds-badge-api - actions: - - description: Verify API - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="true" \ - --set PACKAGE_DIR="packages/api" \ - --no-progress - - - name: verify-uds-badge-ui - actions: - - description: Verify UI - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/ui" \ - --no-progress - - - name: verify-uds-badge-llama-cpp-python - actions: - - description: Verify LLaMA-CPP-Python - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/llama-cpp-python" \ - --no-progress - - - name: verify-uds-badge-vllm - actions: - - description: Verify vLLM - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/vllm" \ - --no-progress - - - name: verify-uds-badge-text-embeddings - actions: - - description: Verify Text-Embeddings - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/text-embeddings" \ - --no-progress - - - name: verify-uds-badge-whisper - actions: - - description: Verify Whisper - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/whisper" \ - --no-progress - - - name: verify-uds-badge-repeater - actions: - - description: Verify Repeater - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/repeater" \ - --no-progress - - - name: verify-uds-badge-supabase - actions: - - description: Verify Supabase - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/supabase" \ - --no-progress + ############# + # Badge Tasks + ############# + - name: verify-uds-badge-cpu + description: "Runs through all CPU UDS bundle packages with the UDS badge verification test" + actions: + - task: badge:verify-uds-badge-api + - task: badge:verify-uds-badge-ui + - task: badge:verify-uds-badge-llama-cpp-python + - task: badge:verify-uds-badge-vllm + - task: badge:verify-uds-badge-text-embeddings + - task: badge:verify-uds-badge-whisper + - task: badge:verify-uds-badge-repeater + - task: badge:verify-uds-badge-supabase + + - name: verify-uds-badge-gpu + description: "Runs through all GPU UDS bundle packages with the UDS badge verification test" + actions: + - task: badge:verify-uds-badge-api + - task: badge:verify-uds-badge-ui + - task: badge:verify-uds-badge-vllm + - task: badge:verify-uds-badge-text-embeddings + - task: badge:verify-uds-badge-whisper + - task: badge:verify-uds-badge-supabase diff --git a/tasks/badge.yaml b/tasks/badge.yaml index 634230c59..7770f948d 100644 --- a/tasks/badge.yaml +++ b/tasks/badge.yaml @@ -2,22 +2,117 @@ variables: - name: CHART_PATH - description: Relative path to the directory with the uds-config chart + description: "Relative path to the directory with the uds-config chart" default: chart/ - name: GROUP_NAME - description: The name of the niche/group that the package fits into. E.g. package,swf,lfai + description: "The name of the niche/group that the package fits into. E.g. package,swf,lfai" default: package - name: COMMON_ZARF - description: Whether or not there is a common zarf.yaml file + description: "Whether or not there is a common zarf.yaml file" default: "true" - name: PACKAGE_DIR - description: Base directory where packages are located, relative to the root of the repository + description: "Base directory where packages are located, relative to the root of the repository" default: . tasks: + ############## + # PACKAGE-LEVEL + ############## + + - name: verify-uds-badge-api + actions: + - description: "Verify API" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="true" \ + --set PACKAGE_DIR="packages/api" \ + --no-progress + + - name: verify-uds-badge-ui + actions: + - description: "Verify UI" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/ui" \ + --no-progress + + - name: verify-uds-badge-llama-cpp-python + actions: + - description: "Verify LLaMA-CPP-Python" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/llama-cpp-python" \ + --no-progress + + - name: verify-uds-badge-vllm + actions: + - description: "Verify vLLM" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/vllm" \ + --no-progress + + - name: verify-uds-badge-text-embeddings + actions: + - description: "Verify Text-Embeddings" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/text-embeddings" \ + --no-progress + + - name: verify-uds-badge-whisper + actions: + - description: "Verify Whisper" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/whisper" \ + --no-progress + + - name: verify-uds-badge-repeater + actions: + - description: "Verify Repeater" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/repeater" \ + --no-progress + + - name: verify-uds-badge-supabase + actions: + - description: "Verify Supabase" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/supabase" \ + --no-progress + + ########### + # RE-USABLE + ########### - name: verify-badge actions: - - description: Verify that the package meets uds badging standards + - description: "Verify that the package meets UDS badging standards" shell: linux: bash darwin: bash From d5a3bcc13ad0322eca61b0e3017d328aee6e77d0 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Thu, 26 Sep 2024 11:25:27 -0400 Subject: [PATCH 11/41] add pretty summary --- .github/scripts/summarizeVerification.sh | 130 ++++++++++++++++++ .../nightly-uds-badge-verification.yaml | 7 +- .gitignore | 1 + 3 files changed, 137 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/summarizeVerification.sh diff --git a/.github/scripts/summarizeVerification.sh b/.github/scripts/summarizeVerification.sh new file mode 100755 index 000000000..658e5a142 --- /dev/null +++ b/.github/scripts/summarizeVerification.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +# Initialize variables +in_package=false +package_name="" +error_count=0 +warning_count=0 +error_descriptions=() +warning_descriptions=() + +# Function to strip ANSI escape sequences +strip_ansi() { + echo -e "$1" | sed -r 's/\x1B\[[0-9;]*[a-zA-Z]//g' +} + +while IFS= read -r line; do + # Strip ANSI escape sequences + clean_line=$(strip_ansi "$line") + # Remove leading/trailing whitespace + clean_line=$(echo "$clean_line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + # Check for the start of a new package + if [[ $clean_line =~ ^ā€¢[[:space:]]+Running\ \"Verify\ ([^\"]+)\"$ ]]; then + new_package_name="${BASH_REMATCH[1]}" + if [[ $new_package_name != "that the package meets UDS badging standards" ]]; then + # If we're already in a package, print its summary + if [ "$in_package" = true ]; then + echo -e "Package: $package_name" + echo -e "āŒ Errors: $error_count" + echo -e "āš ļø Warnings: $warning_count" + if [ $error_count -gt 0 ]; then + echo "āŒ Error Descriptions:" + for err in "${error_descriptions[@]}"; do + echo " - $err" + done + fi + if [ $warning_count -gt 0 ]; then + echo "āš ļø Warning Descriptions:" + for warn in "${warning_descriptions[@]}"; do + echo " - $warn" + done + fi + echo "-----------------------------" + fi + # Start new package + in_package=true + package_name="$new_package_name" + error_count=0 + warning_count=0 + error_descriptions=() + warning_descriptions=() + fi + fi + + # Capture error descriptions + if [[ $in_package == true && $clean_line =~ ^āŒ ]]; then + # Ignore summary lines + if [[ $clean_line =~ errors\ found$ ]]; then + # Extract the error count from the line + error_count_line=$(echo "$clean_line" | grep -oP '(?<=āŒ )[0-9]+(?= errors found)') + if [[ -n $error_count_line ]]; then + error_count="$error_count_line" + fi + elif [[ $clean_line =~ ^āŒ[[:space:]][0-9]+ ]]; then + continue + else + error_descriptions+=("$(echo "$clean_line" | sed 's/^āŒ[[:space:]]*//')") + ((error_count++)) + fi + fi + + # Capture warning descriptions + if [[ $in_package == true && $clean_line =~ ^āš ļø ]]; then + # Ignore summary lines + if [[ $clean_line =~ warnings\ found$ ]]; then + # Extract the warning count from the line + warning_count_line=$(echo "$clean_line" | grep -oP '(?<=āš ļø )[0-9]+(?= warnings found)') + if [[ -n $warning_count_line ]]; then + warning_count="$warning_count_line" + fi + elif [[ $clean_line =~ ^āš ļø[[:space:]][0-9]+ ]]; then + continue + else + warning_descriptions+=("$(echo "$clean_line" | sed 's/^āš ļø[[:space:]]*//')") + ((warning_count++)) + fi + fi + + # Check for the end of a package + if [[ $in_package == true && $clean_line =~ ^āœ”[[:space:]]+Completed\ \"Verify\ $package_name\"$ ]]; then + # Print summary + echo -e "Package: $package_name" + echo -e "āŒ Errors: $error_count" + echo -e "āš ļø Warnings: $warning_count" + if [ $error_count -gt 0 ]; then + echo "āŒ Error Descriptions:" + for err in "${error_descriptions[@]}"; do + echo " - $err" + done + fi + if [ $warning_count -gt 0 ]; then + echo "āš ļø Warning Descriptions:" + for warn in "${warning_descriptions[@]}"; do + echo " - $warn" + done + fi + echo "-----------------------------" + in_package=false + fi +done > output.txt + + - name: Pretty Summary of UDS Badge Verification + run: | + chmod +x .github/scripts/summarizeVerification.sh + .github/scripts/summarizeVerification.sh diff --git a/.gitignore b/.gitignore index 645bd6ff5..dbc0825ce 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ node_modules package.json package-lock.json **/*.schema.json +output.txt # local model and tokenizer files *.bin From 6a1a6d2aa7e95dc2e712f48d2f164b5ef2063c7a Mon Sep 17 00:00:00 2001 From: Justin Law Date: Thu, 26 Sep 2024 11:47:57 -0400 Subject: [PATCH 12/41] graceful multi-Package CR handling --- tasks/badge.yaml | 62 ++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/tasks/badge.yaml b/tasks/badge.yaml index 7770f948d..1313d89b2 100644 --- a/tasks/badge.yaml +++ b/tasks/badge.yaml @@ -2,16 +2,16 @@ variables: - name: CHART_PATH - description: "Relative path to the directory with the uds-config chart" + description: Relative path to the directory with the uds-config chart default: chart/ - name: GROUP_NAME - description: "The name of the niche/group that the package fits into. E.g. package,swf,lfai" + description: The name of the niche/group that the package fits into. E.g. package,swf,lfai default: package - name: COMMON_ZARF - description: "Whether or not there is a common zarf.yaml file" + description: Whether or not there is a common zarf.yaml file default: "true" - name: PACKAGE_DIR - description: "Base directory where packages are located, relative to the root of the repository" + description: Base directory where packages are located, relative to the root of the repository default: . tasks: @@ -112,7 +112,7 @@ tasks: ########### - name: verify-badge actions: - - description: "Verify that the package meets UDS badging standards" + - description: Verify that the package meets UDS badging standards shell: linux: bash darwin: bash @@ -191,12 +191,14 @@ tasks: fi gh_notice " ā„¹ļø Namespace: $NAMESPACE" - UDS_PACKAGE_JSON=$(kubectl get Packages -n "${NAMESPACE}" -o jsonpath='{.items[]}') - gh_notice " ā„¹ļø Retrieved UDS Package JSON" - - UDS_PACKAGE_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.metadata.name') + # Extract UDS_PACKAGE_NAME from zarf.yaml + UDS_PACKAGE_NAME=$(uds zarf tools yq '.metadata.name' "${PACKAGE_DIR}/zarf.yaml") gh_notice " ā„¹ļø Package Name: $UDS_PACKAGE_NAME" + # Retrieve the specific Package resource + UDS_PACKAGE_JSON=$(kubectl get Package "$UDS_PACKAGE_NAME" -n "${NAMESPACE}" -o json) + gh_notice " ā„¹ļø Retrieved UDS Package JSON for $UDS_PACKAGE_NAME" + gh_endgroup gh_group "šŸ” Config Chart Validation" @@ -236,7 +238,7 @@ tasks: ## Must deploy and operate successfully with Istio injection enabled in the namespace. POD_COUNT=$(kubectl get pods -n "${NAMESPACE}" --no-headers | wc -l) - POD_SIDECAR_COUNT=$(kubectl get pods -n "${NAMESPACE}" -o json | jq '.items[].spec.containers[] | select(.name=="istio-proxy") | length' | wc -l) + POD_SIDECAR_COUNT=$(kubectl get pods -n "${NAMESPACE}" -o json | jq '.items[] | select(.spec.containers[].name == "istio-proxy")' | wc -l) if [ "$POD_COUNT" -ne "$POD_SIDECAR_COUNT" ]; then gh_error " āŒ Not all pods have the istio sidecar" @@ -272,10 +274,10 @@ tasks: ## Should minimize network policies to specific selectors needed for Ingress/Egress traffic. - NETPOL_EXT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | length' | wc -l) + NETPOL_EXT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace")' | jq -s 'length') gh_notice " ā„¹ļø Non-IntraNamespace network policies: $NETPOL_EXT_COUNT" - NETPOL_EXT_SELECTOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | select(.selector != null) | length' | wc -l) + NETPOL_EXT_SELECTOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | select(.selector != null)' | jq -s 'length') gh_notice " ā„¹ļø Non-IntraNamespace network policies with selectors: $NETPOL_EXT_SELECTOR_COUNT" if [ "$NETPOL_EXT_COUNT" -ne "$NETPOL_EXT_SELECTOR_COUNT" ]; then @@ -284,10 +286,10 @@ tasks: gh_notice " āœ… All applicable network policies are using selectors" fi - NETPOL_EXT_NOTKUBE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | select(.remoteGenerated != "KubeAPI") | length' | wc -l) + NETPOL_EXT_NOTKUBE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace" and .remoteGenerated != "KubeAPI")' | jq -s 'length') gh_notice " ā„¹ļø Non-IntraNamespace, non-KubeAPI network policies: $NETPOL_EXT_NOTKUBE_COUNT" - NETPOL_EXT_NOTKUBE_PORT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | select(.remoteGenerated != "KubeAPI") | select(.ports != null or .port != null) | length' | wc -l) + NETPOL_EXT_NOTKUBE_PORT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace" and .remoteGenerated != "KubeAPI") | select(.ports != null or .port != null)' | jq -s 'length') gh_notice " ā„¹ļø Non-IntraNamespace, non-KubeAPI network policies with ports: $NETPOL_EXT_NOTKUBE_PORT_COUNT" if [ "$NETPOL_EXT_NOTKUBE_COUNT" -ne "$NETPOL_EXT_NOTKUBE_PORT_COUNT" ]; then @@ -296,7 +298,7 @@ tasks: gh_notice " āœ… All applicable network policies are using ports" fi - NETPOL_ANYWHERE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated == "Anywhere") | length' | wc -l) + NETPOL_ANYWHERE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated == "Anywhere")' | jq -s 'length') if [ "$NETPOL_ANYWHERE_COUNT" -gt 0 ]; then gh_warning " āš ļø Network policies with 'remoteGenerated: Anywhere' are present, review needed" else @@ -307,7 +309,7 @@ tasks: gh_group "šŸ” Keycloak/SSO Validation" - SSO_CLIENT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso | length') + SSO_CLIENT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso | length') ## Must use and create a Keycloak client through the sso key if the application provides a user login. if [ "$SSO_CLIENT_COUNT" -gt 0 ]; then @@ -323,16 +325,18 @@ tasks: # Should name the client Login (i.e. Mattermost Login) to provide login UX consistency. SSO_CLIENT_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].name') - if [ "$SSO_CLIENT_NAME" != "${UDS_PACKAGE_NAME^} Login" ]; then + EXPECTED_CLIENT_NAME="${UDS_PACKAGE_NAME^} Login" + if [ "$SSO_CLIENT_NAME" != "$EXPECTED_CLIENT_NAME" ]; then gh_warning " āš ļø SSO client name not in preferred format, review needed" else gh_notice " āœ… SSO client name is correct" fi - # Should clearly mark the client id with the group and app name uds-- (i.e. uds-swf-mattermost) to provide consistency in the Keycloak UI. + # Should clearly mark the client id with the group and app name uds--- (i.e. uds-swf-mattermost-saml) to provide consistency in the Keycloak UI. SSO_CLIENT_ID=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].clientId') - if [ "$SSO_CLIENT_ID" != "uds-${GROUP_NAME})-${UDS_PACKAGE_NAME}-${SSO_CLIENT_PROTOCOL}" ]; then - gh_warning " āš ļø SSO client id not in the format uds---protocol, review needed" + EXPECTED_CLIENT_ID="uds-${GROUP_NAME}-${UDS_PACKAGE_NAME}-${SSO_CLIENT_PROTOCOL}" + if [ "$SSO_CLIENT_ID" != "$EXPECTED_CLIENT_ID" ]; then + gh_warning " āš ļø SSO client id not in the format uds---, review needed" else gh_notice " āœ… SSO client id is correct" fi @@ -355,19 +359,19 @@ tasks: gh_group "šŸ” Monitoring Validation" - ## Must implement monitors for each application metrics endpoint using it's built-in chart monitors, the Package CR monitor key, or manual monitors in the config chart. + ## Must implement monitors for each application metrics endpoint using its built-in chart monitors, the Package CR monitor key, or manual monitors in the config chart. PKG_MONITOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.monitor | length') if [ "$PKG_MONITOR_COUNT" -eq 0 ]; then gh_notice " ā„¹ļø No monitors defined in the package, checking for ServiceMonitors" - # Check for built-in serviceMonitors + # Check for built-in ServiceMonitors SERVICE_MONITOR_COUNT=$(kubectl get ServiceMonitor -n "${NAMESPACE}" -o json | jq -r '.items | length') if [ "$SERVICE_MONITOR_COUNT" -eq 0 ]; then gh_notice " ā„¹ļø No ServiceMonitors defined, checking for PodMonitors" - # Check for built-in podMonitors + # Check for built-in PodMonitors POD_MONITOR_COUNT=$(kubectl get PodMonitor -n "${NAMESPACE}" -o json | jq -r '.items | length') if [ "$POD_MONITOR_COUNT" -eq 0 ]; then gh_error " āŒ No monitors defined" @@ -404,7 +408,7 @@ tasks: ## Should expose all configuration (uds.dev CRs, additional Secrets/ConfigMaps, etc) through a Helm chart (ideally in a chart or charts directory). if [ "${COMMON_ZARF}" == "true" ]; then - COMMON_ZARF_MANIFEST_COUNT=$(cat "${PACKAGE_DIR}/common/zarf.yaml" | uds zarf tools yq '.components[] | select(.manifests != null) | length' | wc -l) + COMMON_ZARF_MANIFEST_COUNT=$(uds zarf tools yq '.components[] | select(.manifests != null)' "${PACKAGE_DIR}/common/zarf.yaml" | wc -l) if [ "$COMMON_ZARF_MANIFEST_COUNT" -gt 0 ]; then gh_error " āŒ Manifests present in common/zarf.yaml" else @@ -412,7 +416,7 @@ tasks: fi fi - MAIN_ZARF_MANIFEST_COUNT=$(cat "${PACKAGE_DIR}/zarf.yaml" | uds zarf tools yq '.components[] | select(.manifests != null) | length' | wc -l) + MAIN_ZARF_MANIFEST_COUNT=$(uds zarf tools yq '.components[] | select(.manifests != null)' "${PACKAGE_DIR}/zarf.yaml" | wc -l) if [ "$MAIN_ZARF_MANIFEST_COUNT" -gt 0 ]; then gh_error " āŒ Manifests present in zarf.yaml" else @@ -421,7 +425,7 @@ tasks: ## Should implement or allow for multiple flavors (ideally with common definitions in a common directory) - ZARF_COMPONENTS_FLAVOR_COUNT=$(cat "${PACKAGE_DIR}/zarf.yaml" | uds zarf tools yq '.components[] | select(.only.flavor != null) | length' | wc -l) + ZARF_COMPONENTS_FLAVOR_COUNT=$(uds zarf tools yq '.components[] | select(.only.flavor != null)' "${PACKAGE_DIR}/zarf.yaml" | wc -l) if [ "$ZARF_COMPONENTS_FLAVOR_COUNT" -eq 0 ]; then gh_error " āŒ No flavors defined in zarf.yaml" else @@ -432,10 +436,10 @@ tasks: gh_group "šŸ” Package Testing Validation" - ## Must implement Journey Testing to cover the basic user flows and features of the application, especially where an application interacts with an external service / interface. + ## Must implement Journey Testing to cover the basic user flows and features of the application, especially where an application interacts with an external service/interface. if [ -d "${PACKAGE_DIR}/tests" ]; then - if [ "$(ls -A ${PACKAGE_DIR}/tests)" ]; then + if [ "$(ls -A "${PACKAGE_DIR}/tests")" ]; then gh_notice " āœ… Tests folder exists and has files" else gh_error " āŒ Tests folder exists but is empty" @@ -451,7 +455,7 @@ tasks: # Must be consistently versioned across flavors - this can take many forms but flavors should differ in image bases/builds not application versions. ZARF_PKG_VERSION=$(uds zarf tools yq '.metadata.version' "${PACKAGE_DIR}/zarf.yaml" | cut -d '-' -f1) - IMAGE_WITH_ZARF_VERSION_COUNT=$(cat "${PACKAGE_DIR}/zarf.yaml" | uds zarf tools yq '.components[].images' | grep -o "${ZARF_PKG_VERSION}" | wc -l) + IMAGE_WITH_ZARF_VERSION_COUNT=$(uds zarf tools yq '.components[].images[]' "${PACKAGE_DIR}/zarf.yaml" | grep -o "${ZARF_PKG_VERSION}" | wc -l) if [ "$IMAGE_WITH_ZARF_VERSION_COUNT" -ge "$ZARF_COMPONENTS_FLAVOR_COUNT" ]; then gh_notice " āœ… Version is consistent across flavors and package" From c3ff69df2103507584a0a7b511d166dfa778a33a Mon Sep 17 00:00:00 2001 From: Justin Law Date: Thu, 26 Sep 2024 14:18:58 -0400 Subject: [PATCH 13/41] fixed scripts --- .github/scripts/summarizeVerification.sh | 195 +++--- .../nightly-uds-badge-verification.yaml | 12 +- tasks/badge.yaml | 558 ++++++++++-------- 3 files changed, 398 insertions(+), 367 deletions(-) diff --git a/.github/scripts/summarizeVerification.sh b/.github/scripts/summarizeVerification.sh index 658e5a142..9d053a516 100755 --- a/.github/scripts/summarizeVerification.sh +++ b/.github/scripts/summarizeVerification.sh @@ -1,130 +1,105 @@ #!/bin/bash +# Remove ANSI escape sequences from the data +clean_data=$(sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' output.txt) + # Initialize variables -in_package=false package_name="" -error_count=0 -warning_count=0 +failures_count=0 +errors_count=0 +warnings_count=0 +failure_descriptions=() error_descriptions=() warning_descriptions=() -# Function to strip ANSI escape sequences -strip_ansi() { - echo -e "$1" | sed -r 's/\x1B\[[0-9;]*[a-zA-Z]//g' -} - -while IFS= read -r line; do - # Strip ANSI escape sequences - clean_line=$(strip_ansi "$line") - # Remove leading/trailing whitespace - clean_line=$(echo "$clean_line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - # Check for the start of a new package - if [[ $clean_line =~ ^ā€¢[[:space:]]+Running\ \"Verify\ ([^\"]+)\"$ ]]; then - new_package_name="${BASH_REMATCH[1]}" - if [[ $new_package_name != "that the package meets UDS badging standards" ]]; then - # If we're already in a package, print its summary - if [ "$in_package" = true ]; then - echo -e "Package: $package_name" - echo -e "āŒ Errors: $error_count" - echo -e "āš ļø Warnings: $warning_count" - if [ $error_count -gt 0 ]; then - echo "āŒ Error Descriptions:" - for err in "${error_descriptions[@]}"; do - echo " - $err" - done - fi - if [ $warning_count -gt 0 ]; then - echo "āš ļø Warning Descriptions:" - for warn in "${warning_descriptions[@]}"; do - echo " - $warn" - done - fi - echo "-----------------------------" - fi - # Start new package - in_package=true - package_name="$new_package_name" - error_count=0 - warning_count=0 - error_descriptions=() - warning_descriptions=() +# Function to print package information +print_package_info() { + if [[ -n $package_name ]]; then + echo "-----------------------------" + echo "Package: $package_name" + if ((failures_count > 0)); then + echo "ā›” Failures: $failures_count" fi - fi - - # Capture error descriptions - if [[ $in_package == true && $clean_line =~ ^āŒ ]]; then - # Ignore summary lines - if [[ $clean_line =~ errors\ found$ ]]; then - # Extract the error count from the line - error_count_line=$(echo "$clean_line" | grep -oP '(?<=āŒ )[0-9]+(?= errors found)') - if [[ -n $error_count_line ]]; then - error_count="$error_count_line" - fi - elif [[ $clean_line =~ ^āŒ[[:space:]][0-9]+ ]]; then - continue - else - error_descriptions+=("$(echo "$clean_line" | sed 's/^āŒ[[:space:]]*//')") - ((error_count++)) + if ((errors_count > 0)); then + echo "āŒ Errors: $errors_count" fi - fi - - # Capture warning descriptions - if [[ $in_package == true && $clean_line =~ ^āš ļø ]]; then - # Ignore summary lines - if [[ $clean_line =~ warnings\ found$ ]]; then - # Extract the warning count from the line - warning_count_line=$(echo "$clean_line" | grep -oP '(?<=āš ļø )[0-9]+(?= warnings found)') - if [[ -n $warning_count_line ]]; then - warning_count="$warning_count_line" - fi - elif [[ $clean_line =~ ^āš ļø[[:space:]][0-9]+ ]]; then - continue - else - warning_descriptions+=("$(echo "$clean_line" | sed 's/^āš ļø[[:space:]]*//')") - ((warning_count++)) + if ((warnings_count > 0)); then + echo "āš ļø Warnings: $warnings_count" fi - fi - # Check for the end of a package - if [[ $in_package == true && $clean_line =~ ^āœ”[[:space:]]+Completed\ \"Verify\ $package_name\"$ ]]; then - # Print summary - echo -e "Package: $package_name" - echo -e "āŒ Errors: $error_count" - echo -e "āš ļø Warnings: $warning_count" - if [ $error_count -gt 0 ]; then + if ((failures_count > 0)); then + echo + echo "ā›” Failure Descriptions:" + for desc in "${failure_descriptions[@]}"; do + echo " - $desc" + done + fi + if ((errors_count > 0)); then + echo echo "āŒ Error Descriptions:" - for err in "${error_descriptions[@]}"; do - echo " - $err" + for desc in "${error_descriptions[@]}"; do + echo " - $desc" done fi - if [ $warning_count -gt 0 ]; then + if ((warnings_count > 0)); then + echo echo "āš ļø Warning Descriptions:" - for warn in "${warning_descriptions[@]}"; do - echo " - $warn" + for desc in "${warning_descriptions[@]}"; do + echo " - $desc" done fi - echo "-----------------------------" - in_package=false fi -done > output.txt + uds run verify-uds-badge-cpu 2>&1 | tee output.txt - name: Pretty Summary of UDS Badge Verification run: | chmod +x .github/scripts/summarizeVerification.sh - .github/scripts/summarizeVerification.sh + .github/scripts/summarizeVerification.sh >> report.txt + + - name: Archive UDS Badge Verification Report + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 + if: ${{ !cancelled() }} + with: + name: uds-badge-verification-report + path: report.txt + retention-days: 30 diff --git a/tasks/badge.yaml b/tasks/badge.yaml index 1313d89b2..afe42417c 100644 --- a/tasks/badge.yaml +++ b/tasks/badge.yaml @@ -2,16 +2,16 @@ variables: - name: CHART_PATH - description: Relative path to the directory with the uds-config chart + description: "Relative path to the directory with the uds-config chart" default: chart/ - name: GROUP_NAME - description: The name of the niche/group that the package fits into. E.g. package,swf,lfai + description: "The name of the niche/group that the package fits into (e.g. package,swf,lfai)" default: package - name: COMMON_ZARF - description: Whether or not there is a common zarf.yaml file + description: "Whether or not there is a common zarf.yaml file" default: "true" - name: PACKAGE_DIR - description: Base directory where packages are located, relative to the root of the repository + description: "Base directory where packages are located, relative to the root of the repository" default: . tasks: @@ -112,56 +112,75 @@ tasks: ########### - name: verify-badge actions: - - description: Verify that the package meets UDS badging standards + - description: "Verify that the package meets UDS badging standards" shell: linux: bash darwin: bash cmd: | - # Functions + ######## + # COUNTS + ######## - WARNINGS=0 - ERRORS=0 + WARNINGS_COUNT=0 + ERRORS_COUNT=0 + FAILURES_COUNT=0 + + ########### + # FUNCTIONS + ########### gh_notice() { - if [[ "${CI}" == "true" ]]; then - printf "::notice::" - fi - echo "$1" + if [[ "${CI}" == "true" ]]; then + printf "::notice::" + fi + echo "$1" } gh_warning() { - if [[ "${CI}" == "true" ]]; then - printf "::warning::" - fi - WARNINGS=$((WARNINGS + 1)) - echo "$1" + if [[ "${CI}" == "true" ]]; then + printf "::warning::" + fi + WARNINGS_COUNT=$((WARNINGS_COUNT + 1)) + echo "$1" } gh_error() { - if [[ "${CI}" == "true" ]]; then - printf "::error::" - fi - ERRORS=$((ERRORS + 1)) - echo "$1" + if [[ "${CI}" == "true" ]]; then + printf "::error::" + fi + ERRORS_COUNT=$((ERRORS_COUNT + 1)) + echo "$1" } - gh_group() { - if [[ "${CI}" == "true" ]]; then - echo "::group::$1" - else + gh_failure() { + if [[ "${CI}" == "true" ]]; then + printf "::failure::" + fi + FAILURES_COUNT=$((FAILURES_COUNT + 1)) echo "$1" - fi + } + + gh_group() { + if [[ "${CI}" == "true" ]]; then + echo "::group::$1" + else + echo "$1" + fi } gh_endgroup() { - if [[ "${CI}" == "true" ]]; then - echo "::endgroup::" - else - echo - fi + if [[ "${CI}" == "true" ]]; then + echo "::endgroup::" + else + echo + fi } - gh_group "šŸ“ƒ Setup" + ################ + # INITIALIZATION + ################ + + gh_group "šŸ“ƒ Setup Package Variables" # Default values for vars, maru-runner issue workaround CHART_PATH=${CHART_PATH:-chart/} @@ -173,306 +192,335 @@ tasks: PACKAGE_DIR=${PACKAGE_DIR:-"."} gh_notice " ā„¹ļø Package Directory: $PACKAGE_DIR" + # Extract UDS_PACKAGE_NAME from zarf.yaml + UDS_PACKAGE_NAME=$(uds zarf tools yq '.metadata.name' "${PACKAGE_DIR}/zarf.yaml") + gh_notice " ā„¹ļø Package Name: $UDS_PACKAGE_NAME" + # Cover when GOBIN isn't set export GOBIN=/tmp/gobin gh_notice " ā„¹ļø Installing kubectl-validate..." if go install sigs.k8s.io/kubectl-validate@latest; then - gh_notice " āœ… kubectl-validate installed" + gh_notice " āœ… kubectl-validate installed" else - gh_error " āŒ kubectl-validate failed to install" + gh_error " āŒ kubectl-validate failed to install" fi + gh_endgroup + + ######################## + # STATIC CODE VALIDATION + ######################## + + gh_group "šŸ” Package Structure Validation" + if [[ "${COMMON_ZARF}" == "true" ]]; then - NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}/common/zarf.yaml" | uniq) + NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}/common/zarf.yaml" | uniq) else - gh_warning " āš ļø There is no common zarf.yaml file" - NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}/zarf.yaml" | uniq) + gh_warning " āš ļø There is no common zarf.yaml file" + NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}/zarf.yaml" | uniq) fi gh_notice " ā„¹ļø Namespace: $NAMESPACE" - # Extract UDS_PACKAGE_NAME from zarf.yaml - UDS_PACKAGE_NAME=$(uds zarf tools yq '.metadata.name' "${PACKAGE_DIR}/zarf.yaml") - gh_notice " ā„¹ļø Package Name: $UDS_PACKAGE_NAME" - - # Retrieve the specific Package resource - UDS_PACKAGE_JSON=$(kubectl get Package "$UDS_PACKAGE_NAME" -n "${NAMESPACE}" -o json) - gh_notice " ā„¹ļø Retrieved UDS Package JSON for $UDS_PACKAGE_NAME" - - gh_endgroup - - gh_group "šŸ” Config Chart Validation" + # Should expose all configuration (uds.dev CRs, additional Secrets/ConfigMaps, etc) through a Helm chart (ideally in a chart or charts directory). + if [ "${COMMON_ZARF}" == "true" ]; then + COMMON_ZARF_MANIFEST_COUNT=$(uds zarf tools yq '.components[] | select(.manifests != null)' "${PACKAGE_DIR}/common/zarf.yaml" | wc -l) + if [ "$COMMON_ZARF_MANIFEST_COUNT" -gt 0 ]; then + gh_error " āŒ Manifests present in common/zarf.yaml" + else + gh_notice " āœ… No manifests present in common/zarf.yaml" + fi + fi - # Validate UDS Package/Config chart is valid + MAIN_ZARF_MANIFEST_COUNT=$(uds zarf tools yq '.components[] | select(.manifests != null)' "${PACKAGE_DIR}/zarf.yaml" | wc -l) + if [ "$MAIN_ZARF_MANIFEST_COUNT" -gt 0 ]; then + gh_error " āŒ Manifests present in zarf.yaml" + else + gh_notice " āœ… No manifests present in zarf.yaml" + fi - gh_notice " ā„¹ļø Validating uds-config chart..." - printf "\033[A" - if ${GOBIN}/kubectl-validate <(helm template "${PACKAGE_DIR}/${CHART_PATH}" | grep -v '^ *#' | perl -p0e 's/---[\s---]*---/---/sg') &> /tmp/validate-output; then - gh_notice " āœ… uds-config chart is valid" + # Should implement or allow for multiple flavors (ideally with common definitions in a common directory) + ZARF_COMPONENTS_FLAVOR_COUNT=$(uds zarf tools yq '.components[] | select(.only.flavor != null)' "${PACKAGE_DIR}/zarf.yaml" | wc -l) + if [ "$ZARF_COMPONENTS_FLAVOR_COUNT" -eq 0 ]; then + gh_error " āŒ No flavors defined in zarf.yaml" else - gh_error " āŒ uds-config chart is invalid" - cat /tmp/validate-output | sed 's/^/ /' + gh_notice " āœ… At least one flavor defined in zarf.yaml" fi gh_endgroup - gh_group "šŸ” Istio Validation" - - ## Must define any external interfaces under the expose key. + gh_group "šŸ” Package Testing Validation" - ENDPOINTS="$(echo "$UDS_PACKAGE_JSON" | jq -r '.status.endpoints[]')" - gh_notice " ā„¹ļø Endpoints: $ENDPOINTS" + # Must implement Journey Testing to cover the basic user flows and features of the application, especially where an application interacts with an external service/interface. + if [ -d "${PACKAGE_DIR}/tests" ]; then + if [ "$(ls -A "${PACKAGE_DIR}/tests")" ]; then + gh_notice " āœ… Tests folder exists and has files" + else + gh_error " āŒ Tests folder exists but is empty" + fi + else + # Tests might be at the root of a mono-repository + if [ -d "tests" ]; then + if [ "$(ls -A "tests")" ]; then + gh_notice " āœ… Tests folder exists and has files" + else + gh_error " āŒ Tests folder exists but is empty" + fi + else + gh_error " āŒ Tests are not present for this package" + fi + fi - for ENDPOINT in $ENDPOINTS; do - # Curl endpoint and check that status isn't 404 or 5XX - STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://$ENDPOINT") - if [ "$STATUS" -eq 404 ] || [[ $STATUS == 5* ]]; then - gh_error " āŒ Endpoint $ENDPOINT is returning $STATUS" - else - gh_notice " āœ… Endpoint $ENDPOINT was successfully curl'd" - fi - done + gh_endgroup - echo + gh_group "šŸ” Versioning Validation" - ## Must deploy and operate successfully with Istio injection enabled in the namespace. + # Must be consistently versioned across flavors - this can take many forms but flavors should differ in image bases/builds not application versions. - POD_COUNT=$(kubectl get pods -n "${NAMESPACE}" --no-headers | wc -l) - POD_SIDECAR_COUNT=$(kubectl get pods -n "${NAMESPACE}" -o json | jq '.items[] | select(.spec.containers[].name == "istio-proxy")' | wc -l) + ZARF_PKG_VERSION=$(uds zarf tools yq '.metadata.version' "${PACKAGE_DIR}/zarf.yaml" | cut -d '-' -f1) + IMAGE_WITH_ZARF_VERSION_COUNT=$(uds zarf tools yq '.components[].images[]' "${PACKAGE_DIR}/zarf.yaml" | grep -o "${ZARF_PKG_VERSION}" | wc -l) - if [ "$POD_COUNT" -ne "$POD_SIDECAR_COUNT" ]; then - gh_error " āŒ Not all pods have the istio sidecar" + if [ "$IMAGE_WITH_ZARF_VERSION_COUNT" -ge "$ZARF_COMPONENTS_FLAVOR_COUNT" ]; then + gh_notice " āœ… Version is consistent across flavors and package" else - gh_notice " āœ… All pods have the istio sidecar" + gh_warning " āš ļø Version is not consistent across flavors and package" fi - echo - - ## Should avoid workarounds such as disabling strict mTLS peer authentication. + gh_endgroup - PEERAUTH=$(kubectl get Peerauthentication -n "${NAMESPACE}" -o=json | jq -r '.items[].spec.mtls.mode') + gh_group "šŸ” UDS Config Chart Validation" - if [ "$PEERAUTH" != "STRICT" ] && [ "$PEERAUTH" != "" ]; then - gh_warning " āš ļø Peerauth is not strict or inherited, review needed" + # Validate UDS Package/Config chart is valid + gh_notice " ā„¹ļø Validating uds-config chart..." + if ${GOBIN}/kubectl-validate <(helm template "${PACKAGE_DIR}/${CHART_PATH}" | grep -v '^ *#' | perl -p0e 's/---[\s---]*---/---/sg') &>/tmp/validate-output; then + gh_notice " āœ… uds-config chart is valid" else - gh_notice " āœ… Peerauthentication is set to strict" + gh_error " āŒ uds-config chart is invalid" + cat /tmp/validate-output | sed 's/^/ /' fi - echo - - # Network Policies + gh_endgroup - ## Must define network policies under the allow key as required. + ############################# + # DYNAMIC PACKAGE VALIDATIONS + ############################# - NETPOL_AMOUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow | length') + gh_group "šŸ” Exemptions Validation" - if [ "$NETPOL_AMOUNT" -eq 0 ]; then - gh_warning " āš ļø No network policies defined, review needed" + # Must minimize the scope and number of the exemptions to only what is absolutely required by the application + EXEMPTION_COUNT=$(uds zarf tools kubectl get Exemptions -n "${NAMESPACE}" -o json | jq -r '.items | length') + if [ "$EXEMPTION_COUNT" -gt 0 ]; then + gh_warning " āš ļø Exemptions are present, review needed" else - gh_notice " āœ… Network policies are defined" + gh_notice " āœ… No exemptions present" fi - ## Should minimize network policies to specific selectors needed for Ingress/Egress traffic. - - NETPOL_EXT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace")' | jq -s 'length') - gh_notice " ā„¹ļø Non-IntraNamespace network policies: $NETPOL_EXT_COUNT" + gh_endgroup - NETPOL_EXT_SELECTOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | select(.selector != null)' | jq -s 'length') - gh_notice " ā„¹ļø Non-IntraNamespace network policies with selectors: $NETPOL_EXT_SELECTOR_COUNT" + UDS_PACKAGE_CR_EXISTS="false" - if [ "$NETPOL_EXT_COUNT" -ne "$NETPOL_EXT_SELECTOR_COUNT" ]; then - gh_error " āŒ Not all applicable network policies are using selectors" - else - gh_notice " āœ… All applicable network policies are using selectors" - fi + gh_group "šŸ“ƒ Get the UDS Package CR" - NETPOL_EXT_NOTKUBE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace" and .remoteGenerated != "KubeAPI")' | jq -s 'length') - gh_notice " ā„¹ļø Non-IntraNamespace, non-KubeAPI network policies: $NETPOL_EXT_NOTKUBE_COUNT" + # The following may fail if the UDS Package CR does not exist + set +e - NETPOL_EXT_NOTKUBE_PORT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace" and .remoteGenerated != "KubeAPI") | select(.ports != null or .port != null)' | jq -s 'length') - gh_notice " ā„¹ļø Non-IntraNamespace, non-KubeAPI network policies with ports: $NETPOL_EXT_NOTKUBE_PORT_COUNT" + # Try to retrieve the specific Package resource + UDS_PACKAGE_JSON=$(uds zarf tools kubectl get Package "$UDS_PACKAGE_NAME" -n "${NAMESPACE}" -o json 2>/dev/null) - if [ "$NETPOL_EXT_NOTKUBE_COUNT" -ne "$NETPOL_EXT_NOTKUBE_PORT_COUNT" ]; then - gh_error " āŒ Not all applicable network policies are using ports" + # Check if the kubectl command succeeded + if [ $? -eq 0 ]; then + UDS_PACKAGE_CR_EXISTS="true" + gh_notice " ā„¹ļø Retrieved UDS Package JSON for $UDS_PACKAGE_NAME" else - gh_notice " āœ… All applicable network policies are using ports" + gh_failure " ā›” UDS Package CR does not exist for $UDS_PACKAGE_NAME" fi - NETPOL_ANYWHERE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated == "Anywhere")' | jq -s 'length') - if [ "$NETPOL_ANYWHERE_COUNT" -gt 0 ]; then - gh_warning " āš ļø Network policies with 'remoteGenerated: Anywhere' are present, review needed" - else - gh_notice " āœ… No network policies with 'remoteGenerated: Anywhere' are present" - fi + set -e gh_endgroup - gh_group "šŸ” Keycloak/SSO Validation" - - SSO_CLIENT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso | length') - - ## Must use and create a Keycloak client through the sso key if the application provides a user login. - if [ "$SSO_CLIENT_COUNT" -gt 0 ]; then - gh_notice " ā„¹ļø There are $SSO_CLIENT_COUNT SSO clients defined" - - # Should consider security options during implementation to provide the most secure default possible (i.e. SAML w/SCIM vs OIDC). - SSO_CLIENT_PROTOCOL=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].protocol') - if [ "$SSO_CLIENT_PROTOCOL" != "saml" ]; then - gh_warning " āš ļø SAML is not the default protocol, review needed" - else - gh_notice " āœ… Default protocol is SAML" - fi - - # Should name the client Login (i.e. Mattermost Login) to provide login UX consistency. - SSO_CLIENT_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].name') - EXPECTED_CLIENT_NAME="${UDS_PACKAGE_NAME^} Login" - if [ "$SSO_CLIENT_NAME" != "$EXPECTED_CLIENT_NAME" ]; then - gh_warning " āš ļø SSO client name not in preferred format, review needed" - else - gh_notice " āœ… SSO client name is correct" - fi - - # Should clearly mark the client id with the group and app name uds--- (i.e. uds-swf-mattermost-saml) to provide consistency in the Keycloak UI. - SSO_CLIENT_ID=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].clientId') - EXPECTED_CLIENT_ID="uds-${GROUP_NAME}-${UDS_PACKAGE_NAME}-${SSO_CLIENT_PROTOCOL}" - if [ "$SSO_CLIENT_ID" != "$EXPECTED_CLIENT_ID" ]; then - gh_warning " āš ļø SSO client id not in the format uds---, review needed" - else - gh_notice " āœ… SSO client id is correct" - fi - - ## May end any generated secrets with -sso to easily locate them when querying the cluster. - SSO_SECRET_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].secretName') - if [ "$SSO_SECRET_NAME" != "" ]; then - gh_notice " ā„¹ļø SSO secret name is defined" - if [[ "$SSO_SECRET_NAME" != *-sso ]]; then - gh_warning " āš ļø SSO secret name not in the format -sso, review needed" - else - gh_notice " āœ… SSO secret name is in the correct format" - fi - fi - else - gh_warning " āš ļø No SSO configuration found, review needed" - fi - - gh_endgroup + if [ "$UDS_PACKAGE_CR_EXISTS" == "true" ]; then - gh_group "šŸ” Monitoring Validation" + gh_group "šŸ” Istio Validation" - ## Must implement monitors for each application metrics endpoint using its built-in chart monitors, the Package CR monitor key, or manual monitors in the config chart. + # Must define any external interfaces under the expose key. + ENDPOINTS="$(echo "$UDS_PACKAGE_JSON" | jq -r '.status.endpoints[]')" + gh_notice " ā„¹ļø Endpoints: $ENDPOINTS" - PKG_MONITOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.monitor | length') + for ENDPOINT in $ENDPOINTS; do + # Curl endpoint and check that status isn't 404 or 5XX + STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://$ENDPOINT") + if [ "$STATUS" -eq 404 ] || [[ $STATUS == 5* ]]; then + gh_error " āŒ Endpoint $ENDPOINT is returning $STATUS" + else + gh_notice " āœ… Endpoint $ENDPOINT was successfully curl'd" + fi + done - if [ "$PKG_MONITOR_COUNT" -eq 0 ]; then - gh_notice " ā„¹ļø No monitors defined in the package, checking for ServiceMonitors" + echo - # Check for built-in ServiceMonitors - SERVICE_MONITOR_COUNT=$(kubectl get ServiceMonitor -n "${NAMESPACE}" -o json | jq -r '.items | length') - if [ "$SERVICE_MONITOR_COUNT" -eq 0 ]; then - gh_notice " ā„¹ļø No ServiceMonitors defined, checking for PodMonitors" + # Must deploy and operate successfully with Istio injection enabled in the namespace. + POD_COUNT=$(uds zarf tools kubectl get pods -n "${NAMESPACE}" --no-headers | wc -l) + POD_SIDECAR_COUNT=$(uds zarf tools kubectl get pods -n "${NAMESPACE}" -o json | jq '.items[] | select(.spec.containers[].name == "istio-proxy")' | wc -l) - # Check for built-in PodMonitors - POD_MONITOR_COUNT=$(kubectl get PodMonitor -n "${NAMESPACE}" -o json | jq -r '.items | length') - if [ "$POD_MONITOR_COUNT" -eq 0 ]; then - gh_error " āŒ No monitors defined" + if [ "$POD_COUNT" -ne "$POD_SIDECAR_COUNT" ]; then + gh_error " āŒ Not all pods have the istio sidecar" else - gh_notice " āœ… PodMonitor found" - gh_warning " āš ļø There may be more monitors to implement, review needed" + gh_notice " āœ… All pods have the istio sidecar" fi - else - gh_notice " āœ… ServiceMonitor found" - gh_warning " āš ļø There may be more monitors to implement, review needed" - fi - else - gh_notice " āœ… Monitor defined in UDS Package" - gh_warning " āš ļø There may be more monitors to implement, review needed" - fi - - gh_endgroup - - gh_group "šŸ” Exemptions Validation" - ## Must minimize the scope and number of the exemptions to only what is absolutely required by the application - - EXEMPTION_COUNT=$(kubectl get Exemptions -n "${NAMESPACE}" -o json | jq -r '.items | length') - if [ "$EXEMPTION_COUNT" -gt 0 ]; then - gh_warning " āš ļø Exemptions are present, review needed" - else - gh_notice " āœ… No exemptions present" - fi - - gh_endgroup + echo - gh_group "šŸ” Package Structure Validation" + # Should avoid workarounds such as disabling strict mTLS peer authentication. + PEERAUTH=$(uds zarf tools kubectl get Peerauthentication -n "${NAMESPACE}" -o=json | jq -r '.items[].spec.mtls.mode') - ## Should expose all configuration (uds.dev CRs, additional Secrets/ConfigMaps, etc) through a Helm chart (ideally in a chart or charts directory). + if [ "$PEERAUTH" != "STRICT" ] && [ "$PEERAUTH" != "" ]; then + gh_warning " āš ļø Peerauth is not strict or inherited, review needed" + else + gh_notice " āœ… Peerauthentication is set to strict" + fi - if [ "${COMMON_ZARF}" == "true" ]; then - COMMON_ZARF_MANIFEST_COUNT=$(uds zarf tools yq '.components[] | select(.manifests != null)' "${PACKAGE_DIR}/common/zarf.yaml" | wc -l) - if [ "$COMMON_ZARF_MANIFEST_COUNT" -gt 0 ]; then - gh_error " āŒ Manifests present in common/zarf.yaml" - else - gh_notice " āœ… No manifests present in common/zarf.yaml" - fi - fi + echo - MAIN_ZARF_MANIFEST_COUNT=$(uds zarf tools yq '.components[] | select(.manifests != null)' "${PACKAGE_DIR}/zarf.yaml" | wc -l) - if [ "$MAIN_ZARF_MANIFEST_COUNT" -gt 0 ]; then - gh_error " āŒ Manifests present in zarf.yaml" - else - gh_notice " āœ… No manifests present in zarf.yaml" - fi + # Must define network policies under the allow key as required. + NETPOL_AMOUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow | length') - ## Should implement or allow for multiple flavors (ideally with common definitions in a common directory) + if [ "$NETPOL_AMOUNT" -eq 0 ]; then + gh_warning " āš ļø No network policies defined, review needed" + else + gh_notice " āœ… Network policies are defined" + fi - ZARF_COMPONENTS_FLAVOR_COUNT=$(uds zarf tools yq '.components[] | select(.only.flavor != null)' "${PACKAGE_DIR}/zarf.yaml" | wc -l) - if [ "$ZARF_COMPONENTS_FLAVOR_COUNT" -eq 0 ]; then - gh_error " āŒ No flavors defined in zarf.yaml" - else - gh_notice " āœ… At least one flavor defined in zarf.yaml" - fi + # Should minimize network policies to specific selectors needed for Ingress/Egress traffic. + NETPOL_EXT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace")' | jq -s 'length') + gh_notice " ā„¹ļø Non-IntraNamespace network policies: $NETPOL_EXT_COUNT" - gh_endgroup + NETPOL_EXT_SELECTOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | select(.selector != null)' | jq -s 'length') + gh_notice " ā„¹ļø Non-IntraNamespace network policies with selectors: $NETPOL_EXT_SELECTOR_COUNT" - gh_group "šŸ” Package Testing Validation" + if [ "$NETPOL_EXT_COUNT" -ne "$NETPOL_EXT_SELECTOR_COUNT" ]; then + gh_error " āŒ Not all applicable network policies are using selectors" + else + gh_notice " āœ… All applicable network policies are using selectors" + fi - ## Must implement Journey Testing to cover the basic user flows and features of the application, especially where an application interacts with an external service/interface. + NETPOL_EXT_NOTKUBE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace" and .remoteGenerated != "KubeAPI")' | jq -s 'length') + gh_notice " ā„¹ļø Non-IntraNamespace, non-KubeAPI network policies: $NETPOL_EXT_NOTKUBE_COUNT" - if [ -d "${PACKAGE_DIR}/tests" ]; then - if [ "$(ls -A "${PACKAGE_DIR}/tests")" ]; then - gh_notice " āœ… Tests folder exists and has files" - else - gh_error " āŒ Tests folder exists but is empty" - fi - else - gh_error " āŒ Tests folder does not exist" - fi + NETPOL_EXT_NOTKUBE_PORT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace" and .remoteGenerated != "KubeAPI") | select(.ports != null or .port != null)' | jq -s 'length') + gh_notice " ā„¹ļø Non-IntraNamespace, non-KubeAPI network policies with ports: $NETPOL_EXT_NOTKUBE_PORT_COUNT" - gh_endgroup + if [ "$NETPOL_EXT_NOTKUBE_COUNT" -ne "$NETPOL_EXT_NOTKUBE_PORT_COUNT" ]; then + gh_error " āŒ Not all applicable network policies are using ports" + else + gh_notice " āœ… All applicable network policies are using ports" + fi - gh_group "šŸ” Versioning Validation" + NETPOL_ANYWHERE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated == "Anywhere")' | jq -s 'length') + if [ "$NETPOL_ANYWHERE_COUNT" -gt 0 ]; then + gh_warning " āš ļø Network policies with 'remoteGenerated: Anywhere' are present, review needed" + else + gh_notice " āœ… No network policies with 'remoteGenerated: Anywhere' are present" + fi - # Must be consistently versioned across flavors - this can take many forms but flavors should differ in image bases/builds not application versions. + gh_endgroup + + gh_group "šŸ” Keycloak/SSO Validation" + + SSO_CLIENT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso | length') + + # Must use and create a Keycloak client through the sso key if the application provides a user login. + if [ "$SSO_CLIENT_COUNT" -gt 0 ]; then + gh_notice " ā„¹ļø There are $SSO_CLIENT_COUNT SSO clients defined" + + # Should consider security options during implementation to provide the most secure default possible (i.e. SAML w/SCIM vs OIDC). + SSO_CLIENT_PROTOCOL=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].protocol') + if [ "$SSO_CLIENT_PROTOCOL" != "saml" ]; then + gh_warning " āš ļø SAML is not the default protocol, review needed" + else + gh_notice " āœ… Default protocol is SAML" + fi + + # Should name the client Login (i.e. Mattermost Login) to provide login UX consistency. + SSO_CLIENT_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].name') + EXPECTED_CLIENT_NAME="${UDS_PACKAGE_NAME^} Login" + if [ "$SSO_CLIENT_NAME" != "$EXPECTED_CLIENT_NAME" ]; then + gh_warning " āš ļø SSO client name not in preferred format, review needed" + else + gh_notice " āœ… SSO client name is correct" + fi + + # Should clearly mark the client id with the group and app name uds--- (i.e. uds-swf-mattermost-saml) to provide consistency in the Keycloak UI. + SSO_CLIENT_ID=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].clientId') + EXPECTED_CLIENT_ID="uds-${GROUP_NAME}-${UDS_PACKAGE_NAME}-${SSO_CLIENT_PROTOCOL}" + if [ "$SSO_CLIENT_ID" != "$EXPECTED_CLIENT_ID" ]; then + gh_warning " āš ļø SSO client id not in the format uds---, review needed" + else + gh_notice " āœ… SSO client id is correct" + fi + + ## May end any generated secrets with -sso to easily locate them when querying the cluster. + SSO_SECRET_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].secretName') + if [ "$SSO_SECRET_NAME" != "" ]; then + gh_notice " ā„¹ļø SSO secret name is defined" + if [[ "$SSO_SECRET_NAME" != *-sso ]]; then + gh_warning " āš ļø SSO secret name not in the format -sso, review needed" + else + gh_notice " āœ… SSO secret name is in the correct format" + fi + fi + else + gh_warning " āš ļø No SSO configuration found, review needed" + fi - ZARF_PKG_VERSION=$(uds zarf tools yq '.metadata.version' "${PACKAGE_DIR}/zarf.yaml" | cut -d '-' -f1) - IMAGE_WITH_ZARF_VERSION_COUNT=$(uds zarf tools yq '.components[].images[]' "${PACKAGE_DIR}/zarf.yaml" | grep -o "${ZARF_PKG_VERSION}" | wc -l) + gh_endgroup + + gh_group "šŸ” Monitoring Validation" + + # Must implement monitors for each application metrics endpoint using its built-in chart monitors, the Package CR monitor key, or manual monitors in the config chart. + PKG_MONITOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.monitor | length') + + if [ "$PKG_MONITOR_COUNT" -eq 0 ]; then + gh_notice " ā„¹ļø No monitors defined in the package, checking for ServiceMonitors" + + # Check for built-in ServiceMonitors + SERVICE_MONITOR_COUNT=$(uds zarf tools kubectl get ServiceMonitor -n "${NAMESPACE}" -o json | jq -r '.items | length') + if [ "$SERVICE_MONITOR_COUNT" -eq 0 ]; then + gh_notice " ā„¹ļø No ServiceMonitors defined, checking for PodMonitors" + + # Check for built-in PodMonitors + POD_MONITOR_COUNT=$(uds zarf tools kubectl get PodMonitor -n "${NAMESPACE}" -o json | jq -r '.items | length') + if [ "$POD_MONITOR_COUNT" -eq 0 ]; then + gh_error " āŒ No monitors defined" + else + gh_notice " āœ… PodMonitor found" + gh_warning " āš ļø There may be more monitors to implement, review needed" + fi + else + gh_notice " āœ… ServiceMonitor found" + gh_warning " āš ļø There may be more monitors to implement, review needed" + fi + else + gh_notice " āœ… Monitor defined in UDS Package" + gh_warning " āš ļø There may be more monitors to implement, review needed" + fi - if [ "$IMAGE_WITH_ZARF_VERSION_COUNT" -ge "$ZARF_COMPONENTS_FLAVOR_COUNT" ]; then - gh_notice " āœ… Version is consistent across flavors and package" - else - gh_warning " āš ļø Version is not consistent across flavors and package" + gh_endgroup fi - gh_endgroup - - # Summary + ######### + # SUMMARY + ######### - if [ "$ERRORS" -gt 0 ]; then - gh_error "āŒ $ERRORS errors found" + if [ "$FAILURES_COUNT" -gt 0 ]; then + gh_failure "ā›” $FAILURES_COUNT failures found" + fi + if [ "$ERRORS_COUNT" -gt 0 ]; then + gh_error "āŒ $ERRORS_COUNT errors found" fi - if [ "$WARNINGS" -gt 0 ]; then - gh_warning "āš ļø $WARNINGS warnings found" + if [ "$WARNINGS_COUNT" -gt 0 ]; then + gh_warning "āš ļø $WARNINGS_COUNT warnings found" fi - if [ "$ERRORS" -eq 0 ] && [ "$WARNINGS" -eq 0 ]; then - gh_notice "āœ… No errors or warnings found" + if [ "$FAILURES_COUNT" -eq 0 ] && [ "$ERRORS_COUNT" -eq 0 ] && [ "$WARNINGS_COUNT" -eq 0 ]; then + gh_notice "āœ… No failures, errors nor warnings found" fi From 3c1a518e81f7e36b1c8713d038663b93f50701b5 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Thu, 26 Sep 2024 14:28:41 -0400 Subject: [PATCH 14/41] trigger at 3 AM PST --- .github/workflows/nightly-uds-badge-verification.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index ed4012f95..4fddf4146 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -2,7 +2,7 @@ name: nightly-uds-badge-verification on: schedule: - - cron: "0 8 * * *" # Runs daily at 12 AM PST + - cron: "0 11 * * *" # Runs daily at 12 AM PST workflow_dispatch: # trigger manually as needed pull_request: types: From 159c23618052517e63f96f07cc704f0da9a607db Mon Sep 17 00:00:00 2001 From: Justin Law Date: Thu, 26 Sep 2024 15:37:00 -0400 Subject: [PATCH 15/41] fix reporting mechanism --- .github/scripts/summarizeVerification.sh | 2 +- .github/workflows/nightly-uds-badge-verification.yaml | 7 +++---- .gitignore | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/scripts/summarizeVerification.sh b/.github/scripts/summarizeVerification.sh index 9d053a516..8f5907329 100755 --- a/.github/scripts/summarizeVerification.sh +++ b/.github/scripts/summarizeVerification.sh @@ -1,7 +1,7 @@ #!/bin/bash # Remove ANSI escape sequences from the data -clean_data=$(sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' output.txt) +clean_data=$(sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' temp/output.txt) # Initialize variables package_name="" diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index 4fddf4146..3d78f8593 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -74,17 +74,16 @@ jobs: - name: Run UDS Badge Verification Task run: | - uds run verify-uds-badge-cpu 2>&1 | tee output.txt + uds run verify-uds-badge-cpu 2>&1 | tee temp/output.txt - name: Pretty Summary of UDS Badge Verification run: | chmod +x .github/scripts/summarizeVerification.sh - .github/scripts/summarizeVerification.sh >> report.txt + .github/scripts/summarizeVerification.sh 2>&1 | tee temp/report.txt - name: Archive UDS Badge Verification Report uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 - if: ${{ !cancelled() }} with: name: uds-badge-verification-report - path: report.txt + path: temp/report.txt retention-days: 30 diff --git a/.gitignore b/.gitignore index dbc0825ce..645bd6ff5 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ node_modules package.json package-lock.json **/*.schema.json -output.txt # local model and tokenizer files *.bin From d171d6956f8545314e8cc0bcc90e38d5a50f3d5b Mon Sep 17 00:00:00 2001 From: Justin Law Date: Thu, 26 Sep 2024 16:03:12 -0400 Subject: [PATCH 16/41] fixed scripts, pt.2 --- .github/workflows/nightly-uds-badge-verification.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index 3d78f8593..c24c32aba 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -74,7 +74,8 @@ jobs: - name: Run UDS Badge Verification Task run: | - uds run verify-uds-badge-cpu 2>&1 | tee temp/output.txt + mkdir -p temp/ + uds run verify-uds-badge-cpu --no-progress 2>&1 | tee temp/output.txt - name: Pretty Summary of UDS Badge Verification run: | From ca2a230ae92ac0f0f7ed262dd31d6fec97646cac Mon Sep 17 00:00:00 2001 From: Justin Law Date: Thu, 26 Sep 2024 16:47:36 -0400 Subject: [PATCH 17/41] fix report generation, again --- .github/workflows/nightly-uds-badge-verification.yaml | 8 +++++--- tasks/badge.yaml | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index c24c32aba..858e15e8c 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -76,9 +76,11 @@ jobs: run: | mkdir -p temp/ uds run verify-uds-badge-cpu --no-progress 2>&1 | tee temp/output.txt - - - name: Pretty Summary of UDS Badge Verification - run: | + echo + echo "================================================================" + echo "=========================== SUMMARY ============================" + echo "================================================================" + echo chmod +x .github/scripts/summarizeVerification.sh .github/scripts/summarizeVerification.sh 2>&1 | tee temp/report.txt diff --git a/tasks/badge.yaml b/tasks/badge.yaml index afe42417c..dfe15ff80 100644 --- a/tasks/badge.yaml +++ b/tasks/badge.yaml @@ -154,7 +154,7 @@ tasks: gh_failure() { if [[ "${CI}" == "true" ]]; then - printf "::failure::" + printf "::error::" fi FAILURES_COUNT=$((FAILURES_COUNT + 1)) echo "$1" From 2a8aebc8160a718cd161c17f78dcd6171bd1edce Mon Sep 17 00:00:00 2001 From: Justin Law Date: Thu, 26 Sep 2024 18:32:11 -0400 Subject: [PATCH 18/41] fix report generation, finally --- .github/scripts/summarizeVerification.sh | 59 +- .../nightly-uds-badge-verification.yaml | 15 +- tasks.yaml | 122 +++- tasks/badge.yaml | 526 ------------------ 4 files changed, 149 insertions(+), 573 deletions(-) delete mode 100644 tasks/badge.yaml diff --git a/.github/scripts/summarizeVerification.sh b/.github/scripts/summarizeVerification.sh index 8f5907329..baeb32fc7 100755 --- a/.github/scripts/summarizeVerification.sh +++ b/.github/scripts/summarizeVerification.sh @@ -1,7 +1,7 @@ #!/bin/bash # Remove ANSI escape sequences from the data -clean_data=$(sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' temp/output.txt) +clean_data=$(sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g') # Initialize variables package_name="" @@ -14,17 +14,30 @@ warning_descriptions=() # Function to print package information print_package_info() { + if [[ ${CI} == "true" ]]; then + echo "::group::$package_name" + fi if [[ -n $package_name ]]; then echo "-----------------------------" echo "Package: $package_name" if ((failures_count > 0)); then + if [[ ${CI} == "true" ]]; then + printf "::error::" + fi echo "ā›” Failures: $failures_count" - fi - if ((errors_count > 0)); then - echo "āŒ Errors: $errors_count" - fi - if ((warnings_count > 0)); then - echo "āš ļø Warnings: $warnings_count" + else + if ((errors_count > 0)); then + if [[ ${CI} == "true" ]]; then + printf "::error::" + fi + echo "āŒ Errors: $errors_count" + fi + if ((warnings_count > 0)); then + if [[ ${CI} == "true" ]]; then + printf "::warning::" + fi + echo "āš ļø Warnings: $warnings_count" + fi fi if ((failures_count > 0)); then @@ -33,21 +46,25 @@ print_package_info() { for desc in "${failure_descriptions[@]}"; do echo " - $desc" done + else + if ((errors_count > 0)); then + echo + echo "āŒ Error Descriptions:" + for desc in "${error_descriptions[@]}"; do + echo " - $desc" + done + fi + if ((warnings_count > 0)); then + echo + echo "āš ļø Warning Descriptions:" + for desc in "${warning_descriptions[@]}"; do + echo " - $desc" + done + fi fi - if ((errors_count > 0)); then - echo - echo "āŒ Error Descriptions:" - for desc in "${error_descriptions[@]}"; do - echo " - $desc" - done - fi - if ((warnings_count > 0)); then - echo - echo "āš ļø Warning Descriptions:" - for desc in "${warning_descriptions[@]}"; do - echo " - $desc" - done - fi + fi + if [[ ${CI} == "true" ]]; then + echo "::endgroup::" fi } diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index 858e15e8c..798d06f2d 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -74,19 +74,14 @@ jobs: - name: Run UDS Badge Verification Task run: | - mkdir -p temp/ - uds run verify-uds-badge-cpu --no-progress 2>&1 | tee temp/output.txt - echo - echo "================================================================" - echo "=========================== SUMMARY ============================" - echo "================================================================" - echo - chmod +x .github/scripts/summarizeVerification.sh - .github/scripts/summarizeVerification.sh 2>&1 | tee temp/report.txt + mkdir -p temp/report + uds run verify-uds-badge-cpu --no-progress 2>&1 | sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' \ + | tee temp/report/intermediate-report.txt \ + | .github/scripts/summarizeVerification.sh | tee temp/report/final-report.txt - name: Archive UDS Badge Verification Report uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: uds-badge-verification-report - path: temp/report.txt + path: temp/report retention-days: 30 diff --git a/tasks.yaml b/tasks.yaml index 756d09aab..d211ae047 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -1,30 +1,120 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json includes: - - badge: ./tasks/badge.yaml + - badge: https://raw.githubusercontent.com/defenseunicorns/uds-common/refs/heads/main/tasks/badge.yaml tasks: ############# - # Badge Tasks + # BADGE TASKS ############# - name: verify-uds-badge-cpu description: "Runs through all CPU UDS bundle packages with the UDS badge verification test" actions: - - task: badge:verify-uds-badge-api - - task: badge:verify-uds-badge-ui - - task: badge:verify-uds-badge-llama-cpp-python - - task: badge:verify-uds-badge-vllm - - task: badge:verify-uds-badge-text-embeddings - - task: badge:verify-uds-badge-whisper - - task: badge:verify-uds-badge-repeater - - task: badge:verify-uds-badge-supabase + - task: verify-uds-badge-api + - task: verify-uds-badge-ui + - task: verify-uds-badge-llama-cpp-python + - task: verify-uds-badge-text-embeddings + - task: verify-uds-badge-whisper + - task: verify-uds-badge-supabase - name: verify-uds-badge-gpu description: "Runs through all GPU UDS bundle packages with the UDS badge verification test" actions: - - task: badge:verify-uds-badge-api - - task: badge:verify-uds-badge-ui - - task: badge:verify-uds-badge-vllm - - task: badge:verify-uds-badge-text-embeddings - - task: badge:verify-uds-badge-whisper - - task: badge:verify-uds-badge-supabase + - task: verify-uds-badge-api + - task: verify-uds-badge-ui + - task: verify-uds-badge-vllm + - task: verify-uds-badge-text-embeddings + - task: verify-uds-badge-whisper + - task: verify-uds-badge-supabase + + ####################### + # RE-USABLE BADGE TASKS + ####################### + + - name: verify-uds-badge-api + actions: + - description: "Verify API" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="true" \ + --set PACKAGE_DIR="packages/api" \ + --no-progress + + - name: verify-uds-badge-ui + actions: + - description: "Verify UI" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/ui" \ + --no-progress + + - name: verify-uds-badge-llama-cpp-python + actions: + - description: "Verify LLaMA-CPP-Python" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/llama-cpp-python" \ + --no-progress + + - name: verify-uds-badge-vllm + actions: + - description: "Verify vLLM" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/vllm" \ + --no-progress + + - name: verify-uds-badge-text-embeddings + actions: + - description: "Verify Text-Embeddings" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/text-embeddings" \ + --no-progress + + - name: verify-uds-badge-whisper + actions: + - description: "Verify Whisper" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/whisper" \ + --no-progress + + - name: verify-uds-badge-repeater + actions: + - description: "Verify Repeater" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/repeater" \ + --no-progress + + - name: verify-uds-badge-supabase + actions: + - description: "Verify Supabase" + cmd: | + uds run badge:verify-badge \ + --set CHART_PATH="chart" \ + --set GROUP_NAME="package" \ + --set COMMON_ZARF="false" \ + --set PACKAGE_DIR="packages/supabase" \ + --no-progress diff --git a/tasks/badge.yaml b/tasks/badge.yaml deleted file mode 100644 index dfe15ff80..000000000 --- a/tasks/badge.yaml +++ /dev/null @@ -1,526 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/uds-cli/main/tasks.schema.json - -variables: - - name: CHART_PATH - description: "Relative path to the directory with the uds-config chart" - default: chart/ - - name: GROUP_NAME - description: "The name of the niche/group that the package fits into (e.g. package,swf,lfai)" - default: package - - name: COMMON_ZARF - description: "Whether or not there is a common zarf.yaml file" - default: "true" - - name: PACKAGE_DIR - description: "Base directory where packages are located, relative to the root of the repository" - default: . - -tasks: - ############## - # PACKAGE-LEVEL - ############## - - - name: verify-uds-badge-api - actions: - - description: "Verify API" - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="true" \ - --set PACKAGE_DIR="packages/api" \ - --no-progress - - - name: verify-uds-badge-ui - actions: - - description: "Verify UI" - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/ui" \ - --no-progress - - - name: verify-uds-badge-llama-cpp-python - actions: - - description: "Verify LLaMA-CPP-Python" - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/llama-cpp-python" \ - --no-progress - - - name: verify-uds-badge-vllm - actions: - - description: "Verify vLLM" - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/vllm" \ - --no-progress - - - name: verify-uds-badge-text-embeddings - actions: - - description: "Verify Text-Embeddings" - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/text-embeddings" \ - --no-progress - - - name: verify-uds-badge-whisper - actions: - - description: "Verify Whisper" - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/whisper" \ - --no-progress - - - name: verify-uds-badge-repeater - actions: - - description: "Verify Repeater" - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/repeater" \ - --no-progress - - - name: verify-uds-badge-supabase - actions: - - description: "Verify Supabase" - cmd: | - uds run badge:verify-badge \ - --set CHART_PATH="chart" \ - --set GROUP_NAME="package" \ - --set COMMON_ZARF="false" \ - --set PACKAGE_DIR="packages/supabase" \ - --no-progress - - ########### - # RE-USABLE - ########### - - name: verify-badge - actions: - - description: "Verify that the package meets UDS badging standards" - shell: - linux: bash - darwin: bash - cmd: | - ######## - # COUNTS - ######## - - WARNINGS_COUNT=0 - ERRORS_COUNT=0 - FAILURES_COUNT=0 - - ########### - # FUNCTIONS - ########### - - gh_notice() { - if [[ "${CI}" == "true" ]]; then - printf "::notice::" - fi - echo "$1" - } - - gh_warning() { - if [[ "${CI}" == "true" ]]; then - printf "::warning::" - fi - WARNINGS_COUNT=$((WARNINGS_COUNT + 1)) - echo "$1" - } - - gh_error() { - if [[ "${CI}" == "true" ]]; then - printf "::error::" - fi - ERRORS_COUNT=$((ERRORS_COUNT + 1)) - echo "$1" - } - - gh_failure() { - if [[ "${CI}" == "true" ]]; then - printf "::error::" - fi - FAILURES_COUNT=$((FAILURES_COUNT + 1)) - echo "$1" - } - - gh_group() { - if [[ "${CI}" == "true" ]]; then - echo "::group::$1" - else - echo "$1" - fi - } - - gh_endgroup() { - if [[ "${CI}" == "true" ]]; then - echo "::endgroup::" - else - echo - fi - } - - ################ - # INITIALIZATION - ################ - - gh_group "šŸ“ƒ Setup Package Variables" - - # Default values for vars, maru-runner issue workaround - CHART_PATH=${CHART_PATH:-chart/} - gh_notice " ā„¹ļø Chart Path: $CHART_PATH" - GROUP_NAME=${GROUP_NAME:-package} - gh_notice " ā„¹ļø Group Name: $GROUP_NAME" - COMMON_ZARF=${COMMON_ZARF:-true} - gh_notice " ā„¹ļø Common Zarf: $COMMON_ZARF" - PACKAGE_DIR=${PACKAGE_DIR:-"."} - gh_notice " ā„¹ļø Package Directory: $PACKAGE_DIR" - - # Extract UDS_PACKAGE_NAME from zarf.yaml - UDS_PACKAGE_NAME=$(uds zarf tools yq '.metadata.name' "${PACKAGE_DIR}/zarf.yaml") - gh_notice " ā„¹ļø Package Name: $UDS_PACKAGE_NAME" - - # Cover when GOBIN isn't set - export GOBIN=/tmp/gobin - - gh_notice " ā„¹ļø Installing kubectl-validate..." - if go install sigs.k8s.io/kubectl-validate@latest; then - gh_notice " āœ… kubectl-validate installed" - else - gh_error " āŒ kubectl-validate failed to install" - fi - - gh_endgroup - - ######################## - # STATIC CODE VALIDATION - ######################## - - gh_group "šŸ” Package Structure Validation" - - if [[ "${COMMON_ZARF}" == "true" ]]; then - NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}/common/zarf.yaml" | uniq) - else - gh_warning " āš ļø There is no common zarf.yaml file" - NAMESPACE=$(uds zarf tools yq '.components[].charts[].namespace' "${PACKAGE_DIR}/zarf.yaml" | uniq) - fi - gh_notice " ā„¹ļø Namespace: $NAMESPACE" - - # Should expose all configuration (uds.dev CRs, additional Secrets/ConfigMaps, etc) through a Helm chart (ideally in a chart or charts directory). - if [ "${COMMON_ZARF}" == "true" ]; then - COMMON_ZARF_MANIFEST_COUNT=$(uds zarf tools yq '.components[] | select(.manifests != null)' "${PACKAGE_DIR}/common/zarf.yaml" | wc -l) - if [ "$COMMON_ZARF_MANIFEST_COUNT" -gt 0 ]; then - gh_error " āŒ Manifests present in common/zarf.yaml" - else - gh_notice " āœ… No manifests present in common/zarf.yaml" - fi - fi - - MAIN_ZARF_MANIFEST_COUNT=$(uds zarf tools yq '.components[] | select(.manifests != null)' "${PACKAGE_DIR}/zarf.yaml" | wc -l) - if [ "$MAIN_ZARF_MANIFEST_COUNT" -gt 0 ]; then - gh_error " āŒ Manifests present in zarf.yaml" - else - gh_notice " āœ… No manifests present in zarf.yaml" - fi - - # Should implement or allow for multiple flavors (ideally with common definitions in a common directory) - ZARF_COMPONENTS_FLAVOR_COUNT=$(uds zarf tools yq '.components[] | select(.only.flavor != null)' "${PACKAGE_DIR}/zarf.yaml" | wc -l) - if [ "$ZARF_COMPONENTS_FLAVOR_COUNT" -eq 0 ]; then - gh_error " āŒ No flavors defined in zarf.yaml" - else - gh_notice " āœ… At least one flavor defined in zarf.yaml" - fi - - gh_endgroup - - gh_group "šŸ” Package Testing Validation" - - # Must implement Journey Testing to cover the basic user flows and features of the application, especially where an application interacts with an external service/interface. - if [ -d "${PACKAGE_DIR}/tests" ]; then - if [ "$(ls -A "${PACKAGE_DIR}/tests")" ]; then - gh_notice " āœ… Tests folder exists and has files" - else - gh_error " āŒ Tests folder exists but is empty" - fi - else - # Tests might be at the root of a mono-repository - if [ -d "tests" ]; then - if [ "$(ls -A "tests")" ]; then - gh_notice " āœ… Tests folder exists and has files" - else - gh_error " āŒ Tests folder exists but is empty" - fi - else - gh_error " āŒ Tests are not present for this package" - fi - fi - - gh_endgroup - - gh_group "šŸ” Versioning Validation" - - # Must be consistently versioned across flavors - this can take many forms but flavors should differ in image bases/builds not application versions. - - ZARF_PKG_VERSION=$(uds zarf tools yq '.metadata.version' "${PACKAGE_DIR}/zarf.yaml" | cut -d '-' -f1) - IMAGE_WITH_ZARF_VERSION_COUNT=$(uds zarf tools yq '.components[].images[]' "${PACKAGE_DIR}/zarf.yaml" | grep -o "${ZARF_PKG_VERSION}" | wc -l) - - if [ "$IMAGE_WITH_ZARF_VERSION_COUNT" -ge "$ZARF_COMPONENTS_FLAVOR_COUNT" ]; then - gh_notice " āœ… Version is consistent across flavors and package" - else - gh_warning " āš ļø Version is not consistent across flavors and package" - fi - - gh_endgroup - - gh_group "šŸ” UDS Config Chart Validation" - - # Validate UDS Package/Config chart is valid - gh_notice " ā„¹ļø Validating uds-config chart..." - if ${GOBIN}/kubectl-validate <(helm template "${PACKAGE_DIR}/${CHART_PATH}" | grep -v '^ *#' | perl -p0e 's/---[\s---]*---/---/sg') &>/tmp/validate-output; then - gh_notice " āœ… uds-config chart is valid" - else - gh_error " āŒ uds-config chart is invalid" - cat /tmp/validate-output | sed 's/^/ /' - fi - - gh_endgroup - - ############################# - # DYNAMIC PACKAGE VALIDATIONS - ############################# - - gh_group "šŸ” Exemptions Validation" - - # Must minimize the scope and number of the exemptions to only what is absolutely required by the application - EXEMPTION_COUNT=$(uds zarf tools kubectl get Exemptions -n "${NAMESPACE}" -o json | jq -r '.items | length') - if [ "$EXEMPTION_COUNT" -gt 0 ]; then - gh_warning " āš ļø Exemptions are present, review needed" - else - gh_notice " āœ… No exemptions present" - fi - - gh_endgroup - - UDS_PACKAGE_CR_EXISTS="false" - - gh_group "šŸ“ƒ Get the UDS Package CR" - - # The following may fail if the UDS Package CR does not exist - set +e - - # Try to retrieve the specific Package resource - UDS_PACKAGE_JSON=$(uds zarf tools kubectl get Package "$UDS_PACKAGE_NAME" -n "${NAMESPACE}" -o json 2>/dev/null) - - # Check if the kubectl command succeeded - if [ $? -eq 0 ]; then - UDS_PACKAGE_CR_EXISTS="true" - gh_notice " ā„¹ļø Retrieved UDS Package JSON for $UDS_PACKAGE_NAME" - else - gh_failure " ā›” UDS Package CR does not exist for $UDS_PACKAGE_NAME" - fi - - set -e - - gh_endgroup - - if [ "$UDS_PACKAGE_CR_EXISTS" == "true" ]; then - - gh_group "šŸ” Istio Validation" - - # Must define any external interfaces under the expose key. - ENDPOINTS="$(echo "$UDS_PACKAGE_JSON" | jq -r '.status.endpoints[]')" - gh_notice " ā„¹ļø Endpoints: $ENDPOINTS" - - for ENDPOINT in $ENDPOINTS; do - # Curl endpoint and check that status isn't 404 or 5XX - STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://$ENDPOINT") - if [ "$STATUS" -eq 404 ] || [[ $STATUS == 5* ]]; then - gh_error " āŒ Endpoint $ENDPOINT is returning $STATUS" - else - gh_notice " āœ… Endpoint $ENDPOINT was successfully curl'd" - fi - done - - echo - - # Must deploy and operate successfully with Istio injection enabled in the namespace. - POD_COUNT=$(uds zarf tools kubectl get pods -n "${NAMESPACE}" --no-headers | wc -l) - POD_SIDECAR_COUNT=$(uds zarf tools kubectl get pods -n "${NAMESPACE}" -o json | jq '.items[] | select(.spec.containers[].name == "istio-proxy")' | wc -l) - - if [ "$POD_COUNT" -ne "$POD_SIDECAR_COUNT" ]; then - gh_error " āŒ Not all pods have the istio sidecar" - else - gh_notice " āœ… All pods have the istio sidecar" - fi - - echo - - # Should avoid workarounds such as disabling strict mTLS peer authentication. - PEERAUTH=$(uds zarf tools kubectl get Peerauthentication -n "${NAMESPACE}" -o=json | jq -r '.items[].spec.mtls.mode') - - if [ "$PEERAUTH" != "STRICT" ] && [ "$PEERAUTH" != "" ]; then - gh_warning " āš ļø Peerauth is not strict or inherited, review needed" - else - gh_notice " āœ… Peerauthentication is set to strict" - fi - - echo - - # Must define network policies under the allow key as required. - NETPOL_AMOUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow | length') - - if [ "$NETPOL_AMOUNT" -eq 0 ]; then - gh_warning " āš ļø No network policies defined, review needed" - else - gh_notice " āœ… Network policies are defined" - fi - - # Should minimize network policies to specific selectors needed for Ingress/Egress traffic. - NETPOL_EXT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace")' | jq -s 'length') - gh_notice " ā„¹ļø Non-IntraNamespace network policies: $NETPOL_EXT_COUNT" - - NETPOL_EXT_SELECTOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace") | select(.selector != null)' | jq -s 'length') - gh_notice " ā„¹ļø Non-IntraNamespace network policies with selectors: $NETPOL_EXT_SELECTOR_COUNT" - - if [ "$NETPOL_EXT_COUNT" -ne "$NETPOL_EXT_SELECTOR_COUNT" ]; then - gh_error " āŒ Not all applicable network policies are using selectors" - else - gh_notice " āœ… All applicable network policies are using selectors" - fi - - NETPOL_EXT_NOTKUBE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace" and .remoteGenerated != "KubeAPI")' | jq -s 'length') - gh_notice " ā„¹ļø Non-IntraNamespace, non-KubeAPI network policies: $NETPOL_EXT_NOTKUBE_COUNT" - - NETPOL_EXT_NOTKUBE_PORT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated != "IntraNamespace" and .remoteGenerated != "KubeAPI") | select(.ports != null or .port != null)' | jq -s 'length') - gh_notice " ā„¹ļø Non-IntraNamespace, non-KubeAPI network policies with ports: $NETPOL_EXT_NOTKUBE_PORT_COUNT" - - if [ "$NETPOL_EXT_NOTKUBE_COUNT" -ne "$NETPOL_EXT_NOTKUBE_PORT_COUNT" ]; then - gh_error " āŒ Not all applicable network policies are using ports" - else - gh_notice " āœ… All applicable network policies are using ports" - fi - - NETPOL_ANYWHERE_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.network.allow[] | select(.remoteGenerated == "Anywhere")' | jq -s 'length') - if [ "$NETPOL_ANYWHERE_COUNT" -gt 0 ]; then - gh_warning " āš ļø Network policies with 'remoteGenerated: Anywhere' are present, review needed" - else - gh_notice " āœ… No network policies with 'remoteGenerated: Anywhere' are present" - fi - - gh_endgroup - - gh_group "šŸ” Keycloak/SSO Validation" - - SSO_CLIENT_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso | length') - - # Must use and create a Keycloak client through the sso key if the application provides a user login. - if [ "$SSO_CLIENT_COUNT" -gt 0 ]; then - gh_notice " ā„¹ļø There are $SSO_CLIENT_COUNT SSO clients defined" - - # Should consider security options during implementation to provide the most secure default possible (i.e. SAML w/SCIM vs OIDC). - SSO_CLIENT_PROTOCOL=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].protocol') - if [ "$SSO_CLIENT_PROTOCOL" != "saml" ]; then - gh_warning " āš ļø SAML is not the default protocol, review needed" - else - gh_notice " āœ… Default protocol is SAML" - fi - - # Should name the client Login (i.e. Mattermost Login) to provide login UX consistency. - SSO_CLIENT_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].name') - EXPECTED_CLIENT_NAME="${UDS_PACKAGE_NAME^} Login" - if [ "$SSO_CLIENT_NAME" != "$EXPECTED_CLIENT_NAME" ]; then - gh_warning " āš ļø SSO client name not in preferred format, review needed" - else - gh_notice " āœ… SSO client name is correct" - fi - - # Should clearly mark the client id with the group and app name uds--- (i.e. uds-swf-mattermost-saml) to provide consistency in the Keycloak UI. - SSO_CLIENT_ID=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].clientId') - EXPECTED_CLIENT_ID="uds-${GROUP_NAME}-${UDS_PACKAGE_NAME}-${SSO_CLIENT_PROTOCOL}" - if [ "$SSO_CLIENT_ID" != "$EXPECTED_CLIENT_ID" ]; then - gh_warning " āš ļø SSO client id not in the format uds---, review needed" - else - gh_notice " āœ… SSO client id is correct" - fi - - ## May end any generated secrets with -sso to easily locate them when querying the cluster. - SSO_SECRET_NAME=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.sso[].secretName') - if [ "$SSO_SECRET_NAME" != "" ]; then - gh_notice " ā„¹ļø SSO secret name is defined" - if [[ "$SSO_SECRET_NAME" != *-sso ]]; then - gh_warning " āš ļø SSO secret name not in the format -sso, review needed" - else - gh_notice " āœ… SSO secret name is in the correct format" - fi - fi - else - gh_warning " āš ļø No SSO configuration found, review needed" - fi - - gh_endgroup - - gh_group "šŸ” Monitoring Validation" - - # Must implement monitors for each application metrics endpoint using its built-in chart monitors, the Package CR monitor key, or manual monitors in the config chart. - PKG_MONITOR_COUNT=$(echo "$UDS_PACKAGE_JSON" | jq -r '.spec.monitor | length') - - if [ "$PKG_MONITOR_COUNT" -eq 0 ]; then - gh_notice " ā„¹ļø No monitors defined in the package, checking for ServiceMonitors" - - # Check for built-in ServiceMonitors - SERVICE_MONITOR_COUNT=$(uds zarf tools kubectl get ServiceMonitor -n "${NAMESPACE}" -o json | jq -r '.items | length') - if [ "$SERVICE_MONITOR_COUNT" -eq 0 ]; then - gh_notice " ā„¹ļø No ServiceMonitors defined, checking for PodMonitors" - - # Check for built-in PodMonitors - POD_MONITOR_COUNT=$(uds zarf tools kubectl get PodMonitor -n "${NAMESPACE}" -o json | jq -r '.items | length') - if [ "$POD_MONITOR_COUNT" -eq 0 ]; then - gh_error " āŒ No monitors defined" - else - gh_notice " āœ… PodMonitor found" - gh_warning " āš ļø There may be more monitors to implement, review needed" - fi - else - gh_notice " āœ… ServiceMonitor found" - gh_warning " āš ļø There may be more monitors to implement, review needed" - fi - else - gh_notice " āœ… Monitor defined in UDS Package" - gh_warning " āš ļø There may be more monitors to implement, review needed" - fi - - gh_endgroup - fi - - ######### - # SUMMARY - ######### - - if [ "$FAILURES_COUNT" -gt 0 ]; then - gh_failure "ā›” $FAILURES_COUNT failures found" - fi - if [ "$ERRORS_COUNT" -gt 0 ]; then - gh_error "āŒ $ERRORS_COUNT errors found" - fi - if [ "$WARNINGS_COUNT" -gt 0 ]; then - gh_warning "āš ļø $WARNINGS_COUNT warnings found" - fi - if [ "$FAILURES_COUNT" -eq 0 ] && [ "$ERRORS_COUNT" -eq 0 ] && [ "$WARNINGS_COUNT" -eq 0 ]; then - gh_notice "āœ… No failures, errors nor warnings found" - fi From bb65a2dcc2c45e0e7f413a31f99d1d1e1a9cc8ea Mon Sep 17 00:00:00 2001 From: Justin Law Date: Thu, 26 Sep 2024 18:42:02 -0400 Subject: [PATCH 19/41] pin upstream badge action --- tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.yaml b/tasks.yaml index d211ae047..2d8e6c3b2 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -1,7 +1,7 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/uds-cli/v0.14.0/tasks.schema.json includes: - - badge: https://raw.githubusercontent.com/defenseunicorns/uds-common/refs/heads/main/tasks/badge.yaml + - badge: https://raw.githubusercontent.com/defenseunicorns/uds-common/82e63be82766a2e550a847af904b2d738c9d3478/tasks/badge.yaml tasks: ############# From ed368e516eb333736770e890493a043b5cc682df Mon Sep 17 00:00:00 2001 From: Justin Law <81255462+justinthelaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 19:26:17 -0400 Subject: [PATCH 20/41] permissions, break clean into separate line --- .github/workflows/nightly-uds-badge-verification.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index 798d06f2d..eca044b7f 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -75,9 +75,11 @@ jobs: - name: Run UDS Badge Verification Task run: | mkdir -p temp/report + chmod +x .github/scripts/summarizeVerification.sh uds run verify-uds-badge-cpu --no-progress 2>&1 | sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' \ - | tee temp/report/intermediate-report.txt \ - | .github/scripts/summarizeVerification.sh | tee temp/report/final-report.txt + | tee temp/report/intermediate-report.txt + echo temp/report/intermediate-report.txt 2>&1 | .github/scripts/summarizeVerification.sh \ + | tee temp/report/final-report.txt - name: Archive UDS Badge Verification Report uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 From 30543ed92cf0840d18112ff305e7492a5b92a8fe Mon Sep 17 00:00:00 2001 From: Justin Law <81255462+justinthelaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 20:08:47 -0400 Subject: [PATCH 21/41] workflow typo, extraneous triggers --- .github/workflows/nightly-uds-badge-verification.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index eca044b7f..3c1f189ff 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -5,12 +5,6 @@ on: - cron: "0 11 * * *" # Runs daily at 12 AM PST workflow_dispatch: # trigger manually as needed pull_request: - types: - - opened # default trigger - - reopened # default trigger - - synchronize # default trigger - - ready_for_review # don't run on draft PRs - - milestoned # allows us to trigger on bot PRs paths: - .github/workflows/nightly-uds-badge-verification.yaml @@ -76,9 +70,9 @@ jobs: run: | mkdir -p temp/report chmod +x .github/scripts/summarizeVerification.sh - uds run verify-uds-badge-cpu --no-progress 2>&1 | sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' \ + uds run verify-uds-badge-cpu --no-progress 2>&1 | sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' 2>&1 \ | tee temp/report/intermediate-report.txt - echo temp/report/intermediate-report.txt 2>&1 | .github/scripts/summarizeVerification.sh \ + cat temp/report/intermediate-report.txt 2>&1 | .github/scripts/summarizeVerification.sh \ | tee temp/report/final-report.txt - name: Archive UDS Badge Verification Report From a4fe7d84dcb89473c69135be2c24042cf8c8569a Mon Sep 17 00:00:00 2001 From: Justin Law Date: Fri, 27 Sep 2024 14:40:04 -0400 Subject: [PATCH 22/41] ui fix, change to task runner --- .github/scripts/summarizeVerification.sh | 6 +++--- .../workflows/nightly-uds-badge-verification.yaml | 8 ++------ packages/ui/chart/templates/ui/service.yaml | 8 -------- tasks.yaml | 14 ++++++++++++++ 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/scripts/summarizeVerification.sh b/.github/scripts/summarizeVerification.sh index baeb32fc7..430d9d0c0 100755 --- a/.github/scripts/summarizeVerification.sh +++ b/.github/scripts/summarizeVerification.sh @@ -14,11 +14,11 @@ warning_descriptions=() # Function to print package information print_package_info() { - if [[ ${CI} == "true" ]]; then - echo "::group::$package_name" - fi if [[ -n $package_name ]]; then echo "-----------------------------" + if [[ ${CI} == "true" ]]; then + echo "::group::$package_name" + fi echo "Package: $package_name" if ((failures_count > 0)); then if [[ ${CI} == "true" ]]; then diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index 3c1f189ff..4898b0276 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -7,6 +7,7 @@ on: pull_request: paths: - .github/workflows/nightly-uds-badge-verification.yaml + - tasks.yaml concurrency: group: nightly-uds-badge-verification-${{ github.ref }} @@ -68,12 +69,7 @@ jobs: - name: Run UDS Badge Verification Task run: | - mkdir -p temp/report - chmod +x .github/scripts/summarizeVerification.sh - uds run verify-uds-badge-cpu --no-progress 2>&1 | sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' 2>&1 \ - | tee temp/report/intermediate-report.txt - cat temp/report/intermediate-report.txt 2>&1 | .github/scripts/summarizeVerification.sh \ - | tee temp/report/final-report.txt + uds run nightly-uds-badge-verification --no-progress - name: Archive UDS Badge Verification Report uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 diff --git a/packages/ui/chart/templates/ui/service.yaml b/packages/ui/chart/templates/ui/service.yaml index 15243e806..2cb919567 100644 --- a/packages/ui/chart/templates/ui/service.yaml +++ b/packages/ui/chart/templates/ui/service.yaml @@ -18,11 +18,3 @@ spec: protocol: TCP port: {{ .Values.service.port }} targetPort: {{ .Values.service.port }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "chart.serviceAccountName" . }} - namespace: {{ .Release.Namespace | default "leapfrogai" }} - labels: - {{- include "chart.labels" . | nindent 4 }} diff --git a/tasks.yaml b/tasks.yaml index 2d8e6c3b2..28468cfba 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -4,6 +4,20 @@ includes: - badge: https://raw.githubusercontent.com/defenseunicorns/uds-common/82e63be82766a2e550a847af904b2d738c9d3478/tasks/badge.yaml tasks: + - name: nightly-uds-badge-verification + description: "Runs in a pipeline and produces a report for archiving" + actions: + - description: "Create Reports Directory" + cmd: | + mkdir -p temp/report + - description: "Run Upstream UDS Badge Verification Task" + cmd: | + uds run verify-uds-badge-cpu --no-progress 2>&1 | sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' 2>&1 | tee temp/report/intermediate-report.txt > /dev/null + - description: "Clean Up Final Report" + cmd: | + chmod +x .github/scripts/summarizeVerification.sh + cat temp/report/intermediate-report.txt 2>&1 | .github/scripts/summarizeVerification.sh | tee temp/report/final-report.txt + ############# # BADGE TASKS ############# From 6ee71f390d7945d226e9f1518cf0919f037fc984 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Fri, 27 Sep 2024 14:42:33 -0400 Subject: [PATCH 23/41] keep report for 7 days only --- .github/workflows/nightly-uds-badge-verification.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index 4898b0276..f415da1f0 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -76,4 +76,4 @@ jobs: with: name: uds-badge-verification-report path: temp/report - retention-days: 30 + retention-days: 7 From 92d8ceb05b99a247dfafdd9211ec17b1c412bbcc Mon Sep 17 00:00:00 2001 From: Justin Law Date: Fri, 27 Sep 2024 15:19:40 -0400 Subject: [PATCH 24/41] diff dir for reports --- .gitignore | 1 + tasks.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 645bd6ff5..d0c8a20f3 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ node_modules package.json package-lock.json **/*.schema.json +reports # local model and tokenizer files *.bin diff --git a/tasks.yaml b/tasks.yaml index 28468cfba..465c9c597 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -9,14 +9,14 @@ tasks: actions: - description: "Create Reports Directory" cmd: | - mkdir -p temp/report - - description: "Run Upstream UDS Badge Verification Task" + mkdir -p reports + - description: "Run UDS Badge Verification Task" cmd: | - uds run verify-uds-badge-cpu --no-progress 2>&1 | sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' 2>&1 | tee temp/report/intermediate-report.txt > /dev/null + uds run verify-uds-badge-cpu --no-progress 2>&1 | sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' 2>&1 | tee ./reports/intermediate-report.txt > /dev/null - description: "Clean Up Final Report" cmd: | chmod +x .github/scripts/summarizeVerification.sh - cat temp/report/intermediate-report.txt 2>&1 | .github/scripts/summarizeVerification.sh | tee temp/report/final-report.txt + cat ./reports/intermediate-report.txt 2>&1 | .github/scripts/summarizeVerification.sh | tee ./reports/final-report.txt ############# # BADGE TASKS From 8277313174e90bfa8dbc742c4ba700747edf7eb6 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Fri, 27 Sep 2024 15:25:13 -0400 Subject: [PATCH 25/41] test --- .../workflows/nightly-uds-badge-verification.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index f415da1f0..385a49a82 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -59,13 +59,13 @@ jobs: uds zarf tools yq -i '.packages[].repository |= sub("/uds/", "/uds/snapshots/")' bundles/latest/cpu/uds-bundle.yaml - - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }}) - run: | - cd bundles/latest/cpu - uds create . --confirm && \ - uds deploy uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst --confirm --no-progress && \ - rm -rf uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst && \ - docker system prune -af + # - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }}) + # run: | + # cd bundles/latest/cpu + # uds create . --confirm && \ + # uds deploy uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst --confirm --no-progress && \ + # rm -rf uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst && \ + # docker system prune -af - name: Run UDS Badge Verification Task run: | From 3efda1021524a1ccc0b0deefc8a46314c1a04afc Mon Sep 17 00:00:00 2001 From: Justin Law Date: Fri, 27 Sep 2024 16:51:09 -0400 Subject: [PATCH 26/41] test, pt.2 --- .github/workflows/nightly-uds-badge-verification.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index 385a49a82..dd5107a01 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -75,5 +75,5 @@ jobs: uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: uds-badge-verification-report - path: temp/report + path: reports retention-days: 7 From 5be843133e1a6c188062b953d4baaad7e63c8cf7 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Fri, 27 Sep 2024 17:24:47 -0400 Subject: [PATCH 27/41] ignore tasks.yaml moving forward --- .github/workflows/e2e-llama-cpp-python.yaml | 1 + .github/workflows/e2e-playwright.yaml | 3 ++- .github/workflows/e2e-text-backend-full-cpu.yaml | 1 + .github/workflows/e2e-text-embeddings.yaml | 1 + .github/workflows/e2e-vllm.yaml | 1 + .github/workflows/e2e-whisper.yaml | 1 + 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-llama-cpp-python.yaml b/.github/workflows/e2e-llama-cpp-python.yaml index e3d573bba..416146b78 100644 --- a/.github/workflows/e2e-llama-cpp-python.yaml +++ b/.github/workflows/e2e-llama-cpp-python.yaml @@ -32,6 +32,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" diff --git a/.github/workflows/e2e-playwright.yaml b/.github/workflows/e2e-playwright.yaml index 7200155fe..cf2db1d47 100644 --- a/.github/workflows/e2e-playwright.yaml +++ b/.github/workflows/e2e-playwright.yaml @@ -34,6 +34,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" @@ -120,7 +121,7 @@ jobs: - name: UI/API/Supabase E2E Playwright Tests run: | cp src/leapfrogai_ui/.env.example src/leapfrogai_ui/.env - rm src/leapfrogai_ui/tests/global.teardown.ts + rm src/leapfrogai_ui/tests/global.teardown.ts mkdir -p src/leapfrogai_ui/playwright/.auth SERVICE_ROLE_KEY=$(uds zarf tools kubectl get secret -n leapfrogai supabase-bootstrap-jwt -o jsonpath={.data.service-key} | base64 -d) echo "::add-mask::$SERVICE_ROLE_KEY" diff --git a/.github/workflows/e2e-text-backend-full-cpu.yaml b/.github/workflows/e2e-text-backend-full-cpu.yaml index 6e8507ae3..7cb282d47 100644 --- a/.github/workflows/e2e-text-backend-full-cpu.yaml +++ b/.github/workflows/e2e-text-backend-full-cpu.yaml @@ -32,6 +32,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" diff --git a/.github/workflows/e2e-text-embeddings.yaml b/.github/workflows/e2e-text-embeddings.yaml index 20f7eb97a..f60919724 100644 --- a/.github/workflows/e2e-text-embeddings.yaml +++ b/.github/workflows/e2e-text-embeddings.yaml @@ -32,6 +32,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" diff --git a/.github/workflows/e2e-vllm.yaml b/.github/workflows/e2e-vllm.yaml index 07e9f046f..e808f15ea 100644 --- a/.github/workflows/e2e-vllm.yaml +++ b/.github/workflows/e2e-vllm.yaml @@ -32,6 +32,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" diff --git a/.github/workflows/e2e-whisper.yaml b/.github/workflows/e2e-whisper.yaml index dee2cf45a..a7cd17e14 100644 --- a/.github/workflows/e2e-whisper.yaml +++ b/.github/workflows/e2e-whisper.yaml @@ -32,6 +32,7 @@ on: # Ignore local development files - "!.pre-commit-config.yaml" + - "!tasks.yaml" # Ignore non e2e tests changes - "!tests/pytest/**" From 18a92d09cd447f521d5ebcd67ec074b0b0875049 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Fri, 27 Sep 2024 17:34:26 -0400 Subject: [PATCH 28/41] test, pt.3 --- .github/scripts/summarizeVerification.sh | 2 +- .github/workflows/nightly-uds-badge-verification.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/summarizeVerification.sh b/.github/scripts/summarizeVerification.sh index 430d9d0c0..ed8e2c0e7 100755 --- a/.github/scripts/summarizeVerification.sh +++ b/.github/scripts/summarizeVerification.sh @@ -1,7 +1,7 @@ #!/bin/bash # Remove ANSI escape sequences from the data -clean_data=$(sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g') +clean_data=$(sed -E 's/\x1b\[[0-9;]*[a-zA-Z]//g') # Initialize variables package_name="" diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index dd5107a01..2982ad501 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -29,7 +29,7 @@ jobs: uds-badge-verification: runs-on: ai-ubuntu-big-boy-8-core name: nightly_uds_badge_verification - if: ${{ !github.event.pull_request.draft }} + # TODO: make this not run on draft PRs steps: - name: Checkout Repo @@ -59,6 +59,7 @@ jobs: uds zarf tools yq -i '.packages[].repository |= sub("/uds/", "/uds/snapshots/")' bundles/latest/cpu/uds-bundle.yaml + # TODO: add this back in! # - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }}) # run: | # cd bundles/latest/cpu From 049cc94ad1b1a1f685d719c16db3c9b5f0cfbfcb Mon Sep 17 00:00:00 2001 From: Justin Law Date: Fri, 27 Sep 2024 17:46:06 -0400 Subject: [PATCH 29/41] test, pt.4 --- tasks.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tasks.yaml b/tasks.yaml index 465c9c597..33bcb1f37 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -11,11 +11,13 @@ tasks: cmd: | mkdir -p reports - description: "Run UDS Badge Verification Task" + # to clean the intermediate-report.txt, use the `sed` command seen in `clean_data` of summarizeVerification.sh cmd: | - uds run verify-uds-badge-cpu --no-progress 2>&1 | sed -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' 2>&1 | tee ./reports/intermediate-report.txt > /dev/null + uds run verify-uds-badge-cpu --no-progress 2>&1 | tee ./reports/intermediate-report.txt > /dev/null - description: "Clean Up Final Report" cmd: | chmod +x .github/scripts/summarizeVerification.sh + cat ./reports/intermediate-report.txt cat ./reports/intermediate-report.txt 2>&1 | .github/scripts/summarizeVerification.sh | tee ./reports/final-report.txt ############# From 229ba16e0033eef4f2ae3873309ae6746dc1d755 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Fri, 27 Sep 2024 17:48:55 -0400 Subject: [PATCH 30/41] test, pt.5 --- .github/scripts/summarizeVerification.sh | 3 ++- .github/workflows/nightly-uds-badge-verification.yaml | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/scripts/summarizeVerification.sh b/.github/scripts/summarizeVerification.sh index ed8e2c0e7..d1b233abe 100755 --- a/.github/scripts/summarizeVerification.sh +++ b/.github/scripts/summarizeVerification.sh @@ -74,7 +74,8 @@ while IFS= read -r line; do line=$(echo "$line" | sed 's/^[ \t]*//;s/[ \t]*$//') # Match and extract the package name - if [[ $line =~ ^ā„¹ļø[[:space:]]+Package[[:space:]]+Name:[[:space:]]+(.*)$ ]]; then + # ā„¹ļø emoji in UTF-8 hex encoding + if [[ $line =~ ^\xE2\x84\xB9[[:space:]]+Package[[:space:]]+Name:[[:space:]]+(.*)$ ]]; then # Print the previous package's info before starting a new one print_package_info # Reset variables for the new package diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index 2982ad501..f63061e38 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -59,6 +59,16 @@ jobs: uds zarf tools yq -i '.packages[].repository |= sub("/uds/", "/uds/snapshots/")' bundles/latest/cpu/uds-bundle.yaml + # Workaround for summarizeVerification.sh + - name: Set Locale to UTF-8 + run: | + sudo apt-get update + sudo apt-get install -y locales + sudo locale-gen en_US.UTF-8 + export LANG=en_US.UTF-8 + export LANGUAGE=en_US:en + export LC_ALL=en_US.UTF-8 + # TODO: add this back in! # - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }}) # run: | From 0eb3b127c91ab8ec8384b302060f4372a0933b00 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Fri, 27 Sep 2024 17:58:30 -0400 Subject: [PATCH 31/41] clean-up, try again --- .github/scripts/summarizeVerification.sh | 3 +-- .../nightly-uds-badge-verification.yaml | 17 ++++++++--------- tasks.yaml | 1 - 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/scripts/summarizeVerification.sh b/.github/scripts/summarizeVerification.sh index d1b233abe..ed8e2c0e7 100755 --- a/.github/scripts/summarizeVerification.sh +++ b/.github/scripts/summarizeVerification.sh @@ -74,8 +74,7 @@ while IFS= read -r line; do line=$(echo "$line" | sed 's/^[ \t]*//;s/[ \t]*$//') # Match and extract the package name - # ā„¹ļø emoji in UTF-8 hex encoding - if [[ $line =~ ^\xE2\x84\xB9[[:space:]]+Package[[:space:]]+Name:[[:space:]]+(.*)$ ]]; then + if [[ $line =~ ^ā„¹ļø[[:space:]]+Package[[:space:]]+Name:[[:space:]]+(.*)$ ]]; then # Print the previous package's info before starting a new one print_package_info # Reset variables for the new package diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index f63061e38..3f3f3cf2a 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -29,7 +29,7 @@ jobs: uds-badge-verification: runs-on: ai-ubuntu-big-boy-8-core name: nightly_uds_badge_verification - # TODO: make this not run on draft PRs + if: ${{ !github.event.pull_request.draft }} steps: - name: Checkout Repo @@ -69,14 +69,13 @@ jobs: export LANGUAGE=en_US:en export LC_ALL=en_US.UTF-8 - # TODO: add this back in! - # - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }}) - # run: | - # cd bundles/latest/cpu - # uds create . --confirm && \ - # uds deploy uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst --confirm --no-progress && \ - # rm -rf uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst && \ - # docker system prune -af + - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }}) + run: | + cd bundles/latest/cpu + uds create . --confirm && \ + uds deploy uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst --confirm --no-progress && \ + rm -rf uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst && \ + docker system prune -af - name: Run UDS Badge Verification Task run: | diff --git a/tasks.yaml b/tasks.yaml index 33bcb1f37..be9039dd8 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -17,7 +17,6 @@ tasks: - description: "Clean Up Final Report" cmd: | chmod +x .github/scripts/summarizeVerification.sh - cat ./reports/intermediate-report.txt cat ./reports/intermediate-report.txt 2>&1 | .github/scripts/summarizeVerification.sh | tee ./reports/final-report.txt ############# From 6bd0893ee15b5e8548dfa624025dc12a82be890f Mon Sep 17 00:00:00 2001 From: Justin Law <81255462+justinthelaw@users.noreply.github.com> Date: Fri, 27 Sep 2024 22:38:20 -0400 Subject: [PATCH 32/41] testing in draft PR state --- .github/workflows/nightly-uds-badge-verification.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index 3f3f3cf2a..c9145ea79 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -29,7 +29,7 @@ jobs: uds-badge-verification: runs-on: ai-ubuntu-big-boy-8-core name: nightly_uds_badge_verification - if: ${{ !github.event.pull_request.draft }} + # if: ${{ !github.event.pull_request.draft }} steps: - name: Checkout Repo From 7ecefded51f839ed6879b71b033274bb809d2b3d Mon Sep 17 00:00:00 2001 From: Justin Law Date: Mon, 30 Sep 2024 10:11:26 -0400 Subject: [PATCH 33/41] python instead of bash --- .github/scripts/summarizeVerification.sh | 122 --------------- .github/scripts/uds_verification_report.py | 146 ++++++++++++++++++ .../nightly-uds-badge-verification.yaml | 22 ++- tasks.yaml | 4 +- 4 files changed, 161 insertions(+), 133 deletions(-) delete mode 100755 .github/scripts/summarizeVerification.sh create mode 100755 .github/scripts/uds_verification_report.py diff --git a/.github/scripts/summarizeVerification.sh b/.github/scripts/summarizeVerification.sh deleted file mode 100755 index ed8e2c0e7..000000000 --- a/.github/scripts/summarizeVerification.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash - -# Remove ANSI escape sequences from the data -clean_data=$(sed -E 's/\x1b\[[0-9;]*[a-zA-Z]//g') - -# Initialize variables -package_name="" -failures_count=0 -errors_count=0 -warnings_count=0 -failure_descriptions=() -error_descriptions=() -warning_descriptions=() - -# Function to print package information -print_package_info() { - if [[ -n $package_name ]]; then - echo "-----------------------------" - if [[ ${CI} == "true" ]]; then - echo "::group::$package_name" - fi - echo "Package: $package_name" - if ((failures_count > 0)); then - if [[ ${CI} == "true" ]]; then - printf "::error::" - fi - echo "ā›” Failures: $failures_count" - else - if ((errors_count > 0)); then - if [[ ${CI} == "true" ]]; then - printf "::error::" - fi - echo "āŒ Errors: $errors_count" - fi - if ((warnings_count > 0)); then - if [[ ${CI} == "true" ]]; then - printf "::warning::" - fi - echo "āš ļø Warnings: $warnings_count" - fi - fi - - if ((failures_count > 0)); then - echo - echo "ā›” Failure Descriptions:" - for desc in "${failure_descriptions[@]}"; do - echo " - $desc" - done - else - if ((errors_count > 0)); then - echo - echo "āŒ Error Descriptions:" - for desc in "${error_descriptions[@]}"; do - echo " - $desc" - done - fi - if ((warnings_count > 0)); then - echo - echo "āš ļø Warning Descriptions:" - for desc in "${warning_descriptions[@]}"; do - echo " - $desc" - done - fi - fi - fi - if [[ ${CI} == "true" ]]; then - echo "::endgroup::" - fi -} - -# Process each line of the cleaned data -while IFS= read -r line; do - # Remove leading and trailing whitespace - line=$(echo "$line" | sed 's/^[ \t]*//;s/[ \t]*$//') - - # Match and extract the package name - if [[ $line =~ ^ā„¹ļø[[:space:]]+Package[[:space:]]+Name:[[:space:]]+(.*)$ ]]; then - # Print the previous package's info before starting a new one - print_package_info - # Reset variables for the new package - package_name="${BASH_REMATCH[1]}" - failures_count=0 - errors_count=0 - warnings_count=0 - failure_descriptions=() - error_descriptions=() - warning_descriptions=() - # Match and extract counts for failures, errors, and warnings - elif [[ $line =~ ^(āŒ|āš ļø|ā›”)[[:space:]]+([0-9]+)[[:space:]]+([a-z]+)[[:space:]]+found$ ]]; then - count="${BASH_REMATCH[2]}" - type="${BASH_REMATCH[3]}" - case "$type" in - "errors") - errors_count=$count - ;; - "warnings") - warnings_count=$count - ;; - "failures") - failures_count=$count - ;; - esac - # Match and collect issue descriptions - elif [[ $line =~ ^(āŒ|āš ļø|ā›”)[[:space:]]+(.*)$ ]]; then - emoji="${BASH_REMATCH[1]}" - description="${BASH_REMATCH[2]}" - case "$emoji" in - "āŒ") - error_descriptions+=("$description") - ;; - "āš ļø") - warning_descriptions+=("$description") - ;; - "ā›”") - failure_descriptions+=("$description") - ;; - esac - fi -done <<<"$clean_data" - -# Print the last package's information -print_package_info diff --git a/.github/scripts/uds_verification_report.py b/.github/scripts/uds_verification_report.py new file mode 100755 index 000000000..3b758ea95 --- /dev/null +++ b/.github/scripts/uds_verification_report.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + +import os +import re +import sys + +# Explicitly set the encoding, mainly for emoji handling +sys.stdin = open(sys.stdin.fileno(), mode="r", encoding="utf-8", errors="ignore") + + +def remove_ansi_escape_sequences(text): + ansi_escape = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]") + return ansi_escape.sub("", text) + + +# CI environment variable enables GitHub annotations +def print_package_info( + package_name, + failures_count, + errors_count, + warnings_count, + failure_descriptions, + error_descriptions, + warning_descriptions, +): + if package_name: + print("-----------------------------") + if os.getenv("CI") == "true": + print(f"::group::{package_name}") + print(f"Package: {package_name}\n") + if failures_count > 0: + if os.getenv("CI") == "true": + print("::error::", end="") + print(f"ā›” Failures: {failures_count}") + else: + if errors_count > 0: + if os.getenv("CI") == "true": + print("::error::", end="") + print(f"āŒ Errors: {errors_count}") + if warnings_count > 0: + if os.getenv("CI") == "true": + print("::warning::", end="") + print(f"āš ļø Warnings: {warnings_count}") + if failures_count > 0: + print("\nā›” Failure Descriptions:") + for desc in failure_descriptions: + print(f" - {desc}") + else: + if errors_count > 0: + print("\nāŒ Error Descriptions:") + for desc in error_descriptions: + print(f" - {desc}") + if warnings_count > 0: + print("\nāš ļø Warning Descriptions:") + for desc in warning_descriptions: + print(f" - {desc}") + if os.getenv("CI") == "true": + print("::endgroup::") + + +def main(): + # Read data from stdin + data = sys.stdin.read() + # Remove ANSI escape sequences + clean_data = remove_ansi_escape_sequences(data) + # Initialize variables + package_name = "" + failures_count = 0 + errors_count = 0 + warnings_count = 0 + failure_descriptions = [] + error_descriptions = [] + warning_descriptions = [] + previous_package_name = None + + # Process each line + for line in clean_data.splitlines(): + # Remove leading and trailing whitespace + line = line.strip() + + # Match and extract the package name + match = re.match(r"^ā„¹ļø\s+Package\s+Name:\s+(.*)$", line) + if match: + # Print the previous package's info before starting a new one + if previous_package_name is not None: + print_package_info( + previous_package_name, + failures_count, + errors_count, + warnings_count, + failure_descriptions, + error_descriptions, + warning_descriptions, + ) + # Reset variables for the new package + package_name = match.group(1) + failures_count = 0 + errors_count = 0 + warnings_count = 0 + failure_descriptions = [] + error_descriptions = [] + warning_descriptions = [] + previous_package_name = package_name + continue + + # Match and extract counts for failures, errors, and warnings + match = re.match(r"^(āŒ|āš ļø|ā›”)\s+(\d+)\s+([a-z]+)\s+found$", line) + if match: + count = int(match.group(2)) + type_ = match.group(3) + if type_ == "errors": + errors_count = count + elif type_ == "warnings": + warnings_count = count + elif type_ == "failures": + failures_count = count + continue + + # Match and collect issue descriptions + match = re.match(r"^(āŒ|āš ļø|ā›”)\s+(.*)$", line) + if match: + emoji = match.group(1) + description = match.group(2) + if emoji == "āŒ": + error_descriptions.append(description) + elif emoji == "āš ļø": + warning_descriptions.append(description) + elif emoji == "ā›”": + failure_descriptions.append(description) + continue + + # Print the last package's information + if previous_package_name is not None: + print_package_info( + previous_package_name, + failures_count, + errors_count, + warnings_count, + failure_descriptions, + error_descriptions, + warning_descriptions, + ) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index c9145ea79..118f751ce 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -59,7 +59,15 @@ jobs: uds zarf tools yq -i '.packages[].repository |= sub("/uds/", "/uds/snapshots/")' bundles/latest/cpu/uds-bundle.yaml - # Workaround for summarizeVerification.sh + - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }}) + run: | + cd bundles/latest/cpu + uds create . --confirm && \ + uds deploy uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst --confirm --no-progress && \ + rm -rf uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst && \ + docker system prune -af + + # Workaround for handling emojis in the upstream badge verification UDS task - name: Set Locale to UTF-8 run: | sudo apt-get update @@ -69,13 +77,11 @@ jobs: export LANGUAGE=en_US:en export LC_ALL=en_US.UTF-8 - - name: Create and Deploy UDS Bundle (${{ env.SNAPSHOT_VERSION }}) - run: | - cd bundles/latest/cpu - uds create . --confirm && \ - uds deploy uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst --confirm --no-progress && \ - rm -rf uds-bundle-leapfrogai-amd64-${{ env.SNAPSHOT_VERSION }}.tar.zst && \ - docker system prune -af + # Setup Python for the report cleaning script in the next step + - name: Set up Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version-file: "pyproject.toml" - name: Run UDS Badge Verification Task run: | diff --git a/tasks.yaml b/tasks.yaml index be9039dd8..f972f7aa0 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -11,13 +11,11 @@ tasks: cmd: | mkdir -p reports - description: "Run UDS Badge Verification Task" - # to clean the intermediate-report.txt, use the `sed` command seen in `clean_data` of summarizeVerification.sh cmd: | uds run verify-uds-badge-cpu --no-progress 2>&1 | tee ./reports/intermediate-report.txt > /dev/null - description: "Clean Up Final Report" cmd: | - chmod +x .github/scripts/summarizeVerification.sh - cat ./reports/intermediate-report.txt 2>&1 | .github/scripts/summarizeVerification.sh | tee ./reports/final-report.txt + cat ./reports/intermediate-report.txt 2>&1 | python3 .github/scripts/uds_verification_report.py | tee ./reports/final-report.txt ############# # BADGE TASKS From 10cb7b7ff039dcfcc494f0910b62dfc671726a76 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Mon, 30 Sep 2024 10:14:06 -0400 Subject: [PATCH 34/41] remove comment --- .github/workflows/nightly-uds-badge-verification.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index 118f751ce..d2594b42d 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -29,7 +29,6 @@ jobs: uds-badge-verification: runs-on: ai-ubuntu-big-boy-8-core name: nightly_uds_badge_verification - # if: ${{ !github.event.pull_request.draft }} steps: - name: Checkout Repo From 2119cf69ef7fd9032c73c9a218a583f6a271797b Mon Sep 17 00:00:00 2001 From: Justin Law Date: Mon, 30 Sep 2024 10:59:09 -0400 Subject: [PATCH 35/41] read via open instead of stdin --- .github/scripts/uds_verification_report.py | 91 +++++++++++++++------- 1 file changed, 62 insertions(+), 29 deletions(-) diff --git a/.github/scripts/uds_verification_report.py b/.github/scripts/uds_verification_report.py index 3b758ea95..7ff6e2653 100755 --- a/.github/scripts/uds_verification_report.py +++ b/.github/scripts/uds_verification_report.py @@ -2,10 +2,6 @@ import os import re -import sys - -# Explicitly set the encoding, mainly for emoji handling -sys.stdin = open(sys.stdin.fileno(), mode="r", encoding="utf-8", errors="ignore") def remove_ansi_escape_sequences(text): @@ -13,6 +9,13 @@ def remove_ansi_escape_sequences(text): return ansi_escape.sub("", text) +# Capabilities that affect the entire capability, not just a single package +def uds_capability_wide_errors(text: str) -> bool: + if "Not all pods have the istio sidecar" in text: + return True + return False + + # CI environment variable enables GitHub annotations def print_package_info( package_name, @@ -22,7 +25,10 @@ def print_package_info( failure_descriptions, error_descriptions, warning_descriptions, + uds_capability_wide_errors_count, ): + if uds_capability_wide_errors_count >= 1: + errors_count -= uds_capability_wide_errors_count if package_name: print("-----------------------------") if os.getenv("CI") == "true": @@ -59,8 +65,11 @@ def print_package_info( def main(): - # Read data from stdin - data = sys.stdin.read() + # Read data from the specified file instead of stdin + with open( + "reports/intermediate-report.txt", mode="r", encoding="utf-8", errors="ignore" + ) as file: + data = file.read() # Remove ANSI escape sequences clean_data = remove_ansi_escape_sequences(data) # Initialize variables @@ -68,9 +77,11 @@ def main(): failures_count = 0 errors_count = 0 warnings_count = 0 + uds_capability_wide_errors_count = 0 failure_descriptions = [] error_descriptions = [] warning_descriptions = [] + uds_capability_wide_error_descriptions = [] previous_package_name = None # Process each line @@ -91,6 +102,7 @@ def main(): failure_descriptions, error_descriptions, warning_descriptions, + uds_capability_wide_errors_count, ) # Reset variables for the new package package_name = match.group(1) @@ -103,31 +115,38 @@ def main(): previous_package_name = package_name continue - # Match and extract counts for failures, errors, and warnings - match = re.match(r"^(āŒ|āš ļø|ā›”)\s+(\d+)\s+([a-z]+)\s+found$", line) - if match: - count = int(match.group(2)) - type_ = match.group(3) - if type_ == "errors": - errors_count = count - elif type_ == "warnings": - warnings_count = count - elif type_ == "failures": - failures_count = count + if uds_capability_wide_errors(line): + uds_capability_wide_errors_count = 1 + uds_capability_wide_error_descriptions = [ + "Not all pods have the istio sidecar" + ] continue + else: + # Match and extract counts for failures, errors, and warnings + match = re.match(r"^(āŒ|āš ļø|ā›”)\s+(\d+)\s+([a-z]+)\s+found$", line) + if match: + count = int(match.group(2)) + type_ = match.group(3) + if type_ == "errors": + errors_count = count + elif type_ == "warnings": + warnings_count = count + elif type_ == "failures": + failures_count = count + continue - # Match and collect issue descriptions - match = re.match(r"^(āŒ|āš ļø|ā›”)\s+(.*)$", line) - if match: - emoji = match.group(1) - description = match.group(2) - if emoji == "āŒ": - error_descriptions.append(description) - elif emoji == "āš ļø": - warning_descriptions.append(description) - elif emoji == "ā›”": - failure_descriptions.append(description) - continue + # Match and collect issue descriptions + match = re.match(r"^(āŒ|āš ļø|ā›”)\s+(.*)$", line) + if match: + emoji = match.group(1) + description = match.group(2) + if emoji == "āŒ": + error_descriptions.append(description) + elif emoji == "āš ļø": + warning_descriptions.append(description) + elif emoji == "ā›”": + failure_descriptions.append(description) + continue # Print the last package's information if previous_package_name is not None: @@ -139,8 +158,22 @@ def main(): failure_descriptions, error_descriptions, warning_descriptions, + uds_capability_wide_errors_count, ) + if uds_capability_wide_errors_count >= 1: + print("-----------------------------") + if os.getenv("CI") == "true": + print("::group::UDS Capability-Wide Issues") + print("::error::", end="") + print("UDS Capability Issues") + print("\nāŒ Error Descriptions:") + for desc in uds_capability_wide_error_descriptions: + print(f" - {desc}") + if os.getenv("CI") == "true": + print("::endgroup::") if __name__ == "__main__": main() + # Print the final ending separator + print("-----------------------------") From b12c7893c10e094bfc01514eda12677b684df507 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Mon, 30 Sep 2024 11:05:05 -0400 Subject: [PATCH 36/41] add docs --- docs/DEVELOPMENT.md | 46 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 897bfaf5d..7aa8d5c02 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -62,6 +62,52 @@ Many of the directories and sub-directories within this project contain Make tar Please refer to each Makefile for more arguments and details on what each target does and is dependent on. +## UDS Tasks + +UDS tasks use the UDS CLI runner, and are defined in the root `tasks.yaml` file. + +Currently, the only tasks within the file are for checking the progress of the LeapfrogAI towards the `Made for UDS` packaging standards. To run the task verification task you must have a [UDS Kubernetes cluster](../packages/k3d-gpu/README.md) and LeapfrogAI (GPU or CPU) deployed. After deploying both major capabilities, you can execute the following: + +```bash +uds run nightly-uds-badge-verification --no-progress +``` + +You should get an output similar to this, depending on how many components of LeapfrogAI are actually deployed: + +```bash + ā€¢ Running "Create Reports Directory" + + āœ” Completed "Create Reports Directory" + + ā€¢ Running "Run UDS Badge Verification Task" + + āœ” Completed "Run UDS Badge Verification Task" + + ā€¢ Running "Clean Up Final Report" +----------------------------- +Package: leapfrogai-api + +āŒ Errors: 4 +āš ļø Warnings: 3 + +āŒ Error Descriptions: + - Endpoint leapfrogai-api.uds.dev is returning 404 + - Not all applicable network policies are using selectors + - Not all applicable network policies are using ports + - No monitors defined + +āš ļø Warning Descriptions: + - Version is not consistent across flavors and package + - Network policies with 'remoteGenerated: Anywhere' are present, review needed + - No SSO configuration found, review needed +----------------------------- +UDS Capability Issues + +āŒ Error Descriptions: + - Not all pods have the istio sidecar +----------------------------- +``` + ## Environment Variables Be wary of `*config*.yaml` or `.env*` files that are in individual components of the stack. The component's README will usually tell the developer when to fill them out or supply environment variables to a script. From 4061a86b8675c312a19dde41370c477da900e9ad Mon Sep 17 00:00:00 2001 From: Justin Law Date: Mon, 30 Sep 2024 11:24:55 -0400 Subject: [PATCH 37/41] remove extraneous cat cmd --- tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.yaml b/tasks.yaml index f972f7aa0..b7d36d966 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -15,7 +15,7 @@ tasks: uds run verify-uds-badge-cpu --no-progress 2>&1 | tee ./reports/intermediate-report.txt > /dev/null - description: "Clean Up Final Report" cmd: | - cat ./reports/intermediate-report.txt 2>&1 | python3 .github/scripts/uds_verification_report.py | tee ./reports/final-report.txt + python3 .github/scripts/uds_verification_report.py | tee ./reports/final-report.txt ############# # BADGE TASKS From 2e7afdd2ab003d184faff197c1ece40a5fbcf8c8 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Mon, 30 Sep 2024 14:27:30 -0400 Subject: [PATCH 38/41] merge with main, fix read --- .github/scripts/uds_verification_report.py | 8 +++++--- tasks.yaml | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/scripts/uds_verification_report.py b/.github/scripts/uds_verification_report.py index 7ff6e2653..ad4d2ebcc 100755 --- a/.github/scripts/uds_verification_report.py +++ b/.github/scripts/uds_verification_report.py @@ -66,9 +66,11 @@ def print_package_info( def main(): # Read data from the specified file instead of stdin - with open( - "reports/intermediate-report.txt", mode="r", encoding="utf-8", errors="ignore" - ) as file: + # Use the GitHub runner ENV to find the file, if in CI + file_path = os.path.join( + os.getenv("GITHUB_WORKSPACE", ""), "reports/intermediate-report.txt" + ) + with open(file_path, mode="r", encoding="utf-8", errors="ignore") as file: data = file.read() # Remove ANSI escape sequences clean_data = remove_ansi_escape_sequences(data) diff --git a/tasks.yaml b/tasks.yaml index b7d36d966..a3426874b 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -13,6 +13,7 @@ tasks: - description: "Run UDS Badge Verification Task" cmd: | uds run verify-uds-badge-cpu --no-progress 2>&1 | tee ./reports/intermediate-report.txt > /dev/null + chmod 644 reports/intermediate-report.txt - description: "Clean Up Final Report" cmd: | python3 .github/scripts/uds_verification_report.py | tee ./reports/final-report.txt From f6cc6744824f184488fec0551881721312f714bd Mon Sep 17 00:00:00 2001 From: Justin Law Date: Mon, 30 Sep 2024 15:46:05 -0400 Subject: [PATCH 39/41] merge with main, print to stdout --- .github/scripts/uds_verification_report.py | 1 - tasks.yaml | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/uds_verification_report.py b/.github/scripts/uds_verification_report.py index ad4d2ebcc..0e4d4e8fe 100755 --- a/.github/scripts/uds_verification_report.py +++ b/.github/scripts/uds_verification_report.py @@ -66,7 +66,6 @@ def print_package_info( def main(): # Read data from the specified file instead of stdin - # Use the GitHub runner ENV to find the file, if in CI file_path = os.path.join( os.getenv("GITHUB_WORKSPACE", ""), "reports/intermediate-report.txt" ) diff --git a/tasks.yaml b/tasks.yaml index a3426874b..2298757ba 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -12,8 +12,7 @@ tasks: mkdir -p reports - description: "Run UDS Badge Verification Task" cmd: | - uds run verify-uds-badge-cpu --no-progress 2>&1 | tee ./reports/intermediate-report.txt > /dev/null - chmod 644 reports/intermediate-report.txt + uds run verify-uds-badge-cpu --no-progress 2>&1 | tee ./reports/intermediate-report.txt - description: "Clean Up Final Report" cmd: | python3 .github/scripts/uds_verification_report.py | tee ./reports/final-report.txt From 6bc5cd37a970c3c0dedccec7a2fe771341a56098 Mon Sep 17 00:00:00 2001 From: Justin Law Date: Tue, 1 Oct 2024 14:05:36 -0400 Subject: [PATCH 40/41] fix pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c9883c5e9..401bcba03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -157,6 +157,6 @@ repos: hooks: - id: check-jsonschema name: "Validate UDS Bundles Against Schema" - files: "uds-bundle.yaml" + files: "tasks.yaml" types: [yaml] args: ["--schemafile", "tasks-v0.14.0.schema.json"] From 201681c107ff3e4988b5dbfc5f8dec20d6a5a8e0 Mon Sep 17 00:00:00 2001 From: Justin Law <81255462+justinthelaw@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:54:26 -0400 Subject: [PATCH 41/41] typo in schedule cron, 3am not 12am --- .github/workflows/nightly-uds-badge-verification.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-uds-badge-verification.yaml b/.github/workflows/nightly-uds-badge-verification.yaml index d2594b42d..6be419ebb 100644 --- a/.github/workflows/nightly-uds-badge-verification.yaml +++ b/.github/workflows/nightly-uds-badge-verification.yaml @@ -2,7 +2,7 @@ name: nightly-uds-badge-verification on: schedule: - - cron: "0 11 * * *" # Runs daily at 12 AM PST + - cron: "0 11 * * *" # Runs daily at 3 AM PST workflow_dispatch: # trigger manually as needed pull_request: paths: