diff --git a/.github/workflows/bash_formatter.yaml b/.github/workflows/bash_formatter.yaml new file mode 100644 index 0000000000..fd1d254bf6 --- /dev/null +++ b/.github/workflows/bash_formatter.yaml @@ -0,0 +1,40 @@ +name: Proper Formatting on bash files + +on: [push, pull_request] + +jobs: + format_bash_files: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install ShellCheck + run: sudo apt install -y shellcheck + + - name: Bash Formatting Guidelines + run: | + echo "### Bash Files Formatting Guidelines ### + If there are errors and warnings regarding your bash files, + You can check that error code definitions in https://www.shellcheck.net/wiki/ site. + You can correct them using the https://www.shellcheck.net/ site. + You have to ignore disable errors in .shellcheckrc file. + " + + - name: Fetch master branch + run: git fetch origin master + + - name: Set up changed files + run: | + git diff --name-only origin/master...HEAD | grep -E '^.*\.sh$' | grep -v '^apps/' > changed_files_in_PR.txt || true + + - name: Display changed files + run: cat changed_files_in_PR.txt + + - name: Run ShellCheck on changed files + run: | + cat changed_files_in_PR.txt | xargs -I {} shellcheck {} + shell: bash + + diff --git a/.github/workflows/python_formatter.yaml b/.github/workflows/python_formatter.yaml new file mode 100644 index 0000000000..ba470cad84 --- /dev/null +++ b/.github/workflows/python_formatter.yaml @@ -0,0 +1,32 @@ +name: Proper Formatting on Python files + +on: [push, pull_request] + +jobs: + format_python_files: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Python Files Formatting Guidelines + run: | + echo "### Python Files Formatting Guidelines ### + If there is a formatting errors in your python files, + 1. First install black + It requires Python 3.8+ to run. + Install with "pip install black" and if you use pipx, install Black with "pipx install black" + If you want to format Jupyter Notebooks, install with pip install "black[jupyter]" + + 2. Run the command + "python -m black {source_file_or_directory}" or + "black {source_file_or_directory}" + to format python files. + " + + - uses: psf/black@stable + with: + src: | + ./common + ./example + + \ No newline at end of file diff --git a/.github/workflows/yaml_formatter.yaml b/.github/workflows/yaml_formatter.yaml new file mode 100644 index 0000000000..dca73409d4 --- /dev/null +++ b/.github/workflows/yaml_formatter.yaml @@ -0,0 +1,59 @@ +name: Proper Formatting on YAML files + +on: [push, pull_request] + +jobs: + format_YAML_files: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install yamllint + run: pip install yamllint + + - name: YAML Formatting Guidelines + run: | + echo "### YAML Formatting Guidelines ### + If there is a formatting error in your YAML file, you will see errors like the one below: + 'Error: 6:4 [indentation] wrong indentation: expected 2 but found 3' + + To fix these errors, refer to the YAML formatting rules at: + https://yamllint.readthedocs.io/en/stable/rules.html# + + Search for the keyword inside the brackets [] in the error message. In this example, it's 'indentation'. + + Note: Some rules have been customized in the '.yamllint.yaml' file. Below is the content of that file: + + extends: default + + rules: + document-start: + present: false + document-end: + present: false + indentation: + indent-sequences: false + line-length: + max: 400 + " + + - name: Fetch master branch + run: git fetch origin master + + - name: Set up changed files + run: | + git diff --name-only origin/master...HEAD | grep -E '^common/.*\.ya?ml$|^example/.*\.ya?ml$' > changed_files_in_PR.txt || true + + - name: Display changed files + run: cat changed_files_in_PR.txt + + - name: Run yamllint on changed files + run: | + chmod +x ./run_yamllint.sh + ./run_yamllint.sh + shell: bash + + + diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000000..9c339b78ba --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,2 @@ +# ~/.shellcheckrc +disable=SC1017,SC2086,SC2070,SC2046,SC2155,SC2006 diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000000..46153d48f6 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,12 @@ +# It extends the default conf by adjusting some options. +extends: default + +rules: + document-start: + present: false + document-end: + present: false + indentation: + indent-sequences: false + line-length: + max: 400 \ No newline at end of file diff --git a/contrib/kserve/tests/test_sklearn.py b/contrib/kserve/tests/test_sklearn.py index 2c17257019..0e2d21241c 100644 --- a/contrib/kserve/tests/test_sklearn.py +++ b/contrib/kserve/tests/test_sklearn.py @@ -50,7 +50,9 @@ def test_sklearn_kserve(): spec=V1beta1InferenceServiceSpec(predictor=predictor), ) - kserve_client = KServeClient(config_file=os.environ.get("KUBECONFIG", "~/.kube/config")) + kserve_client = KServeClient( + config_file=os.environ.get("KUBECONFIG", "~/.kube/config") + ) kserve_client.create(isvc) kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE) res = predict(service_name, "./data/iris_input.json") diff --git a/hack/synchronize-istio-cni-manifests.sh b/hack/synchronize-istio-cni-manifests.sh index b2fcd7c3af..1f8a22c3e5 100644 --- a/hack/synchronize-istio-cni-manifests.sh +++ b/hack/synchronize-istio-cni-manifests.sh @@ -1,5 +1,5 @@ -@ -1,88 +0,0 @@ #!/usr/bin/env bash +@ -1,88 +0,0 @@ # This script aims at helping create a PR to update the manifests of the # knative. diff --git a/hack/synchronize-istio-manifests.sh b/hack/synchronize-istio-manifests.sh index 6a4c8987de..ad75d68022 100644 --- a/hack/synchronize-istio-manifests.sh +++ b/hack/synchronize-istio-manifests.sh @@ -1,4 +1,5 @@ -# #!/usr/bin/env bash +#!/usr/bin/env bash + # # This script aims at helping create a PR to update the manifests of Istio # # This script: diff --git a/hack/synchronize-kserve-web-app-manifests.sh b/hack/synchronize-kserve-web-app-manifests.sh index f7b5b102b2..beabded784 100644 --- a/hack/synchronize-kserve-web-app-manifests.sh +++ b/hack/synchronize-kserve-web-app-manifests.sh @@ -24,7 +24,7 @@ if [ -n "$(git status --porcelain)" ]; then echo "WARNING: You have uncommitted changes" fi -if [ `git branch --list $BRANCH` ] +if [ "$(git branch --list $BRANCH)" ] then echo "WARNING: Branch $BRANCH already exists." fi @@ -39,11 +39,11 @@ echo "Checking out in $SRC_DIR to $COMMIT..." # Checkout the Model Registry repository mkdir -p $SRC_DIR -cd $SRC_DIR +cd $SRC_DIR || exit if [ ! -d "models-web-app/.git" ]; then git clone https://github.com/kserve/models-web-app.git fi -cd $SRC_DIR/models-web-app +cd $SRC_DIR/models-web-app || exit if ! git rev-parse --verify --quiet $COMMIT; then git checkout -b $COMMIT else @@ -71,7 +71,7 @@ DST_TXT="\[$COMMIT\](https://github.com/kserve/models-web-app/tree/$COMMIT/confi sed -i "s|$SRC_TXT|$DST_TXT|g" "${MANIFESTS_DIR}"/README.md echo "Committing the changes..." -cd $MANIFESTS_DIR +cd $MANIFESTS_DIR || exit git add contrib/kserve/models-web-app git add README.md git commit -s -m "Update kserve models web application manifests from ${COMMIT}" diff --git a/hack/trivy_scan.py b/hack/trivy_scan.py index d76536604f..ff7706bb1b 100644 --- a/hack/trivy_scan.py +++ b/hack/trivy_scan.py @@ -7,8 +7,8 @@ # - Summary of security counts with images a JSON file inside ../image_lists/summary_of_severity_counts_for_WG folder # 4. Generate a summary of the security scan reports # - The summary will be saved in JSON format inside ../image_lists/summary_of_severity_counts_for_WG folder -# 5. Before run this file you have to -# 1. Install kustomize +# 5. Before run this file you have to +# 1. Install kustomize # - sudo apt install snapd # - sudo snap install kustomize # 2. Install trivy @@ -37,31 +37,36 @@ "manifests": "../common/cert-manager/cert-manager/base ../common/cert-manager/kubeflow-issuer/base ../common/istio-1-22/istio-crds/base ../common/istio-1-22/istio-namespace/base ../common/istio-1-22/istio-install/overlays/oauth2-proxy ../common/oidc-client/oauth2-proxy/overlays/m2m-self-signed ../common/dex/overlays/oauth2-proxy ../common/knative/knative-serving/overlays/gateways ../common/knative/knative-eventing/base ../common/istio-1-22/cluster-local-gateway/base ../common/kubeflow-namespace/base ../common/kubeflow-roles/base ../common/istio-1-22/kubeflow-istio-resources/base", "workbenches": "../apps/pvcviewer-controller/upstream/base ../apps/admission-webhook/upstream/overlays ../apps/centraldashboard/upstream/overlays/oauth2-proxy ../apps/jupyter/jupyter-web-app/upstream/overlays ../apps/volumes-web-app/upstream/overlays ../apps/tensorboard/tensorboards-web-app/upstream/overlays ../apps/profiles/upstream/overlays ../apps/jupyter/notebook-controller/upstream/overlays ../apps/tensorboard/tensorboard-controller/upstream/overlays", "serving": "../contrib/kserve - ../contrib/kserve/models-web-app/overlays/kubeflow", - "model-registry": "../apps/model-registry/upstream" + "model-registry": "../apps/model-registry/upstream", } DIRECTORY = "../image_lists" os.makedirs(DIRECTORY, exist_ok=True) SCAN_REPORTS_DIR = os.path.join(DIRECTORY, "security_scan_reports") ALL_SEVERITY_COUNTS = os.path.join(DIRECTORY, "severity_counts_with_images_for_WG") -SUMMARY_OF_SEVERITY_COUNTS = os.path.join(DIRECTORY, "summary_of_severity_counts_for_WG") +SUMMARY_OF_SEVERITY_COUNTS = os.path.join( + DIRECTORY, "summary_of_severity_counts_for_WG" +) os.makedirs(SCAN_REPORTS_DIR, exist_ok=True) os.makedirs(ALL_SEVERITY_COUNTS, exist_ok=True) os.makedirs(SUMMARY_OF_SEVERITY_COUNTS, exist_ok=True) + def log(*args, **kwargs): # Custom log function that print messages with flush=True by default. - kwargs.setdefault('flush', True) + kwargs.setdefault("flush", True) print(*args, **kwargs) + def save_images(wg, images, version): # Saves a list of container images to a text file named after the workgroup and version. output_file = f"../image_lists/kf_{version}_{wg}_images.txt" - with open(output_file, 'w') as f: - f.write('\n'.join(images)) + with open(output_file, "w") as f: + f.write("\n".join(images)) log(f"File {output_file} successfully created") + def validate_semantic_version(version): # Validates a semantic version string (e.g., "0.1.2" or "latest"). regex = r"^[0-9]+\.[0-9]+\.[0-9]+$" @@ -70,29 +75,46 @@ def validate_semantic_version(version): else: raise ValueError(f"Invalid semantic version: '{version}'") + def extract_images(version): version = validate_semantic_version(version) log(f"Running the script using Kubeflow version: {version}") - all_images = set() # Collect all unique images across workgroups + all_images = set() # Collect all unique images across workgroups for wg, dirs in wg_dirs.items(): wg_images = set() # Collect unique images for this workgroup for dir_path in dirs.split(): for root, _, files in os.walk(dir_path): for file in files: - if file in ["kustomization.yaml", "kustomization.yml", "Kustomization"]: + if file in [ + "kustomization.yaml", + "kustomization.yml", + "Kustomization", + ]: full_path = os.path.join(root, file) try: # Execute `kustomize build` to render the kustomization file - result = subprocess.run(['kustomize', 'build', root], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run( + ["kustomize", "build", root], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) except subprocess.CalledProcessError as e: - log(f"ERROR:\t Failed \"kustomize build\" command for directory: {root}. See error above") + log( + f'ERROR:\t Failed "kustomize build" command for directory: {root}. See error above' + ) continue - + # Use regex to find lines with 'image: :' or 'image: ' # and '- image: :' but avoid environment variables - kustomize_images = re.findall(r'^\s*-?\s*image:\s*([^$\s:]+(?:\:[^\s]+)?)$', result.stdout, re.MULTILINE) + kustomize_images = re.findall( + r"^\s*-?\s*image:\s*([^$\s:]+(?:\:[^\s]+)?)$", + result.stdout, + re.MULTILINE, + ) wg_images.update(kustomize_images) # Ensure uniqueness within workgroup images @@ -104,24 +126,36 @@ def extract_images(version): uniq_images = sorted(all_images) save_images("all", uniq_images, version) -parser = argparse.ArgumentParser(description="Extract images from Kubeflow kustomizations.") + +parser = argparse.ArgumentParser( + description="Extract images from Kubeflow kustomizations." +) # Define a positional argument 'version' with optional occurrence and default value 'latest'. You can run this file as python3 .py or python .py -parser.add_argument("version", nargs="?", type=str, default="latest", help="Kubeflow version to use (defaults to latest).") +parser.add_argument( + "version", + nargs="?", + type=str, + default="latest", + help="Kubeflow version to use (defaults to latest).", +) args = parser.parse_args() extract_images(args.version) - log("Started scanning images") # Get list of text files excluding "kf_latest_all_images.txt" -files = [f for f in glob.glob(os.path.join(DIRECTORY, "*.txt")) if not f.endswith("kf_latest_all_images.txt")] +files = [ + f + for f in glob.glob(os.path.join(DIRECTORY, "*.txt")) + if not f.endswith("kf_latest_all_images.txt") +] # Loop through each text file in the specified directory for file in files: log(f"Scanning images in {file}") - file_base_name = os.path.basename(file).replace('.txt', '') + file_base_name = os.path.basename(file).replace(".txt", "") # Directory to save reports for this specific file file_reports_dir = os.path.join(SCAN_REPORTS_DIR, file_base_name) @@ -131,68 +165,86 @@ def extract_images(version): severity_count = os.path.join(file_reports_dir, "severity_counts") os.makedirs(severity_count, exist_ok=True) - with open(file, 'r') as f: + with open(file, "r") as f: lines = f.readlines() for line in lines: line = line.strip() - image_name = line.split(':')[0] - image_tag = line.split(':')[1] if ':' in line else '' + image_name = line.split(":")[0] + image_tag = line.split(":")[1] if ":" in line else "" - image_name_scan = image_name.split('/')[-1] + image_name_scan = image_name.split("/")[-1] if image_tag: image_name_scan = f"{image_name_scan}_{image_tag}" - - scan_output_file = os.path.join(file_reports_dir, f"{image_name_scan}_scan.json") - log(f"Scanning ",line) + scan_output_file = os.path.join( + file_reports_dir, f"{image_name_scan}_scan.json" + ) - try: - result = subprocess.run(["trivy", "image", "--format", "json", "--output", scan_output_file, line], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + log(f"Scanning ", line) - with open(scan_output_file, 'r') as json_file: + try: + result = subprocess.run( + [ + "trivy", + "image", + "--format", + "json", + "--output", + scan_output_file, + line, + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + with open(scan_output_file, "r") as json_file: scan_data = json.load(json_file) - if not scan_data.get('Results'): + if not scan_data.get("Results"): log(f"No vulnerabilities found in {image_name}:{image_tag}") else: vulnerabilities_list = [ - result['Vulnerabilities'] - for result in scan_data['Results'] - if 'Vulnerabilities' in result and result['Vulnerabilities'] + result["Vulnerabilities"] + for result in scan_data["Results"] + if "Vulnerabilities" in result and result["Vulnerabilities"] ] if not vulnerabilities_list: - log(f"The vulnerabilities detection may be insufficient because security updates are not provided for {image_name}:{image_tag}\n") + log( + f"The vulnerabilities detection may be insufficient because security updates are not provided for {image_name}:{image_tag}\n" + ) else: severity_counts = {"LOW": 0, "MEDIUM": 0, "HIGH": 0, "CRITICAL": 0} for vulnerabilities in vulnerabilities_list: for vulnerability in vulnerabilities: - severity = vulnerability.get('Severity', 'UNKNOWN') - if severity == 'UNKNOWN': + severity = vulnerability.get("Severity", "UNKNOWN") + if severity == "UNKNOWN": continue elif severity in severity_counts: severity_counts[severity] += 1 - - report = { - "image": line, - "severity_counts": severity_counts - } + report = {"image": line, "severity_counts": severity_counts} image_table = PrettyTable() image_table.field_names = ["Critical", "High", "Medium", "Low"] - image_table.add_row([ + image_table.add_row( + [ severity_counts["CRITICAL"], severity_counts["HIGH"], severity_counts["MEDIUM"], - severity_counts["LOW"] - ]) + severity_counts["LOW"], + ] + ) log(f"{image_table}\n") - severity_report_file = os.path.join(severity_count, f"{image_name_scan}_severity_report.json") - with open(severity_report_file, 'w') as report_file: + severity_report_file = os.path.join( + severity_count, f"{image_name_scan}_severity_report.json" + ) + with open(severity_report_file, "w") as report_file: json.dump(report, report_file, indent=4) except subprocess.CalledProcessError as e: @@ -209,16 +261,18 @@ def extract_images(version): else: combined_data = [] for json_file in json_files: - with open(json_file, 'r') as jf: + with open(json_file, "r") as jf: combined_data.append(json.load(jf)) - with open(output_file, 'w') as of: + with open(output_file, "w") as of: json.dump({"data": combined_data}, of, indent=4) log(f"JSON files successfully combined into '{output_file}'") # File to save summary of the severity counts for WGs as JSON format. -summary_file = os.path.join(SUMMARY_OF_SEVERITY_COUNTS, "severity_summary_in_json_format.json") +summary_file = os.path.join( + SUMMARY_OF_SEVERITY_COUNTS, "severity_summary_in_json_format.json" +) # Initialize counters total_images = 0 @@ -233,27 +287,27 @@ def extract_images(version): # Loop through each JSON file in the ALL_SEVERITY_COUNTS for file_path in glob.glob(os.path.join(ALL_SEVERITY_COUNTS, "*.json")): # Split filename based on underscores - filename_parts = os.path.basename(file_path).split('_') + filename_parts = os.path.basename(file_path).split("_") # Check if there are at least 3 parts (prefix, name, _images) if len(filename_parts) >= 4: - # Extract name (second part) - filename = filename_parts[2] - filename = filename.capitalize() + # Extract name (second part) + filename = filename_parts[2] + filename = filename.capitalize() else: - log(f"Skipping invalid filename format: {file_path}") - continue + log(f"Skipping invalid filename format: {file_path}") + continue - with open(file_path, 'r') as f: - data = json.load(f)['data'] + with open(file_path, "r") as f: + data = json.load(f)["data"] # Initialize counts for this file image_count = len(data) - low = sum(entry['severity_counts']['LOW'] for entry in data) - medium = sum(entry['severity_counts']['MEDIUM'] for entry in data) - high = sum(entry['severity_counts']['HIGH'] for entry in data) - critical = sum(entry['severity_counts']['CRITICAL'] for entry in data) + low = sum(entry["severity_counts"]["LOW"] for entry in data) + medium = sum(entry["severity_counts"]["MEDIUM"] for entry in data) + high = sum(entry["severity_counts"]["HIGH"] for entry in data) + critical = sum(entry["severity_counts"]["CRITICAL"] for entry in data) # Update the total counts total_images += image_count @@ -268,67 +322,85 @@ def extract_images(version): "LOW": low, "MEDIUM": medium, "HIGH": high, - "CRITICAL": critical + "CRITICAL": critical, } # Update merged_data with filename as key merged_data[filename] = file_data # Add total counts to merged_data - merged_data['total'] = { + merged_data["total"] = { "images": total_images, "LOW": total_low, "MEDIUM": total_medium, "HIGH": total_high, - "CRITICAL": total_critical + "CRITICAL": total_critical, } log("Summary in Json Format:") -log(json.dumps(merged_data, indent=4)) +log(json.dumps(merged_data, indent=4)) # Write the final output to a file -with open(summary_file, 'w') as summary_f: +with open(summary_file, "w") as summary_f: json.dump(merged_data, summary_f, indent=4) log(f"Summary written to: {summary_file} as JSON format") # Load JSON content from the file -with open(summary_file, 'r') as file: +with open(summary_file, "r") as file: data = json.load(file) # Define a mapping for working group names groupnames = { "Automl": "AutoML", "Pipelines": "Pipelines", - "Workbenches":"Workbenches(Notebooks)", + "Workbenches": "Workbenches(Notebooks)", "Serving": "Kserve", - "Manifests":"Manifests", + "Manifests": "Manifests", "Training": "Training", - "Model-registry":"Model Registry", + "Model-registry": "Model Registry", "total": "All Images", } # Create PrettyTable table = PrettyTable() -table.field_names = ["Working Group", "Images", "Critical CVE", "High CVE", "Medium CVE", "Low CVE"] +table.field_names = [ + "Working Group", + "Images", + "Critical CVE", + "High CVE", + "Medium CVE", + "Low CVE", +] # Populate the table with data for group_name in groupnames: if group_name in data: # Check if group_name exists in data value = data[group_name] - table.add_row([groupnames[group_name], value["images"], value["CRITICAL"], value["HIGH"], value["MEDIUM"], value["LOW"]]) + table.add_row( + [ + groupnames[group_name], + value["images"], + value["CRITICAL"], + value["HIGH"], + value["MEDIUM"], + value["LOW"], + ] + ) # log the table log(table) # Write the table output to a file in the specified folder -output_file = SUMMARY_OF_SEVERITY_COUNTS + '/summary_of_severity_counts_for_WGs_in_table.txt' -with open(output_file, 'w') as f: +output_file = ( + SUMMARY_OF_SEVERITY_COUNTS + "/summary_of_severity_counts_for_WGs_in_table.txt" +) +with open(output_file, "w") as f: f.write(str(table)) log("Output saved to:", output_file) log("Severity counts with images respect to WGs are saved in the",ALL_SEVERITY_COUNTS) -log("Scanned Json reports on images are saved in" ,SCAN_REPORTS_DIR) \ No newline at end of file +log("Scanned Json reports on images are saved in",SCAN_REPORTS_DIR) \ No newline at end of file diff --git a/run_yamllint.sh b/run_yamllint.sh new file mode 100644 index 0000000000..3225710f59 --- /dev/null +++ b/run_yamllint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ -s changed_files_in_PR.txt ]; then + while IFS= read -r file; do + echo "Running yamllint on $file" + yamllint "$file" + done < changed_files_in_PR.txt +else + echo "No YAML files changed in this PR." +fi diff --git a/tests/gh-actions/kf-objects/test_pipeline.py b/tests/gh-actions/kf-objects/test_pipeline.py index 9bd8228e5a..6755d30ff4 100755 --- a/tests/gh-actions/kf-objects/test_pipeline.py +++ b/tests/gh-actions/kf-objects/test_pipeline.py @@ -7,22 +7,23 @@ def echo_op(): print("Test pipeline") -@dsl.pipeline( - name='test-pipeline', - description='A test pipeline.' -) + +@dsl.pipeline(name="test-pipeline", description="A test pipeline.") def hello_world_pipeline(): echo_task = echo_op() + if __name__ == "__main__": # Run the Kubeflow Pipeline in the user's namespace. - kfp_client = kfp.Client(host="http://localhost:3000", - namespace="kubeflow-user-example-com") + kfp_client = kfp.Client( + host="http://localhost:3000", namespace="kubeflow-user-example-com" + ) kfp_client.runs.api_client.default_headers.update( - {"kubeflow-userid": "kubeflow-user-example-com"}) + {"kubeflow-userid": "kubeflow-user-example-com"} + ) # create the KFP run run_id = kfp_client.create_run_from_pipeline_func( hello_world_pipeline, namespace="kubeflow-user-example-com", arguments={}, - ).run_id \ No newline at end of file + ).run_id diff --git a/tests/gh-actions/runasnonroot.sh b/tests/gh-actions/runasnonroot.sh index 921623c9c8..532424cd37 100644 --- a/tests/gh-actions/runasnonroot.sh +++ b/tests/gh-actions/runasnonroot.sh @@ -1,102 +1,102 @@ -#!/bin/bash - -namespace="kubeflow" -error_flag=0 - -# Function to check if 'id' command is available in a container -has_id_command() { - local pod_name="$1" - local container_name="$2" - - # Execute 'id' command and capture the output - if kubectl exec -it -n "$namespace" "$pod_name" -c "$container_name" -- id -u >/dev/null 2>&1; then - return 0 # 'id' command is available - else - return 1 # 'id' command is not available - fi -} - -# Function to check 'securityContext' and 'runAsNonRoot' at the pod or container level -has_securityContext_and_runAsNonRoot() { - local pod_name="$1" - local container_name="$2" - - # Use jq to check if 'securityContext' is defined at the pod level - local securityContextPod=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.securityContext') - - if [ "$securityContextPod" = "null" ]; then - : # 'securityContext' is missing at the pod level, continue checking at the container level - else - # Check 'runAsNonRoot' at the pod level - local runAsNonRootPod=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.securityContext.runAsNonRoot // "Missing"') - - if [ "$runAsNonRootPod" = "Missing" ]; then - : # 'runAsNonRoot' is missing at the pod level, continue checking at the container level - else - return 0 # 'runAsNonRoot' is present at the pod level (success) - fi - fi - - # Use jq to check 'securityContext' at the container level - local securityContextContainer=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[] | select(.name == "'"$container_name"'").securityContext') - - if [ "$securityContextContainer" = "null" ]; then - if [ "$securityContextPod" = "null" ]; then - echo "Error: 'securityContext' is missing at the pod and container level in container $container_name of pod $pod_name" - return 1 - else - echo "Error: There is no runasnonroot on pod level and 'securityContext' is missing at container level in container $container_name of pod $pod_name" - return 1 - fi - fi - - # Check 'runAsNonRoot' at the container level - local runAsNonRootContainer=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[] | select(.name == "'"$container_name"'").securityContext.runAsNonRoot // "Missing"') - - if [ "$runAsNonRootContainer" = "Missing" ]; then - echo "Error: There is no runasnonroot on pod level and'runAsNonRoot' is missing in container $container_name of pod $pod_name" - return 1 # 'runAsNonRoot' is missing at the container level (fail) - fi - - return 0 # 'securityContext' and 'runAsNonRoot' are defined at the container level -} - -# Get a list of pod names in the specified namespace that are not in the "Completed" state -pod_names=$(kubectl get pods -n "$namespace" --field-selector=status.phase!=Succeeded,status.phase!=Failed -o json | jq -r '.items[].metadata.name') - -# Loop through the pod names and execute checks -for pod_name in $pod_names; do - echo "Entering pod $pod_name in namespace $namespace..." - - container_names=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[].name') - - for container_name in $container_names; do - if has_securityContext_and_runAsNonRoot "$pod_name" "$container_name"; then - error_flag=1 - fi - - if has_id_command "$pod_name" "$container_name"; then - user_id=$(kubectl exec -it -n "$namespace" "$pod_name" -c "$container_name" -- id -u) - - # Clean up whitespace in the user_id using tr - user_id_cleaned=$(echo -n "$user_id" | tr -d '[:space:]') - - if [ "$user_id_cleaned" = "0" ]; then - echo "Error: Pod $pod_name contains user ID 0 in container $container_name" - error_flag=1 - else - echo "Container: $container_name - User ID: $user_id_cleaned" - fi - else - echo "Warning: 'id' command not available in container $container_name" - fi - done -done - -# Exit with an error if any pod contains an error condition -if [ $error_flag -eq 1 ]; then - exit 1 -fi - -# Exit successfully -exit 0 +#!/bin/bash + +namespace="kubeflow" +error_flag=0 + +# Function to check if 'id' command is available in a container +has_id_command() { + local pod_name="$1" + local container_name="$2" + + # Execute 'id' command and capture the output + if kubectl exec -it -n "$namespace" "$pod_name" -c "$container_name" -- id -u >/dev/null 2>&1; then + return 0 # 'id' command is available + else + return 1 # 'id' command is not available + fi +} + +# Function to check 'securityContext' and 'runAsNonRoot' at the pod or container level +has_securityContext_and_runAsNonRoot() { + local pod_name="$1" + local container_name="$2" + + # Use jq to check if 'securityContext' is defined at the pod level + local securityContextPod=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.securityContext') + + if [ "$securityContextPod" = "null" ]; then + : # 'securityContext' is missing at the pod level, continue checking at the container level + else + # Check 'runAsNonRoot' at the pod level + local runAsNonRootPod=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.securityContext.runAsNonRoot // "Missing"') + + if [ "$runAsNonRootPod" = "Missing" ]; then + : # 'runAsNonRoot' is missing at the pod level, continue checking at the container level + else + return 0 # 'runAsNonRoot' is present at the pod level (success) + fi + fi + + # Use jq to check 'securityContext' at the container level + local securityContextContainer=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[] | select(.name == "'"$container_name"'").securityContext') + + if [ "$securityContextContainer" = "null" ]; then + if [ "$securityContextPod" = "null" ]; then + echo "Error: 'securityContext' is missing at the pod and container level in container $container_name of pod $pod_name" + return 1 + else + echo "Error: There is no runasnonroot on pod level and 'securityContext' is missing at container level in container $container_name of pod $pod_name" + return 1 + fi + fi + + # Check 'runAsNonRoot' at the container level + local runAsNonRootContainer=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[] | select(.name == "'"$container_name"'").securityContext.runAsNonRoot // "Missing"') + + if [ "$runAsNonRootContainer" = "Missing" ]; then + echo "Error: There is no runasnonroot on pod level and'runAsNonRoot' is missing in container $container_name of pod $pod_name" + return 1 # 'runAsNonRoot' is missing at the container level (fail) + fi + + return 0 # 'securityContext' and 'runAsNonRoot' are defined at the container level +} + +# Get a list of pod names in the specified namespace that are not in the "Completed" state +pod_names=$(kubectl get pods -n "$namespace" --field-selector=status.phase!=Succeeded,status.phase!=Failed -o json | jq -r '.items[].metadata.name') + +# Loop through the pod names and execute checks +for pod_name in $pod_names; do + echo "Entering pod $pod_name in namespace $namespace..." + + container_names=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[].name') + + for container_name in $container_names; do + if has_securityContext_and_runAsNonRoot "$pod_name" "$container_name"; then + error_flag=1 + fi + + if has_id_command "$pod_name" "$container_name"; then + user_id=$(kubectl exec -it -n "$namespace" "$pod_name" -c "$container_name" -- id -u) + + # Clean up whitespace in the user_id using tr + user_id_cleaned=$(echo -n "$user_id" | tr -d '[:space:]') + + if [ "$user_id_cleaned" = "0" ]; then + echo "Error: Pod $pod_name contains user ID 0 in container $container_name" + error_flag=1 + else + echo "Container: $container_name - User ID: $user_id_cleaned" + fi + else + echo "Warning: 'id' command not available in container $container_name" + fi + done +done + +# Exit with an error if any pod contains an error condition +if [ $error_flag -eq 1 ]; then + exit 1 +fi + +# Exit successfully +exit 0