Enable canary report generation (#643)

-Merge release branch PR to main
#634

tests/canary/canary.buildspec.yaml

@@ -6,7 +6,6 @@ phases:
       # Get cached test image
       - aws ecr get-login-password --region $CLUSTER_REGION | docker login --username AWS --password-stdin $ECR_CACHE_URI || true
       - docker pull ${ECR_CACHE_URI}:latest --quiet || true
-
       # Build test image
       - >
         docker build -f ./tests/canary/Dockerfile.canary . -t ${ECR_CACHE_URI}:latest --quiet
@@ -15,8 +14,15 @@ phases:
     commands:
       # Run tests
       - docker run --name kf-distro-canary $(env | cut -f1 -d= | sed 's/^/-e /') --mount type=bind,source="$(pwd)/",target="/kubeflow-manifests/" ${ECR_CACHE_URI}:latest
-
-      # Push test image to cache ECR repo
+  post_build:
+    commands:
+      - docker cp kf-distro-canary:/kubeflow-manifests/tests/canary/integration_tests.xml /tmp/results.xml || true
+        # Push test image to cache ECR repo
       - docker push ${ECR_CACHE_URI}:latest || true
-
+
+reports:
+  IntegrationTestReport:
+    files:
+      - "results.xml"
+    base-directory: "/tmp"    
 

tests/canary/scripts/push_stats_to_cloudwatch.py

@@ -0,0 +1,90 @@
+import boto3
+from datetime import datetime
+import xml.etree.ElementTree as ET
+import os
+
+
+xml_path = "../canary/integration_tests.xml"
+
+
+def readXML_and_publish_metrics_to_cw():
+    if os.path.isfile(xml_path):
+        tree = ET.parse(xml_path)
+        testsuite = tree.find("testsuite")
+        failures = testsuite.attrib["failures"]
+        tests = testsuite.attrib["tests"]
+        successes = int(tests) - int(failures)
+    else:
+        failures = 0
+        successes = 0
+        tests = 1
+
+    timestamp = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
+
+    print(f"Failures: {failures}")
+    print(f"Total tests: {tests}")
+    print(f"Success: {successes}")
+
+    # push to cloudwatch
+    cw_client = boto3.client("cloudwatch")
+    project_name = "CodeBuild-Run-All-Tests"
+
+    # Define the metric data
+    metric_data = [
+        {
+            "MetricName": "failures",
+            "Timestamp": timestamp,
+            "Dimensions": [
+                {"Name": "CodeBuild Project Name", "Value": project_name},
+            ],
+            "Value": int(failures),
+            "Unit": "Count",
+        },
+        {
+            "MetricName": "total_tests",
+            "Timestamp": timestamp,
+            "Dimensions": [
+                {"Name": "CodeBuild Project Name", "Value": project_name},
+            ],
+            "Value": int(tests),
+            "Unit": "Count",
+        },
+        {
+            "MetricName": "successes",
+            "Timestamp": timestamp,
+            "Dimensions": [
+                {"Name": "CodeBuild Project Name", "Value": project_name},
+            ],
+            "Value": int(successes),
+            "Unit": "Count",
+        },
+    ]
+
+    # Use the put_metric_data method to push the metric data to CloudWatch
+    try:
+        response = cw_client.put_metric_data(
+            Namespace="Canary_Metrics", MetricData=metric_data
+        )
+        if response["ResponseMetadata"]["HTTPStatusCode"] == 200:
+            print("Successfully pushed data to CloudWatch")
+            # return 200 status code if successful
+            return 200
+        else:
+            # raise exception if the status code is not 200
+            raise Exception(
+                "Unexpected response status code: {}".format(
+                    response["ResponseMetadata"]["HTTPStatusCode"]
+                )
+            )
+    except Exception as e:
+        print("Error pushing data to CloudWatch: {}".format(e))
+        # raise exception if there was an error pushing data to CloudWatch
+        raise
+
+
+def main():
+    readXML_and_publish_metrics_to_cw()
+
+
+if __name__ == "__main__":
+    main()

tests/canary/scripts/run_test.sh

@@ -9,10 +9,11 @@
 # Script configuration
 set -euo pipefail
 
-function onError {
-  echo "Run test FAILED. Exiting."
+function push_to_cloudwatch {
+  echo "Pushing Codebuild stats to Cloudwatch."
+  python ../canary/scripts/push_stats_to_cloudwatch.py
 }
-trap onError ERR
+trap push_to_cloudwatch EXIT
 
 export CANARY_TEST_DIR=${REPO_PATH}/tests/canary
 export E2E_TEST_DIR=${REPO_PATH}/tests/e2e
@@ -31,6 +32,5 @@ mkdir -p $E2E_TEST_DIR/.metadata/
 cp metadata-canary $E2E_TEST_DIR/.metadata/
 
 cd $E2E_TEST_DIR
-pytest tests/test_sanity_portforward.py -s -q --metadata .metadata/metadata-canary --region $CLUSTER_REGION --installation_option $INSTALLATION_OPTION
-
+pytest tests/test_sanity_portforward.py -s -q --metadata .metadata/metadata-canary --region $CLUSTER_REGION --installation_option $INSTALLATION_OPTION --junitxml ../canary/integration_tests.xml