Added Test Publisher

bundle-workflow/src/test_workflow/test_publisher.py

@@ -0,0 +1,39 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src"))
+from aws.s3_bucket import S3Bucket
+
+
+class TestPublisher:
+    def __init__(self, s3_bucket, bundle_manifest, test_recorder):
+        self.s3_bucket = s3_bucket
+        self.bundle_manifest = bundle_manifest
+        self.test_recorder = test_recorder
+
+    def publish_test_results_to_s3(self):
+        """
+            Publishes tests results to S3 pulling information from {self.test_recorder}
+            And cleans up all local storage after publishing ({self.test_recorder}.clean_up())
+        """
+        s3_bucket = S3Bucket(self.s3_bucket, '<role-arn>', 'test-publisher-session')
+        base_path = self._get_base_path()
+
+        for subdir, dirs, files in os.walk(self.test_recorder.location):
+            test_path = subdir[subdir.find('tests'):]
+            for file_name in files:
+                file_path = os.path.join(subdir, file_name)
+                if not file_path.startswith('.'):
+                    s3_path = os.path.join(base_path, test_path, file_name)
+                    s3_bucket.upload_file(s3_path, file_path)
+
+    def _get_base_path(self):
+        """
+            Returns the base path to store logs: /builds/bundles/<bundle-version>/<build-id>/<arch-id>/tests/
+        """
+        work_dir = 'builds/bundles'
+        bundle_version = self.bundle_manifest.build.version
+        build_id = self.bundle_manifest.build.id
+        arch = self.bundle_manifest.build.architecture
+        s3_path = os.path.join(work_dir, bundle_version, build_id, arch)
+        return s3_path

bundle-workflow/tests/test_publisher/__init__.py

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src"))

bundle-workflow/tests/test_publisher/data/bundle_manifest.yaml

@@ -0,0 +1,63 @@
+build:
+  architecture: x64
+  id: 41d5ae25183d4e699e92debfbe3f83bd
+  location: https://artifacts.opensearch.org/bundles/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/opensearch-1.0.0-linux-x64.tar.gz
+  name: OpenSearch
+  version: 1.0.0
+components:
+  - commit_id: fb25458f38c30a7ab06de21b0068f1fe3ad56134
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/bundle/opensearch-min-1.0.0-linux-x64.tar.gz
+    name: OpenSearch
+    ref: 1.0
+    repository: https://github.com/saratvemulapalli/OpenSearch.git
+  - commit_id: 7fad9529358259de529763c1c923fd947817a3bd
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-job-scheduler-1.0.0.0.zip
+    name: job-scheduler
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/job-scheduler.git
+  - commit_id: 65bb94fb7d46a88b07b61622585ed701918b19c5
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-sql-1.0.0.0.zip
+    name: sql
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/sql.git
+  - commit_id: a14ccd49389ca41446acc3200e3e870cde15a68e
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-alerting-1.0.0.0.zip
+    name: alerting
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/alerting.git
+  - commit_id: 2e21d59749526baa8e4666168643c4594cdadf79
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-security-1.0.0.0.zip
+    name: security
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/security.git
+  - commit_id: 091fe9f6612cd7e85054918036587bcd3c67eab1
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-index-management-1.0.0.0.zip
+    name: index-management
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/index-management.git
+  - commit_id: 9b29a99b05f2c8cd9d54dc868994cab8460ff0db
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-knn-1.0.0.0.zip
+    name: k-NN
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/k-NN.git
+  - commit_id: 502c96e54fae1cec9fee1fafd77ad92fde9d2459
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-anomaly-detection-1.0.0.0.zip
+    name: anomaly-detection
+    ref: 1.0
+    repository: https://github.com/opensearch-project/anomaly-detection.git
+  - commit_id: bd31e80adf6d52c1b4662d0d2cc9b30d8ae14309
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-asynchronous-search-1.0.0.0.zip
+    name: asynchronous-search
+    ref: main
+    repository: https://github.com/opensearch-project/asynchronous-search.git
+  - commit_id: 72705e2dfcad760c5de7609891700aa11d767884
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-reports-scheduler-1.0.0.0.zip
+    name: dashboards-reports
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/dashboards-reports.git
+  - commit_id: fd745a77c19df4991254b495cf0ec3730c66534d
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-notebooks-1.0.0.0.zip
+    name: dashboards-notebooks
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/dashboards-notebooks.git
+schema-version: '1.0'

bundle-workflow/tests/test_publisher/test_test_publisher.py

@@ -0,0 +1,42 @@
+import os
+import unittest
+from unittest.mock import patch
+
+from aws.s3_bucket import S3Bucket
+from manifests.bundle_manifest import BundleManifest
+from test_workflow.test_publisher import TestPublisher
+
+TestPublisher.__test__ = False
+
+
+class TestTestPublisher(unittest.TestCase):
+    def setUp(self):
+        self.data_path = os.path.realpath(
+            os.path.join(os.path.dirname(__file__), "data")
+        )
+        self.manifest_filename = os.path.join(
+            self.data_path, "bundle_manifest.yaml"
+        )
+        self.manifest = BundleManifest.from_path(self.manifest_filename)
+        self.bucket_name = "unitTestBucket"
+        self.test_publisher = TestPublisher(
+            s3_bucket=self.bucket_name, bundle_manifest=self.manifest, test_recorder=None
+        )
+
+    def test_get_base_path(self):
+        s3_path = self.test_publisher._get_base_path()
+        self.assertEqual(s3_path, 'builds/bundles/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/x64')
+
+    @patch("boto3.client")
+    def test_publish_test_results_to_s3(self, mock_boto_client):
+        s3bucket = S3Bucket(self.bucket_name)
+        self.test_publisher.publish_test_results_to_s3()
+        s3bucket.upload_file(
+            "tests/1.1.0/x64/opensearch-1.1.0-linux-x64.tar.gz",
+            "/tmp/opensearch-1.1.0-linux-x64.tar.gz",
+        )
+        mock_boto_client("s3").upload_file.assert_called_with(
+            "/tmp/opensearch-1.1.0-linux-x64.tar.gz",
+            self.bucket_name,
+            "tests/1.1.0/x64/opensearch-1.1.0-linux-x64.tar.gz",
+        )