Added support for assembling on Windows. (#809)

* Added support for assembling on Windows.

Signed-off-by: dblock <dblock@dblock.org>

* Use platform provided.

Signed-off-by: dblock <dblock@amazon.com>

README.md

@@ -28,7 +28,9 @@
   - [Making a Release](#making-a-release)
     - [Releasing for Linux](#releasing-for-linux)
     - [Releasing for FreeBSD](#releasing-for-freebsd)
-  - [Deploying infrastructure](#deploying-infrastructure)
+    - [Releasing for Windows](#releasing-for-windows)
+    - [Releasing for MacOS](#releasing-for-macos)
+  - [Deploying Infrastructure](#deploying-infrastructure)
 - [Contributing](#contributing)
 - [Getting Help](#getting-help)
 - [Code of Conduct](#code-of-conduct)
@@ -353,7 +355,15 @@ The Linux release is managed by a team at Amazon following [this release templat
 
 The FreeBSD ports and packages for OpenSearch are managed by a community [OpenSearch Team](https://wiki.freebsd.org/OpenSearch) at FreeBSD.  When a new release is rolled out, this team will update the port and commit it to the FreeBSD ports tree. Anybody is welcome to help the team by providing patches for [upgrading the ports](https://docs.freebsd.org/en/books/porters-handbook/book/#port-upgrading) following the [FreeBSD Porter's Handbook](https://docs.freebsd.org/en/books/porters-handbook/book/) instructions.
 
-### Deploying infrastructure
+#### Releasing for Windows
+
+At this moment there's no official Windows distribution. However, this project does support building and assembling OpenSearch for Windows, with some caveats. See [opensearch-build#33](https://github.com/opensearch-project/opensearch-build/issues/33) for details.
+
+#### Releasing for MacOS
+
+At this moment there's no official MacOS distribution. However, this project does support building and assembling OpenSearch for MacOS. See [opensearch-build#37](https://github.com/opensearch-project/opensearch-build/issues/37) and [#38](https://github.com/opensearch-project/opensearch-build/issues/38) for more details.
+
+### Deploying Infrastructure
 
 Storage and access roles for the OpenSearch release process are codified in a [CDK project](deployment/README.md).
 

manifests/1.2.0/opensearch-1.2.0.yml

@@ -35,6 +35,9 @@ components:
   - name: k-NN
     repository: https://github.com/opensearch-project/k-NN.git
     ref: "main"
+    platforms:
+      - darwin
+      - linux
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version

scripts/components/OpenSearch/build.sh

@@ -78,25 +78,21 @@ cp -r ./build/local-test-repo/org/opensearch "${OUTPUT}"/maven/org
 [ -z "$PLATFORM" ] && PLATFORM=`uname -s` | awk '{print tolower($0)}'
 [ -z "$ARCHITECTURE" ] && ARCHITECTURE=`uname -m`
 
-case "$(uname -s)" in
-    Linux*) 
+case $PLATFORM in
+    linux*)
         PACKAGE="tar"
         EXT="tar.gz"
         ;;
-    Darwin*)
+    darwin*)
         PACKAGE="tar"
         EXT="tar.gz"
         ;;
-    CYGWIN*)
-        PACKAGE="zip"
-        EXT="zip"
-        ;;
-    MINGW*)
+    windows*)
         PACKAGE="zip"
         EXT="zip"
         ;;
     *)
-        echo "Unsupported system: $(uname -s)"
+        echo "Unsupported platform: $PLATFORM"
         exit 1
         ;;
 esac
@@ -111,7 +107,7 @@ case $ARCHITECTURE in
         QUALIFIER="$PLATFORM-arm64"
         ;;
     *)
-        echo "Unsupported architecture: ${ARCHITECTURE}"
+        echo "Unsupported architecture: $ARCHITECTURE"
         exit 1
         ;;
 esac

src/assemble_workflow/bundle.py

@@ -9,9 +9,9 @@
 import os
 import shutil
 import subprocess
-import tarfile
 from abc import ABC, abstractmethod
 
+from assemble_workflow.dist import Dist
 from paths.script_finder import ScriptFinder
 from system.temporary_directory import TemporaryDirectory
 
@@ -23,69 +23,56 @@
 
 
 class Bundle(ABC):
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        self.tmp_dir.__exit__(exc_type, exc_value, exc_traceback)
+
     def __init__(self, build_manifest, artifacts_dir, bundle_recorder):
         """
         Construct a new Bundle instance.
         :param build_manifest: A BuildManifest created from the build workflow.
         :param artifacts_dir: Dir location where build artifacts can be found locally
         :param bundle_recorder: The bundle recorder that will capture and build a BundleManifest
         """
-        self.min_tarball = self.__get_min_bundle(build_manifest.components.values())
         self.plugins = self.__get_plugins(build_manifest.components.values())
         self.artifacts_dir = artifacts_dir
         self.bundle_recorder = bundle_recorder
         self.tmp_dir = TemporaryDirectory()
+        self.min_dist = self.__get_min_dist(build_manifest.components.values())
         self.installed_plugins = []
-        self.min_tarball_path = self._copy_component(self.min_tarball, "dist")
-        self.__unpack_min_tarball(self.tmp_dir.name)
 
     def install_min(self):
-        post_install_script = ScriptFinder.find_install_script(self.min_tarball.name)
-        self._execute(f'{post_install_script} -a "{self.artifacts_dir}" -o "{self.archive_path}"')
+        post_install_script = ScriptFinder.find_install_script(self.min_dist.name)
+        self._execute(f'bash {post_install_script} -a "{self.artifacts_dir}" -o "{self.min_dist.archive_path}"')
 
     def install_plugins(self):
         for plugin in self.plugins:
             logging.info(f"Installing {plugin.name}")
             self.install_plugin(plugin)
-        plugins_path = os.path.join(self.archive_path, "plugins")
+        plugins_path = os.path.join(self.min_dist.archive_path, "plugins")
         if os.path.isdir(plugins_path):
             self.installed_plugins = os.listdir(plugins_path)
 
     @abstractmethod
     def install_plugin(self, plugin):
         post_install_script = ScriptFinder.find_install_script(plugin.name)
-        self._execute(f'{post_install_script} -a "{self.artifacts_dir}" -o "{self.archive_path}"')
+        self._execute(f'bash {post_install_script} -a "{self.artifacts_dir}" -o "{self.min_dist.archive_path}"')
 
-    def build_tar(self, dest):
-        tar_name = self.bundle_recorder.tar_name
-        with tarfile.open(tar_name, "w:gz") as tar:
-            tar.add(self.archive_path, arcname=os.path.basename(self.archive_path))
-        shutil.copyfile(tar_name, os.path.join(dest, tar_name))
+    def package(self, dest):
+        self.min_dist.build(self.bundle_recorder.package_name, dest)
 
     def _execute(self, command):
-        logging.info(f'Executing "{command}" in {self.archive_path}')
-        subprocess.check_call(command, cwd=self.archive_path, shell=True)
+        logging.info(f'Executing "{command}" in {self.min_dist.archive_path}')
+        subprocess.check_call(command, cwd=self.min_dist.archive_path, shell=True)
 
     def _copy_component(self, component, component_type):
         rel_path = self.__get_rel_path(component, component_type)
         tmp_path = self.__copy_component_files(rel_path, self.tmp_dir.name)
         self.bundle_recorder.record_component(component, rel_path)
         return tmp_path
 
-    def __unpack_min_tarball(self, dest):
-        with tarfile.open(self.min_tarball_path) as tar:
-            tar.extractall(dest)
-
-        self.archive_path = self.__get_archive_path(dest)
-
-    # OpenSearch & Dashboard tars will include only a single folder at the top level of the tar.
-    def __get_archive_path(self, dest):
-        for file in os.scandir(dest):
-            if file.is_dir():
-                return file.path
-
-        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), os.path.join(dest, "*"))
-
     def __get_rel_path(self, component, component_type):
         return next(iter(component.artifacts.get(component_type, [])), None)
 
@@ -102,8 +89,11 @@ def __copy_component_files(self, rel_path, dest):
     def __get_plugins(self, build_components):
         return [c for c in build_components if "plugins" in c.artifacts]
 
-    def __get_min_bundle(self, build_components):
+    def __get_min_dist(self, build_components):
         min_bundle = next(iter([c for c in build_components if "dist" in c.artifacts]), None)
         if min_bundle is None:
             raise ValueError('Missing min "dist" in input artifacts.')
-        return min_bundle
+        min_dist_path = self._copy_component(min_bundle, "dist")
+        min_dist = Dist.from_path(min_bundle.name, min_dist_path)
+        min_dist.extract(self.tmp_dir.name)
+        return min_dist

src/assemble_workflow/bundle_opensearch.py

@@ -7,11 +7,16 @@
 import os
 
 from assemble_workflow.bundle import Bundle
+from system.os import current_platform
 
 
 class BundleOpenSearch(Bundle):
+    @property
+    def install_plugin_script(self):
+        return "opensearch-plugin.bat" if current_platform() == "windows" else "opensearch-plugin"
+
     def install_plugin(self, plugin):
         tmp_path = self._copy_component(plugin, "plugins")
-        cli_path = os.path.join(self.archive_path, "bin", "opensearch-plugin")
+        cli_path = os.path.join(self.min_dist.archive_path, "bin", self.install_plugin_script)
         self._execute(f"{cli_path} install --batch file:{tmp_path}")
         super().install_plugin(plugin)

src/assemble_workflow/bundle_opensearch_dashboards.py

@@ -12,6 +12,6 @@
 class BundleOpenSearchDashboards(Bundle):
     def install_plugin(self, plugin):
         tmp_path = self._copy_component(plugin, "plugins")
-        cli_path = os.path.join(self.archive_path, "bin", "opensearch-dashboards-plugin")
+        cli_path = os.path.join(self.min_dist.archive_path, "bin", "opensearch-dashboards-plugin")
         self._execute(f"{cli_path} --allow-root install file:{tmp_path}")
         super().install_plugin(plugin)

src/assemble_workflow/bundle_recorder.py

@@ -16,7 +16,7 @@ def __init__(self, build, output_dir, artifacts_dir, base_url):
         self.build_id = build.id
         self.base_url = base_url
         self.version = build.version
-        self.tar_name = self.__get_tar_name(build)
+        self.package_name = self.__get_package_name(build)
         self.artifacts_dir = artifacts_dir
         self.architecture = build.architecture
         self.bundle_manifest = self.BundleManifestBuilder(
@@ -25,17 +25,17 @@ def __init__(self, build, output_dir, artifacts_dir, base_url):
             build.version,
             build.platform,
             build.architecture,
-            self.__get_tar_location(),
+            self.__get_package_location(),
         )
 
-    def __get_tar_name(self, build):
+    def __get_package_name(self, build):
         parts = [
             build.name.lower().replace(" ", "-"),
             build.version,
             build.platform,
             build.architecture,
         ]
-        return "-".join(parts) + ".tar.gz"
+        return "-".join(parts) + (".zip" if build.platform == "windows" else ".tar.gz")
 
     def __get_public_url_path(self, folder, rel_path):
         path = "/".join((folder, rel_path))
@@ -48,8 +48,8 @@ def __get_location(self, folder_name, rel_path, abs_path):
 
     # Assembled bundles are expected to be served from a separate "bundles" folder
     # Example: https://artifacts.opensearch.org/bundles/1.0.0/<build-id
-    def __get_tar_location(self):
-        return self.__get_location("dist", self.tar_name, os.path.join(self.output_dir, self.tar_name))
+    def __get_package_location(self):
+        return self.__get_location("dist", self.package_name, os.path.join(self.output_dir, self.package_name))
 
     # Build artifacts are expected to be served from a "builds" folder
     # Example: https://artifacts.opensearch.org/builds/1.0.0/<build-id>

src/assemble_workflow/dist.py

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import errno
+import logging
+import os
+import shutil
+import tarfile
+import zipfile
+from abc import ABC, abstractmethod
+
+
+class Dist(ABC):
+    def __init__(self, name, path):
+        self.name = name
+        self.path = path
+
+    @abstractmethod
+    def __extract__(self, dest):
+        pass
+
+    def extract(self, dest):
+        self.__extract__(dest)
+
+        # OpenSearch & Dashboard tars will include only a single folder at the top level of the tar.
+
+        for file in os.scandir(dest):
+            if file.is_dir():
+                self.archive_path = file.path
+                return self.archive_path
+
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), os.path.join(dest, "*"))
+
+    def build(self, name, dest):
+        self.__build__(name, dest)
+        path = os.path.join(dest, name)
+        shutil.copyfile(name, path)
+        logging.info(f"Published {path}.")
+
+    @classmethod
+    def from_path(cls, name, path):
+        ext = os.path.splitext(path)[1]
+        if ext == ".gz":
+            return DistTar(name, path)
+        elif ext == ".zip":
+            return DistZip(name, path)
+        else:
+            raise ValueError(f'Invalid min "dist" extension in input artifacts: {ext} ({path}).')
+
+
+class DistZip(Dist):
+    def __extract__(self, dest):
+        with zipfile.ZipFile(self.path, "r") as zip:
+            zip.extractall(dest)
+
+    def __build__(self, name, dest):
+        with zipfile.ZipFile(name, "w", zipfile.ZIP_DEFLATED) as zip:
+            rootlen = len(self.archive_path) + 1
+            for base, dirs, files in os.walk(self.archive_path):
+                for file in files:
+                    fn = os.path.join(base, file)
+                    zip.write(fn, fn[rootlen:])
+
+
+class DistTar(Dist):
+    def __extract__(self, dest):
+        with tarfile.open(self.path, "r") as tar:
+            tar.extractall(dest)
+
+    def __build__(self, name, dest):
+        with tarfile.open(name, "w:gz") as tar:
+            tar.add(self.archive_path, arcname=os.path.basename(self.archive_path))

src/run_assemble.py

@@ -30,7 +30,7 @@ def main():
         const=logging.DEBUG,
         dest="logging_level",
     )
-    parser.add_argument("-b", "--base-url", dest='base_url', help="The base url to download the artifacts.")
+    parser.add_argument("-b", "--base-url", dest="base_url", help="The base url to download the artifacts.")
     args = parser.parse_args()
 
     console.configure(level=args.logging_level)
@@ -46,17 +46,16 @@ def main():
 
         bundle_recorder = BundleRecorder(build, output_dir, artifacts_dir, args.base_url)
 
-        bundle = Bundles.create(build_manifest, artifacts_dir, bundle_recorder)
+        with Bundles.create(build_manifest, artifacts_dir, bundle_recorder) as bundle:
+            bundle.install_min()
+            bundle.install_plugins()
+            logging.info(f"Installed plugins: {bundle.installed_plugins}")
 
-        bundle.install_min()
-        bundle.install_plugins()
-        logging.info(f"Installed plugins: {bundle.installed_plugins}")
+            #  Save a copy of the manifest inside of the tar
+            bundle_recorder.write_manifest(bundle.min_dist.archive_path)
+            bundle.package(output_dir)
 
-        #  Save a copy of the manifest inside of the tar
-        bundle_recorder.write_manifest(bundle.archive_path)
-        bundle.build_tar(output_dir)
-
-        bundle_recorder.write_manifest(output_dir)
+            bundle_recorder.write_manifest(output_dir)
 
     logging.info("Done.")
 

tests/test_run_assemble.py

@@ -33,20 +33,23 @@ def test_usage(self, *mocks):
     @patch("os.makedirs")
     @patch("os.getcwd", return_value="curdir")
     @patch("argparse._sys.argv", ["run_assemble.py", BUILD_MANIFEST])
-    @patch("run_assemble.Bundles", return_value=MagicMock())
+    @patch("run_assemble.Bundles.create")
     @patch("run_assemble.BundleRecorder", return_value=MagicMock())
     @patch("run_assemble.TemporaryDirectory")
     @patch("shutil.copy2")
     def test_main(self, mock_copy, mock_temp, mock_recorder, mock_bundles, *mocks):
         mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir()
-        mock_bundle = MagicMock(archive_path="path")
-        mock_bundles.create.return_value = mock_bundle
+        mock_bundle = MagicMock(min_dist=MagicMock(archive_path="path"))
+        mock_bundles.return_value.__enter__.return_value = mock_bundle
 
         main()
 
         mock_bundle.install_min.assert_called()
         mock_bundle.install_plugins.assert_called()
 
-        mock_bundle.build_tar.assert_called_with(os.path.join("curdir", "dist"))
+        mock_bundle.package.assert_called_with(os.path.join("curdir", "dist"))
 
-        mock_recorder.return_value.write_manifest.assert_has_calls([call("path"), call(os.path.join("curdir", "dist"))])  # manifest included in tar
+        mock_recorder.return_value.write_manifest.assert_has_calls([
+            call("path"),
+            call(os.path.join("curdir", "dist"))
+        ])  # manifest included in package