Add minimumCompilerVersion check for manifest update on OpenSearch Mi…

…n branches (#4953)

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

src/manifests_workflow/component_opensearch_min.py

@@ -5,6 +5,7 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
+import os
 from typing import Any, List
 
 from git.git_repository import GitRepository
@@ -14,6 +15,8 @@
 
 
 class ComponentOpenSearchMin(Component):
+ path = None
+
  def __init__(self, repo: GitRepository, snapshot: bool = False) -> None:
  super().__init__(
  "OpenSearch",
@@ -23,19 +26,31 @@ def __init__(self, repo: GitRepository, snapshot: bool = False) -> None:
  )
 
  @classmethod
- def branches(self, url: str = "https://github.com/opensearch-project/OpenSearch.git") -> List[str]:
+ def branches(cls, url: str = "https://github.com/opensearch-project/OpenSearch.git") -> List[str]:
  return Component.branches(url)
 
  @classmethod
- def checkout(self, path: str, branch: str = "main", snapshot: bool = False) -> 'ComponentOpenSearchMin':
+ def checkout(cls, path: str, branch: str = "main", snapshot: bool = False) -> 'ComponentOpenSearchMin':
+ cls.path = path
  return ComponentOpenSearchMin(
  GitRepository("https://github.com/opensearch-project/OpenSearch.git", branch, path),
  snapshot,
  )
 
  @property
  def properties(self) -> PropertiesFile:
- cmd = ComponentOpenSearch.gradle_cmd("properties", {"build.snapshot": str(self.snapshot).lower()})
+ min_comp_version_path = os.path.join(self.path, "buildSrc", "src", "main", "resources", "minimumCompilerVersion")
+ # Trying to read the minimumCompilerVersion file
+ # And force gradle to apply java home path defined with env var JAVA<Version>_HOME, i.e. JAVA11_HOME
+ # If file is not found then fallback to the default java home defined by host
+ java_home_path = None
+ with open(min_comp_version_path, "r") as file:
+ java_home_path = os.getenv(f"JAVA{file.read().strip()}_HOME", None)
+
+ if java_home_path is None:
+ cmd = ComponentOpenSearch.gradle_cmd("properties", {"build.snapshot": str(self.snapshot).lower()})
+ else:
+ cmd = ComponentOpenSearch.gradle_cmd("properties", {"build.snapshot": str(self.snapshot).lower(), "org.gradle.java.home": java_home_path})
  return PropertiesFile(self.git_repo.output(cmd))
 
  @property

src/manifests_workflow/input_manifests.py

@@ -110,8 +110,9 @@ def update(
  logging.info(f"Ignoring {self.name} {sorted(set(all_branches) - set(branches))} branches as they are legacy")
 
  for branch in branches:
+ repo_path = os.path.join(work_dir.name, self.name.replace(" ", ""), branch)
  min_component_klass = min_klass.checkout(
- path=os.path.join(work_dir.name, self.name.replace(" ", ""), branch),
+ path=repo_path,
  branch=branch,
  )
 

tests/tests_manifests_workflow/data/repository/OpenSearch/buildSrc/src/main/resources/minimumCompilerVersion

@@ -0,0 +1 @@
+11

tests/tests_manifests_workflow/test_component_opensearch_min.py

@@ -5,6 +5,7 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
+import os
 import unittest
 from unittest.mock import MagicMock, patch
 
@@ -13,6 +14,9 @@
 
 
 class TestComponentOpenSearchMin(unittest.TestCase):
+ TEST_DATA_PATH = os.path.join(os.path.dirname(__file__), "data")
+ TEST_REPO_PATH = os.path.join(TEST_DATA_PATH, "repository", "OpenSearch")
+
  @patch("subprocess.check_output")
  def test_branches(self, mock: MagicMock) -> None:
  mock.return_value = "\n".join(["main", "1.x", "1.21", "20.1", "something", "else"]).encode()
@@ -25,7 +29,7 @@ def test_branches(self, mock: MagicMock) -> None:
  @patch("os.makedirs")
  @patch.object(GitRepository, "__checkout__")
  def test_checkout(self, *mocks: MagicMock) -> None:
- component = ComponentOpenSearchMin.checkout("path")
+ component = ComponentOpenSearchMin.checkout(self.TEST_REPO_PATH)
  self.assertEqual(component.name, "OpenSearch")
  self.assertFalse(component.snapshot)
 

tests/tests_manifests_workflow/test_input_manifests_opensearch.py

@@ -38,26 +38,26 @@ def test_update(self, mock_component_opensearch_min: MagicMock, mock_manifest_to
  mock_add_to_cron: MagicMock, mock_add_to_versionincrement_workflow: MagicMock,
  *mocks: MagicMock) -> None:
  mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch")
- mock_component_opensearch_min.branches.return_value = ["2.12"]
- mock_component_opensearch_min.checkout.return_value = MagicMock(version="2.12.1000")
+ mock_component_opensearch_min.branches.return_value = ["2.1000"]
+ mock_component_opensearch_min.checkout.return_value = MagicMock(version="2.1000.1000")
  manifests = InputManifestsOpenSearch()
  manifests.update()
  self.assertEqual(mock_manifest_to_file.call_count, 1)
  calls = [
  call(
  os.path.join(
  InputManifestsOpenSearch.manifests_path(),
- "2.12.1000",
- "opensearch-2.12.1000.yml",
+ "2.1000.1000",
+ "opensearch-2.1000.1000.yml",
  )
  )
  ]
  mock_manifest_to_file.assert_has_calls(calls)
  mock_add_to_cron.assert_has_calls([
- call('2.12.1000'),
+ call('2.1000.1000'),
  ])
  mock_add_to_versionincrement_workflow.assert_has_calls([
- call('2.12.1000'),
+ call('2.1000.1000'),
  ])
 
  @patch("manifests_workflow.input_manifests.InputManifests.add_to_versionincrement_workflow")

tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py

@@ -38,8 +38,8 @@ def test_update(self, mock_component_opensearch_dashboards_min: MagicMock, mock_
  mock_add_to_cron: MagicMock, mock_add_to_versionincrement_workflow: MagicMock,
  mock_os_chdir: MagicMock, mock_os_makedirs: MagicMock) -> None:
  mock_component_opensearch_dashboards_min.return_value = MagicMock(name="OpenSearch-Dashboards")
- mock_component_opensearch_dashboards_min.branches.return_value = ["2.12"]
- mock_component_opensearch_dashboards_min.checkout.return_value = MagicMock(version="2.12.1000")
+ mock_component_opensearch_dashboards_min.branches.return_value = ["2.1000"]
+ mock_component_opensearch_dashboards_min.checkout.return_value = MagicMock(version="2.1000.1000")
 
  manifests = InputManifestsOpenSearchDashboards()
  manifests.update()
@@ -48,17 +48,17 @@ def test_update(self, mock_component_opensearch_dashboards_min: MagicMock, mock_
  call(
  os.path.join(
  InputManifestsOpenSearchDashboards.manifests_path(),
- "2.12.1000",
- "opensearch-dashboards-2.12.1000.yml",
+ "2.1000.1000",
+ "opensearch-dashboards-2.1000.1000.yml",
  )
  )
  ]
  mock_manifest_to_file.assert_has_calls(calls)
  mock_add_to_cron.assert_has_calls([
- call('2.12.1000')
+ call('2.1000.1000')
  ])
  mock_add_to_versionincrement_workflow.assert_has_calls([
- call('2.12.1000')
+ call('2.1000.1000')
  ])
 
  @patch("manifests_workflow.input_manifests.InputManifests.add_to_versionincrement_workflow")