Moving cdsw from yarndevtools: Make Tests Great Again

pyproject.toml

@@ -38,7 +38,7 @@ yarndevtools = ["cdsw/unit-test-result-aggregator/*.txt", "cdsw/scripts/*.sh"]
 python = "^3.8.12"
 python-common-lib = "1.0.11"
 google-api-wrapper2 = "1.0.12"
-cdsw-job-launcher = "1.0.3"
+cdsw-job-launcher = "1.0.5"
 gitpython = "*"
 humanize = "*"
 bs4 = "*"
@@ -61,6 +61,7 @@ coolname = "*"
 httpretty = "*"
 black = "*"
 autopep8 = "*"
+setuptools = "72.1.0"
 
 [tool.poetry.group.localdev.dependencies]
 python-common-lib = { path = "../python-commons/", develop = false }
@@ -88,6 +89,23 @@ build-backend = "poetry.core.masonry.api"
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 

tests/cdsw/test_branch_comparator.py

@@ -4,6 +4,10 @@
 from enum import Enum
 from typing import Dict, List
 
+from cdswjoblauncher.cdsw.cdsw_common import TestExecMode, PythonModuleMode, DEFAULT_TEST_EXECUTION_MODE
+from cdswjoblauncher.cdsw.constants import CdswEnvVar
+from cdswjoblauncher.cdsw.testutils.test_utils import LocalDirs, CDSW_DIRNAME, CdswTestingCommons
+from cdswjoblauncher.commands.send_latest_command_data_in_mail import EnvVar
 from pythoncommons.constants import ExecutionMode
 from pythoncommons.docker_wrapper import DockerTestSetup, CreatePathMode, DockerMountMode, DockerMount
 from pythoncommons.file_utils import FileUtils, FindResultType
@@ -17,17 +21,10 @@
     ProjectUtils,
     ProjectUtilsEnvVar,
 )
-
-from tests.cdsw.common.testutils.cdsw_testing_common import CdswTestingCommons, CDSW_DIRNAME, LocalDirs
-from yarndevtools.cdsw.cdsw_common import (
-    DEFAULT_TEST_EXECUTION_MODE,
-)
-from yarndevtools.cdsw.constants import (
-    CdswEnvVar,
-    BranchComparatorEnvVar,
-)
+from cdswjoblauncher.cdsw.cdsw_common import CommonDirs as CommonDirsCdsw
+from yarndevtools.cdsw.constants import BranchComparatorEnvVar
 from yarndevtools.cdsw.start_job import CommonDirs
-from yarndevtools.common.shared_command_utils import RepoType, EnvVar, CommandType
+from yarndevtools.common.shared_command_utils import RepoType, CommandType
 from yarndevtools.constants import (
     ORIGIN_BRANCH_3_3,
     ORIGIN_TRUNK,
@@ -51,17 +48,15 @@
 
 
 class ContainerFiles:
-    START_JOB_SCRIPT = FileUtils.join_path(CommonDirs.YARN_DEV_TOOLS_SCRIPTS_BASEDIR, START_JOB_PY)
-    INITIAL_CDSW_SETUP_SCRIPT = FileUtils.join_path(
-        CommonDirs.YARN_DEV_TOOLS_SCRIPTS_BASEDIR, INITIAL_CDSW_SETUP_SCRIPT
-    )
+    START_JOB_SCRIPT = FileUtils.join_path(CommonDirsCdsw.SCRIPTS_BASEDIR, START_JOB_PY)
+    INITIAL_CDSW_SETUP_SCRIPT = FileUtils.join_path(CommonDirsCdsw.SCRIPTS_BASEDIR, INITIAL_CDSW_SETUP_SCRIPT)
 
 
 class ContainerDirs:
     CDSW_BASEDIR = CommonDirs.CDSW_BASEDIR
     YARN_DEV_TOOLS_OUTPUT_DIR = FileUtils.join_path(CDSW_BASEDIR, PROJECTS_BASEDIR_NAME, YARNDEVTOOLS_MODULE_NAME)
-    YARN_DEV_TOOLS_SCRIPTS_BASEDIR = CommonDirs.YARN_DEV_TOOLS_SCRIPTS_BASEDIR
-    YARN_DEV_TOOLS_SCRIPTS_EXPERIMENTS = FileUtils.join_path(CommonDirs.YARN_DEV_TOOLS_SCRIPTS_BASEDIR, "experiments")
+    YARN_DEV_TOOLS_SCRIPTS_BASEDIR = CommonDirsCdsw.SCRIPTS_BASEDIR
+    YARN_DEV_TOOLS_SCRIPTS_EXPERIMENTS = FileUtils.join_path(CommonDirsCdsw.SCRIPTS_BASEDIR, "experiments")
     HADOOP_CLOUDERA_BASEDIR = CommonDirs.HADOOP_CLOUDERA_BASEDIR
     HADOOP_UPSTREAM_BASEDIR = CommonDirs.HADOOP_UPSTREAM_BASEDIR
     CDSW_SECRET_DIR = FileUtils.join_path("/root", ".secret", "projects", "cloudera", CDSW_DIRNAME)
@@ -163,12 +158,12 @@ def make_key(prefix, conf_value):
             make_key(p_exec_mode, get_str(TestExecMode.CLOUDERA)): {
                 # We need both upstream / downstream repos for Cloudera-mode
                 get_str(CdswEnvVar.CLOUDERA_HADOOP_ROOT): FileUtils.join_path(
-                    CommonDirs.USER_DEV_ROOT, CLOUDERA, HADOOP
+                    CommonDirsCdsw.USER_DEV_ROOT, CLOUDERA, HADOOP
                 ),
-                get_str(CdswEnvVar.HADOOP_DEV_DIR): FileUtils.join_path(CommonDirs.USER_DEV_ROOT, APACHE, HADOOP),
+                get_str(CdswEnvVar.HADOOP_DEV_DIR): FileUtils.join_path(CommonDirsCdsw.USER_DEV_ROOT, APACHE, HADOOP),
             },
             make_key(p_exec_mode, get_str(TestExecMode.UPSTREAM)): {
-                get_str(CdswEnvVar.HADOOP_DEV_DIR): FileUtils.join_path(CommonDirs.USER_DEV_ROOT, APACHE, HADOOP),
+                get_str(CdswEnvVar.HADOOP_DEV_DIR): FileUtils.join_path(CommonDirsCdsw.USER_DEV_ROOT, APACHE, HADOOP),
                 get_str(BranchComparatorEnvVar.BRANCH_COMP_REPO_TYPE): RepoType.UPSTREAM.value,
                 get_str(BranchComparatorEnvVar.BRANCH_COMP_FEATURE_BRANCH): ORIGIN_BRANCH_3_3,
                 get_str(BranchComparatorEnvVar.BRANCH_COMP_MASTER_BRANCH): ORIGIN_TRUNK,
@@ -259,7 +254,7 @@ def setup_docker_mounts(self):
         # Mount results dir so all output files will be available on the host machine
         mounts.append(
             DockerMount(
-                host_dir=LocalDirs.YARNDEVTOOLS_RESULT_DIR,
+                host_dir=LocalDirs.TEST_MODULE_RESULT_DIR,
                 container_dir=ContainerDirs.YARN_DEV_TOOLS_OUTPUT_DIR,
                 mode=DockerMountMode.READ_WRITE,
             )
@@ -386,9 +381,9 @@ def tearDown(self) -> None:
         self.docker_test_setup.cleanup()
 
     def save_latest_zip_from_container(self):
-        zip_link = FileUtils.join_path(LocalDirs.YARNDEVTOOLS_RESULT_DIR, "latest-command-data-zip")
+        zip_link = FileUtils.join_path(LocalDirs.TEST_MODULE_RESULT_DIR, "latest-command-data-zip")
         cont_src_path = os.readlink(zip_link)
-        local_target_path = FileUtils.join_path(LocalDirs.YARNDEVTOOLS_RESULT_DIR, "latest-command-data-real.zip")
+        local_target_path = FileUtils.join_path(LocalDirs.TEST_MODULE_RESULT_DIR, "latest-command-data-real.zip")
         self.docker_test_setup.docker_cp_from_container(cont_src_path, local_target_path)
 
     def copy_yarndevtools_cdsw_recursively(self):

tests/test_backporter.py

@@ -1,7 +1,6 @@
 import logging
 import unittest
 
-import pytest
 from pythoncommons.git_constants import ORIGIN
 
 from tests.test_utilities import TestUtilities, Object, SANDBOX_REPO_DOWNSTREAM_HOTFIX

tests/test_unit_test_result_fetcher.py

@@ -15,7 +15,6 @@
 
 import httpretty as httpretty
 import mongomock
-import pytest
 from coolname import generate_slug
 from pythoncommons.date_utils import DateUtils
 from pythoncommons.project_utils import ProjectUtils

yarndevtools/cdsw/cdsw_common.py

@@ -18,7 +18,6 @@
 LOG = logging.getLogger(__name__)
 
 
-
 class GenericCdswConfigUtils:
     @staticmethod
     def quote_list_items(lst):
@@ -120,6 +119,7 @@ def _auto_discover_skip_aggregation_result_file(cls, module_root):
 
 
 class JobPreparation:
+    # TODO cdsw-separation Better way to handle this?
     @staticmethod
     def execute(cdsw_runner: CdswRunner, job_config: CdswJobConfig, setup_result: CdswSetupResult):
         basedir = setup_result.basedir
@@ -144,4 +144,3 @@ def execute(cdsw_runner: CdswRunner, job_config: CdswJobConfig, setup_result: Cd
                 FileUtils.change_cwd(CommonDirs.HADOOP_CLOUDERA_BASEDIR)
                 os.system("git init")
                 cdsw_runner.execute_script("clone_upstream_repos.sh")
-

yarndevtools/cdsw/job_configs/branch_comparator_job_config.py

@@ -46,7 +46,7 @@ def generate_runs(conf):
 
 config = {
     "job_name": "Branch comparator",
-    "command_type": CommandType.BRANCH_COMPARATOR,
+    "command_type": CommandType.BRANCH_COMPARATOR.real_name,
     "env_sanitize_exceptions": [],
     "mandatory_env_vars": [
         "MAIL_ACC_USER",

yarndevtools/cdsw/job_configs/jira_umbrella_data_fetcher_job_config.py

@@ -34,7 +34,7 @@ def generate_runs(conf):
 
 config = {
     "job_name": "Jira umbrella data fetcher",
-    "command_type": CommandType.JIRA_UMBRELLA_DATA_FETCHER,
+    "command_type": CommandType.JIRA_UMBRELLA_DATA_FETCHER.real_name,
     "env_sanitize_exceptions": ["UMBRELLA_IDS"],
     "mandatory_env_vars": [
         "MAIL_ACC_USER",

yarndevtools/cdsw/job_configs/review_sheet_backport_updater_job_config.py

@@ -6,7 +6,7 @@
 
 config = {
     "job_name": "Review sheet backport updater",
-    "command_type": CommandType.REVIEW_SHEET_BACKPORT_UPDATER,
+    "command_type": CommandType.REVIEW_SHEET_BACKPORT_UPDATER.real_name,
     "env_sanitize_exceptions": ["BRANCHES"],
     "mandatory_env_vars": [
         "GSHEET_CLIENT_SECRET",

yarndevtools/cdsw/job_configs/reviewsync_job_config.py

@@ -6,7 +6,7 @@
 
 config = {
     "job_name": "Reviewsync",
-    "command_type": CommandType.REVIEWSYNC,
+    "command_type": CommandType.REVIEWSYNC.real_name,
     "env_sanitize_exceptions": ["BRANCHES"],
     "mandatory_env_vars": [
         "GSHEET_CLIENT_SECRET",

yarndevtools/cdsw/job_configs/unit_test_result_aggregator_email_job_config.py

@@ -11,7 +11,7 @@
 
 config = {
     "job_name": "Unit test result aggregator",
-    "command_type": CommandType.UNIT_TEST_RESULT_AGGREGATOR,
+    "command_type": CommandType.UNIT_TEST_RESULT_AGGREGATOR.real_name,
     "env_sanitize_exceptions": ["MATCH_EXPRESSION", "GSHEET_COMPARE_WITH_JIRA_TABLE", "AGGREGATE_FILTERS"],
     "mandatory_env_vars": [
         "GSHEET_CLIENT_SECRET",

yarndevtools/cdsw/job_configs/unit_test_result_fetcher_job_config.py

@@ -13,7 +13,7 @@
 
 config = {
     "job_name": "Unit test result fetcher",
-    "command_type": CommandType.UNIT_TEST_RESULT_FETCHER,
+    "command_type": CommandType.UNIT_TEST_RESULT_FETCHER.real_name,
     "env_sanitize_exceptions": [],
     "mandatory_env_vars": ["MAIL_ACC_USER", "MAIL_ACC_PASSWORD", "JENKINS_USER", "JENKINS_PASSWORD"],
     "optional_env_vars": ["BUILD_PROCESSING_LIMIT", "FORCE_SENDING_MAIL", "RESET_JOB_BUILD_DATA"],

yarndevtools/cdsw/start_job.py

@@ -6,10 +6,20 @@
 from cdswjoblauncher.cdsw.libreloader.reload_dependencies import Reloader
 from pythoncommons.file_utils import FileUtils
 
-from yarndevtools.cdsw.constants import BranchComparatorEnvVar, JiraUmbrellaFetcherEnvVar, UnitTestResultFetcherEnvVar, \
-    UnitTestResultAggregatorEmailEnvVar, ReviewSheetBackportUpdaterEnvVar, ReviewSyncEnvVar
+from yarndevtools.cdsw.constants import (
+    BranchComparatorEnvVar,
+    JiraUmbrellaFetcherEnvVar,
+    UnitTestResultFetcherEnvVar,
+    UnitTestResultAggregatorEmailEnvVar,
+    ReviewSheetBackportUpdaterEnvVar,
+    ReviewSyncEnvVar,
+)
 from yarndevtools.common.shared_command_utils import CommandType, YarnDevToolsEnvVar
-from yarndevtools.constants import YARNDEVTOOLS_MODULE_NAME, CDSW_JOB_LAUNCHER_MODULE_ROOT
+from yarndevtools.constants import (
+    YARNDEVTOOLS_MODULE_NAME,
+    CDSW_JOB_LAUNCHER_MODULE_ROOT,
+    YARNDEVTOOLS_MAIN_SCRIPT_NAME,
+)
 
 # THESE FUNCTION DEFINITIONS AND CALL TO fix_pythonpast MUST PRECEDE THE IMPORT OF libreloader: from libreloader import reload_dependencies
 # TODO same as CdswEnvVar.PYTHONPATH --> Migrate
@@ -51,7 +61,11 @@ def parse_args():
 
 
 class Config:
-    def __init__(self, parser, args,):
+    def __init__(
+        self,
+        parser,
+        args,
+    ):
         self._validate_args(parser, args)
         self.command_type = self._parse_command_type(args)
 
@@ -108,12 +122,12 @@ def prepare_args_for_cdsw_runner(config, valid_env_vars):
     append_arg_and_value("--command-type-valid-env-vars", " ".join(valid_env_vars))
     append_arg_and_value("--default-email-recipients", MAIL_ADDR_YARN_ENG_BP)
     append_arg_and_value("--module-name", YARNDEVTOOLS_MODULE_NAME)
-    append_arg_and_value("--main-script-name", "yarn_dev_tools.py")
+    append_arg_and_value("--main-script-name", YARNDEVTOOLS_MAIN_SCRIPT_NAME)
     append_arg_and_value("--job-preparation-callback", "JobPreparation.execute")
-    append_arg_and_value("--env",
-                         f"{YarnDevToolsEnvVar.ENV_CLOUDERA_HADOOP_ROOT.value}={CommonDirs.HADOOP_CLOUDERA_BASEDIR}")
-    append_arg_and_value("--env",
-                         f"{YarnDevToolsEnvVar.ENV_HADOOP_DEV_DIR.value}={CommonDirs.HADOOP_UPSTREAM_BASEDIR}")
+    append_arg_and_value(
+        "--env", f"{YarnDevToolsEnvVar.ENV_CLOUDERA_HADOOP_ROOT.value}={CommonDirs.HADOOP_CLOUDERA_BASEDIR}"
+    )
+    append_arg_and_value("--env", f"{YarnDevToolsEnvVar.ENV_HADOOP_DEV_DIR.value}={CommonDirs.HADOOP_UPSTREAM_BASEDIR}")
 
 
 def main():
@@ -131,5 +145,5 @@ def main():
     exec(open(cdsw_runner_path).read())
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

yarndevtools/commands/unittestresultaggregator/email/processor.py

@@ -2,14 +2,14 @@
 from pprint import pformat
 from typing import List, Iterable
 
+from cdswjoblauncher.cdsw.constants import SECRET_PROJECTS_DIR
 from googleapiwrapper.common import ServiceType
 from googleapiwrapper.gmail_api import GmailWrapper, ThreadQueryResults
 from googleapiwrapper.gmail_domain import GmailMessage
 from googleapiwrapper.google_auth import GoogleApiAuthorizer
 from googleapiwrapper.google_sheet import GSheetWrapper
 from pythoncommons.url_utils import UrlUtils
 
-from yarndevtools.cdsw.constants import SECRET_PROJECTS_DIR
 from yarndevtools.commands.unittestresultaggregator.common.aggregation import (
     AggregationResults,
 )

yarndevtools/commands/unittestresultaggregator/representation.py

@@ -5,6 +5,7 @@
 from enum import Enum
 from typing import Dict, List, Sized, Callable
 
+from cdswjoblauncher.cdsw.cdsw_common import ReportFile
 from googleapiwrapper.gmail_api import ThreadQueryResults
 from pythoncommons.file_utils import FileUtils
 from pythoncommons.html_utils import HtmlGenerator
@@ -32,9 +33,6 @@
 )
 from yarndevtools.commands.unittestresultaggregator.common.aggregation import AggregationResults
 from yarndevtools.commands.unittestresultaggregator.db.model import EmailContent
-from yarndevtools.constants import (
-    ReportFile,
-)
 
 LOG = logging.getLogger(__name__)
 

yarndevtools/commands/unittestresultfetcher/cache.py

@@ -8,6 +8,7 @@
 from enum import Enum
 from typing import Any, List, Dict, Set
 
+from cdswjoblauncher.cdsw.constants import SECRET_PROJECTS_DIR
 from googleapiwrapper.common import ServiceType
 from googleapiwrapper.google_auth import GoogleApiAuthorizer
 from googleapiwrapper.google_drive import (
@@ -26,7 +27,6 @@
 from pythoncommons.string_utils import StringUtils
 from pythoncommons.url_utils import UrlUtils
 
-from yarndevtools.cdsw.constants import SECRET_PROJECTS_DIR
 from yarndevtools.commands.unittestresultfetcher.common import (
     UnitTestResultFetcherMode,
     CACHED_DATA_DIRNAME,

yarndevtools/common/shared_command_utils.py

@@ -2,6 +2,7 @@
 from enum import Enum
 from typing import List, Dict
 
+from cdswjoblauncher.commands.cmd_type import LATEST_DATA_ZIP_LINK_NAME
 from pythoncommons.email import EmailAccount, EmailConfig
 from pythoncommons.file_utils import FileUtils
 from pythoncommons.git_constants import ORIGIN
@@ -19,12 +20,11 @@
     JiraIdChoosePreference,
 )
 
-from yarndevtools.constants import LATEST_DATA_ZIP_LINK_NAME, ANY_JIRA_ID_PATTERN
+from yarndevtools.constants import ANY_JIRA_ID_PATTERN
 
 LOG = logging.getLogger(__name__)
 
 
-
 class YarnDevToolsTestEnvVar(Enum):
     FORCE_COLLECTING_ARTIFACTS = "FORCE_COLLECTING_ARTIFACTS"
 

yarndevtools/constants.py

@@ -4,6 +4,7 @@
 PYTHON3 = "python3"
 REPO_ROOT_DIRNAME = "yarn-dev-tools"
 YARNDEVTOOLS_MODULE_NAME = "yarndevtools"
+YARNDEVTOOLS_MAIN_SCRIPT_NAME = "yarn_dev_tools.py"
 CDSW_JOB_LAUNCHER_MODULE_ROOT = "cdsw-job-launcher"
 APACHE = "apache"
 HADOOP = "hadoop"