Skip to content

Commit

Permalink
Moving cdsw from yarndevtools: Fixing YarnCdswBranchDiffTests
Browse files Browse the repository at this point in the history
Make yarndevtools branch and module name configurable
  • Loading branch information
szilard-nemeth committed Aug 2, 2024
1 parent b028ff4 commit ad0ce35
Show file tree
Hide file tree
Showing 6 changed files with 93 additions and 37 deletions.
80 changes: 47 additions & 33 deletions tests/cdsw/test_branch_comparator.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from pythoncommons.constants import ExecutionMode
from pythoncommons.docker_wrapper import DockerTestSetup, CreatePathMode, DockerMountMode, DockerMount
from pythoncommons.file_utils import FileUtils, FindResultType
from pythoncommons.git_wrapper import GitWrapper
from pythoncommons.github_utils import GitHubUtils
from pythoncommons.logging_setup import SimpleLoggingSetupConfig, SimpleLoggingSetup
from pythoncommons.object_utils import ObjUtils
Expand All @@ -24,7 +25,7 @@
from cdswjoblauncher.cdsw.cdsw_common import CommonDirs as CommonDirsCdsw
from yarndevtools.cdsw.constants import BranchComparatorEnvVar
from yarndevtools.cdsw.start_job import CommonDirs
from yarndevtools.common.shared_command_utils import RepoType, CommandType
from yarndevtools.common.shared_command_utils import RepoType, CommandType, YarnDevToolsEnvVar
from yarndevtools.constants import (
ORIGIN_BRANCH_3_3,
ORIGIN_TRUNK,
Expand All @@ -33,6 +34,7 @@
HADOOP,
CLOUDERA,
PYTHON3,
REPO_ROOT_DIRNAME,
)

PROJECT_NAME = "branch-comparator"
Expand Down Expand Up @@ -93,7 +95,7 @@ def __init__(
self.python_module_root = None
self.exec_mode: TestExecMode = self.determine_execution_mode()
self.python_module_mode_query_cmd = self.determine_python_module_mode_query_command()
self.cdsw_testing_commons = CdswTestingCommons()
self.cdsw_testing_commons = CdswTestingCommons(YARNDEVTOOLS_MODULE_NAME)
if self.github_ci_execution:
self.mount_cdsw_dirs_from_local = False
self.env_dict = self.setup_env_vars()
Expand Down Expand Up @@ -139,6 +141,9 @@ def get_str(key):
def make_key(prefix, conf_value):
return f"{prefix}_{conf_value}"

basedir = FileUtils.find_repo_root_dir(__file__, REPO_ROOT_DIRNAME, raise_error=False)
repo_wrapper = GitWrapper(basedir)

p_common = "common"
p_exec_mode = "exec_mode"
p_module_mode = "module_mode"
Expand All @@ -153,6 +158,8 @@ def make_key(prefix, conf_value):
get_str(CdswEnvVar.RESTART_PROCESS_WHEN_REQUIREMENTS_INSTALLED): False,
# TODO Investigate this later to check why number of loggers are not correct
get_str(CdswEnvVar.ENABLE_LOGGER_HANDLER_SANITY_CHECK): False,
# TODO cdsw-separation Maybe this works differently on Github actions CI
get_str(YarnDevToolsEnvVar.YARNDEVTOOLS_BRANCH): repo_wrapper.get_current_branch_name(),
},
# !! WARNING: User-specific settings below !!
make_key(p_exec_mode, get_str(TestExecMode.CLOUDERA)): {
Expand Down Expand Up @@ -212,6 +219,8 @@ def make_key(prefix, conf_value):
)

env_dict = {env_name: OsUtils.get_env_value(env_name) for env_name in env_keys}
env_dict = {k: v for k, v in env_dict.items() if v is not None}

return env_dict

@staticmethod
Expand Down Expand Up @@ -280,44 +289,19 @@ def setup_docker_mounts(self):
return mounts


PROD_CONFIG = DockerBasedTestConfig(
create_image=True,
mount_cdsw_dirs_from_local=False,
run_cdsw_initial_setup_script=True,
container_sleep_seconds=400,
install_requirements=True,
)
DEV_CONFIG = DockerBasedTestConfig(
create_image=False,
mount_cdsw_dirs_from_local=True,
run_cdsw_initial_setup_script=False,
container_sleep_seconds=1000,
install_requirements=False,
)
QUICK_DEV_CONFIG = DockerBasedTestConfig(
create_image=False,
mount_cdsw_dirs_from_local=True,
run_cdsw_initial_setup_script=False,
container_sleep_seconds=1000,
install_requirements=False,
)
ACTIVE_CONFIG = PROD_CONFIG # <-- !!! CHANGE THE ACTIVE CONFIG HERE !!!


class YarnCdswBranchDiffTests(unittest.TestCase):
docker_test_setup = None
config: DockerBasedTestConfig = ACTIVE_CONFIG
PROD_CONFIG = None
DEV_CONFIG = None
QUICK_DEV_CONFIG = None
config: DockerBasedTestConfig

@classmethod
def setUpClass(cls):
if GitHubUtils.is_github_ci_execution():
# Always use PROD config when GitHub CI is executed
LOG.info("Changing configuration to PROD as Github Actions CI is being executed...")
YarnCdswBranchDiffTests.config = PROD_CONFIG

ProjectUtils.set_root_determine_strategy(ProjectRootDeterminationStrategy.COMMON_FILE)
ProjectUtils.get_test_output_basedir(YARNDEVTOOLS_MODULE_NAME)
cls._setup_logging()
cls.config = cls.setup_test_configs()
cwd = os.getcwd()
if cwd != LocalDirs.CDSW_ROOT_DIR:
os.chdir(LocalDirs.CDSW_ROOT_DIR)
Expand All @@ -326,6 +310,36 @@ def setUpClass(cls):
)
cls.setup_default_docker_mounts()

@classmethod
def setup_test_configs(cls):
YarnCdswBranchDiffTests.PROD_CONFIG = DockerBasedTestConfig(
create_image=True,
mount_cdsw_dirs_from_local=False,
run_cdsw_initial_setup_script=True,
container_sleep_seconds=400,
install_requirements=True,
)
YarnCdswBranchDiffTests.DEV_CONFIG = DockerBasedTestConfig(
create_image=False,
mount_cdsw_dirs_from_local=True,
run_cdsw_initial_setup_script=False,
container_sleep_seconds=1000,
install_requirements=False,
)
YarnCdswBranchDiffTests.QUICK_DEV_CONFIG = DockerBasedTestConfig(
create_image=False,
mount_cdsw_dirs_from_local=True,
run_cdsw_initial_setup_script=False,
container_sleep_seconds=1000,
install_requirements=False,
)
if GitHubUtils.is_github_ci_execution():
# Always use PROD config when GitHub CI is executed
LOG.info("Changing configuration to PROD as Github Actions CI is being executed...")
return YarnCdswBranchDiffTests.PROD_CONFIG
else:
return YarnCdswBranchDiffTests.PROD_CONFIG # <-- !!! CHANGE THE ACTIVE CONFIG HERE !!!

@classmethod
def tearDownClass(cls) -> None:
OsUtils.clear_env_vars(list(cls.config.env_dict.keys()))
Expand Down Expand Up @@ -411,7 +425,7 @@ def _callback(cmd, cmd_output, docker_setup):
self.setup_default_docker_mounts()
self.docker_test_setup.run_container(sleep=self.config.container_sleep_seconds)
self.exec_get_python_module_root(callback=_callback)
self.exec_initial_cdsw_setup_script()
self.exec_initial_cdsw_setup_script(env=self.config.env_dict)
if self.config.mount_cdsw_dirs_from_local:
# TODO Copy python-commons, google-api-wrapper as well, control this with an enum
self.copy_yarndevtools_cdsw_recursively()
Expand Down
3 changes: 2 additions & 1 deletion yarndevtools/cdsw/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ __pycache__
.idea/
*.pyc
venv
/yarndevtools-results/
/yarndevtools-results/
/testmodule-results/
10 changes: 9 additions & 1 deletion yarndevtools/cdsw/scripts/initial-cdsw-setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,14 @@ elif [[ "$1" == "cloudera" ]]; then
shift
fi

branch="master"
if [[ ! -z "$YARNDEVTOOLS_BRANCH" ]]; then
echo "Recognized YARNDEVTOOLS_BRANCH=$YARNDEVTOOLS_BRANCH"
branch=$YARNDEVTOOLS_BRANCH
fi

echo "Using branch to checkout after cloning yarndevtools: $branch"

#Validations

(( "$PYTHON_MODULE_MODE" != "global" || "$PYTHON_MODULE_MODE" != "user" )) && echo "Python module mode should be either 'user' or 'global'!" && exit 1
Expand All @@ -46,7 +54,7 @@ cd $REPOS_ROOT

set +e
YARNDEVTOOLS_REPO_NAME="yarn-dev-tools"
git -C $YARNDEVTOOLS_REPO_NAME pull || git clone https://github.com/szilard-nemeth/yarn-dev-tools.git $YARNDEVTOOLS_REPO_NAME
git -C $YARNDEVTOOLS_REPO_NAME pull || git clone https://github.com/szilard-nemeth/yarn-dev-tools.git $YARNDEVTOOLS_REPO_NAME --branch $branch

CDSW_ROOT="/home/cdsw/"
SCRIPTS_ROOT="$CDSW_ROOT/scripts"
Expand Down
13 changes: 12 additions & 1 deletion yarndevtools/cdsw/scripts/install-requirements.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,17 @@
#!/bin/bash
set -x

# Latest version
yarndevtools_pip_version_spec="yarn-dev-tools"
if [[ ! -z "$YARNDEVTOOLS_MODULE_VERSION" ]]; then
echo "Recognized YARNDEVTOOLS_MODULE_VERSION=$YARNDEVTOOLS_MODULE_VERSION"
yarndevtools_pip_version_spec="yarn-dev-tools==$YARNDEVTOOLS_MODULE_VERSION"
fi

echo "Using module version for yarndevtools: $yarndevtools_pip_version_spec"



echo "Uninstalling package: yarn-dev-tools"
set +e
pip3 -V
Expand All @@ -18,5 +29,5 @@ fi

EXEC_MODE="$1"
echo "Installing package: yarn-dev-tools"
pip3 install yarn-dev-tools --force-reinstall
pip3 install $yarndevtools_pip_version_spec --force-reinstall
pip3 show yarn-dev-tools
22 changes: 21 additions & 1 deletion yarndevtools/cdsw/start_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from cdswjoblauncher.cdsw.libreloader.reload_dependencies import Reloader
from pythoncommons.file_utils import FileUtils
from pythoncommons.os_utils import OsUtils

from yarndevtools.cdsw.constants import (
BranchComparatorEnvVar,
Expand All @@ -22,7 +23,7 @@
)

# THESE FUNCTION DEFINITIONS AND CALL TO fix_pythonpast MUST PRECEDE THE IMPORT OF libreloader: from libreloader import reload_dependencies
# TODO same as CdswEnvVar.PYTHONPATH --> Migrate
# TODO cdsw-separation same as CdswEnvVar.PYTHONPATH --> Migrate
PYTHONPATH_ENV_VAR = "PYTHONPATH"
MAIL_ADDR_YARN_ENG_BP = "yarn_eng_bp@cloudera.com"
POSSIBLE_COMMAND_TYPES = [e.real_name for e in CommandType] + [e.output_dir_name for e in CommandType]
Expand Down Expand Up @@ -129,6 +130,24 @@ def prepare_args_for_cdsw_runner(config, valid_env_vars):
)
append_arg_and_value("--env", f"{YarnDevToolsEnvVar.ENV_HADOOP_DEV_DIR.value}={CommonDirs.HADOOP_UPSTREAM_BASEDIR}")

# Set module version if yarndevtools branch is defined.
# initial-cdsw-setup.sh and install-requirements.sh should be in sync for yarndevtools version
if YarnDevToolsEnvVar.YARNDEVTOOLS_BRANCH.value in os.environ:
branch = OsUtils.get_env_value(YarnDevToolsEnvVar.YARNDEVTOOLS_BRANCH.value, default_value=None)
if branch:
# TODO cdsw-separation ugly as hell :(
version = os.system(
'wget -q -O - https://raw.githubusercontent.com/szilard-nemeth/yarn-dev-tools/master/pyproject.toml | grep -A2 "name = "yarn-dev-tools"" | grep -m 1 version | tr -s '
" | tr -d "
"' | tr -d "
'" | cut -d'
" -f3"
)
OsUtils.set_env_value(YarnDevToolsEnvVar.YARNDEVTOOLS_MODULE_VERSION.value, version)

append_arg_and_value("--env", f"{YarnDevToolsEnvVar.YARNDEVTOOLS_BRANCH.value}={branch}")
append_arg_and_value("--env", f"{YarnDevToolsEnvVar.YARNDEVTOOLS_MODULE_VERSION.value}={version}")


def main():
module_root = Reloader.get_python_module_root()
Expand All @@ -142,6 +161,7 @@ def main():

# Start the CDSW runner
prepare_args_for_cdsw_runner(config, valid_env_vars)
print("Arguments for CDSW runner: " + str(sys.argv))
exec(open(cdsw_runner_path).read())


Expand Down
2 changes: 2 additions & 0 deletions yarndevtools/common/shared_command_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ class YarnDevToolsEnvVar(Enum):
PROJECT_DETERMINATION_STRATEGY = "PROJECT_DETERMINATION_STRATEGY"
ENV_CLOUDERA_HADOOP_ROOT = "CLOUDERA_HADOOP_ROOT"
ENV_HADOOP_DEV_DIR = "HADOOP_DEV_DIR"
YARNDEVTOOLS_BRANCH = "YARNDEVTOOLS_BRANCH"
YARNDEVTOOLS_MODULE_VERSION = "YARNDEVTOOLS_MODULE_VERSION"


class RepoType(Enum):
Expand Down

0 comments on commit ad0ce35

Please sign in to comment.