From de5a4ca0a184115efcbccbef5f5d3b64a4b964c1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 17 Dec 2021 17:48:25 +0100 Subject: [PATCH 1/4] added functin to parse environment data based on passed information --- openpype/lib/applications.py | 94 ++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index 6eb44a9694e..bbbd335a732 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -41,6 +41,100 @@ _logger = None +PLATFORM_NAMES = {"windows", "linux", "darwin"} +DEFAULT_ENV_SUBGROUP = "standard" + + +def parse_environments(env_data, env_group=None, platform_name=None): + """Parse environment values from settings byt group and platfrom. + + Data may contain up to 2 hierarchical levels of dictionaries. At the end + of the last level must be string or list. List is joined using platform + specific joiner (';' for windows and ':' for linux and mac). + + Hierarchical levels can contain keys for subgroups and platform name. + Platform specific values must be always last level of dictionary. Platform + names are "windows" (MS Windows), "linux" (any linux distribution) and + "darwin" (any MacOS distribution). + + Subgroups are helpers added mainly for standard and on farm usage. Farm + may require different environments for e.g. licence related values or + plugins. Default subgroup is "standard". + + Examples: + ``` + { + # Unchanged value + "ENV_KEY1": "value", + # Empty values are kept (unset environment variable) + "ENV_KEY2": "", + + # Join list values with ':' or ';' + "ENV_KEY3": ["value1", "value2"], + + # Environment groups + "ENV_KEY4": { + "standard": "DEMO_SERVER_URL", + "farm": "LICENCE_SERVER_URL" + }, + + # Platform specific (and only for windows and mac) + "ENV_KEY5": { + "windows": "windows value", + "darwin": ["value 1", "value 2"] + }, + + # Environment groups and platform combination + "ENV_KEY6": { + "farm": "FARM_VALUE", + "standard": { + "windows": ["value1", "value2"], + "linux": "value1", + "darwin": "" + } + } + } + ``` + + Args: + + """ + output = {} + if not env_data: + return output + + if not env_group: + env_group = DEFAULT_ENV_SUBGROUP + + if not platform_name: + platform_name = platform.system().lower() + + for key, value in env_data.items(): + if isinstance(value, dict): + # Look if any key is platform key + # - expect that represents environment group if does not contain + # platform keys + if not PLATFORM_NAMES.intersection(set(value.keys())): + # Skip the key if group is not available + if env_group not in value: + continue + value = value[env_group] + + # Check again if value is dictionary + # - this time there should be only platform keys + if isinstance(value, dict): + value = value.get(platform_name) + + # Check if value is list and join it's values + # QUESTION Should empty values be skipped? + if isinstance(value, (list, tuple)): + value = os.pathsep.join(value) + + # Set key to output if value is string + if isinstance(value, six.string_types): + output[key] = value + return output + def get_logger(): """Global lib.applications logger getter.""" From e3532074c50e9cf1e6f1342fa896f8201dff568a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 17 Dec 2021 17:49:51 +0100 Subject: [PATCH 2/4] environment group is part of all environment related functions and application launch context --- openpype/hooks/pre_global_host_data.py | 2 +- openpype/lib/applications.py | 28 +++++++++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/openpype/hooks/pre_global_host_data.py b/openpype/hooks/pre_global_host_data.py index b32fb5e44a2..6b08cdb444c 100644 --- a/openpype/hooks/pre_global_host_data.py +++ b/openpype/hooks/pre_global_host_data.py @@ -48,7 +48,7 @@ def execute(self): "log": self.log }) - prepare_host_environments(temp_data) + prepare_host_environments(temp_data, self.launch_context.env_group) prepare_context_environments(temp_data) temp_data.pop("log") diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index bbbd335a732..2e301adf03b 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -795,7 +795,7 @@ class ApplicationLaunchContext: preparation to store objects usable in multiple places. """ - def __init__(self, application, executable, **data): + def __init__(self, application, executable, env_group=None, **data): # Application object self.application = application @@ -805,6 +805,11 @@ def __init__(self, application, executable, **data): self.executable = executable + if env_group is None: + env_group = DEFAULT_ENV_SUBGROUP + + self.env_group = env_group + self.data = dict(data) # subprocess.Popen launch arguments (first argument in constructor) @@ -1141,7 +1146,7 @@ def __init__(self, data): def get_app_environments_for_context( - project_name, asset_name, task_name, app_name, env=None + project_name, asset_name, task_name, app_name, env_group=None, env=None ): """Prepare environment variables by context. Args: @@ -1193,8 +1198,8 @@ def get_app_environments_for_context( "env": env }) - prepare_host_environments(data) - prepare_context_environments(data) + prepare_host_environments(data, env_group) + prepare_context_environments(data, env_group) # Discard avalon connection dbcon.uninstall() @@ -1214,7 +1219,7 @@ def _merge_env(env, current_env): return result -def prepare_host_environments(data, implementation_envs=True): +def prepare_host_environments(data, env_group=None, implementation_envs=True): """Modify launch environments based on launched app and context. Args: @@ -1268,7 +1273,7 @@ def prepare_host_environments(data, implementation_envs=True): continue # Choose right platform - tool_env = acre.parse(_env_values) + tool_env = parse_environments(_env_values, env_group) # Merge dictionaries env_values = _merge_env(tool_env, env_values) @@ -1300,7 +1305,9 @@ def prepare_host_environments(data, implementation_envs=True): data["env"].pop(key, None) -def apply_project_environments_value(project_name, env, project_settings=None): +def apply_project_environments_value( + project_name, env, project_settings=None, env_group=None +): """Apply project specific environments on passed environments. The enviornments are applied on passed `env` argument value so it is not @@ -1328,14 +1335,15 @@ def apply_project_environments_value(project_name, env, project_settings=None): env_value = project_settings["global"]["project_environments"] if env_value: + parsed_value = parse_environments(env_value, env_group) env.update(acre.compute( - _merge_env(acre.parse(env_value), env), + _merge_env(parsed_value, env), cleanup=False )) return env -def prepare_context_environments(data): +def prepare_context_environments(data, env_group=None): """Modify launch environemnts with context data for launched host. Args: @@ -1365,7 +1373,7 @@ def prepare_context_environments(data): data["project_settings"] = project_settings # Apply project specific environments on current env value apply_project_environments_value( - project_name, data["env"], project_settings + project_name, data["env"], project_settings, env_group ) app = data["app"] From 4de80689ed7666cfd87866d591872e2052accf1e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 17 Dec 2021 17:54:02 +0100 Subject: [PATCH 3/4] added environment group to extractenvironments command line argument --- openpype/cli.py | 7 +++++-- openpype/lib/applications.py | 3 --- openpype/pype_commands.py | 9 ++++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index 6b20fb52037..6e9c237b0e1 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -138,7 +138,10 @@ def webpublisherwebserver(debug, executable, upload_dir, host=None, port=None): @click.option("--asset", help="Asset name", default=None) @click.option("--task", help="Task name", default=None) @click.option("--app", help="Application name", default=None) -def extractenvironments(output_json_path, project, asset, task, app): +@click.option( + "--envgroup", help="Environment group (e.g. \"farm\")", default=None +) +def extractenvironments(output_json_path, project, asset, task, app, envgroup): """Extract environment variables for entered context to a json file. Entered output filepath will be created if does not exists. @@ -149,7 +152,7 @@ def extractenvironments(output_json_path, project, asset, task, app): Context options are "project", "asset", "task", "app" """ PypeCommands.extractenvironments( - output_json_path, project, asset, task, app + output_json_path, project, asset, task, app, envgroup ) diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index 2e301adf03b..bf52daba7c3 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -95,9 +95,6 @@ def parse_environments(env_data, env_group=None, platform_name=None): } } ``` - - Args: - """ output = {} if not env_data: diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index a6330bae1fd..e25b56744e8 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -305,13 +305,16 @@ def remotepublish(project, batch_path, user, targets=None): log.info("Publish finished.") @staticmethod - def extractenvironments(output_json_path, project, asset, task, app): - env = os.environ.copy() + def extractenvironments( + output_json_path, project, asset, task, app, env_group + ): if all((project, asset, task, app)): from openpype.api import get_app_environments_for_context env = get_app_environments_for_context( - project, asset, task, app, env + project, asset, task, app, env_group ) + else: + env = os.environ.copy() output_dir = os.path.dirname(output_json_path) if not os.path.exists(output_dir): From eb465b4d266c444193dcfe83f09693ba7e817abf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 17 Dec 2021 17:55:39 +0100 Subject: [PATCH 4/4] deadline global plugin is using new argument "envgroup" --- vendor/deadline/custom/plugins/GlobalJobPreLoad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vendor/deadline/custom/plugins/GlobalJobPreLoad.py b/vendor/deadline/custom/plugins/GlobalJobPreLoad.py index 0aa5adaa209..ba1e5f6c6a5 100644 --- a/vendor/deadline/custom/plugins/GlobalJobPreLoad.py +++ b/vendor/deadline/custom/plugins/GlobalJobPreLoad.py @@ -48,6 +48,7 @@ def inject_openpype_environment(deadlinePlugin): add_args['asset'] = job.GetJobEnvironmentKeyValue('AVALON_ASSET') add_args['task'] = job.GetJobEnvironmentKeyValue('AVALON_TASK') add_args['app'] = job.GetJobEnvironmentKeyValue('AVALON_APP_NAME') + add_args["envgroup"] = "farm" if all(add_args.values()): for key, value in add_args.items():