Skip to content
This repository has been archived by the owner on Sep 20, 2024. It is now read-only.

Commit

Permalink
Merge pull request #2424 from pypeclub/feature/OP_2395_env_groups
Browse files Browse the repository at this point in the history
General: Environment variables groups
  • Loading branch information
iLLiCiTiT authored Dec 20, 2021
2 parents 9c39f8f + a61e3f7 commit fdcbd03
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 16 deletions.
7 changes: 5 additions & 2 deletions openpype/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,10 @@ def webpublisherwebserver(debug, executable, upload_dir, host=None, port=None):
@click.option("--asset", help="Asset name", default=None)
@click.option("--task", help="Task name", default=None)
@click.option("--app", help="Application name", default=None)
def extractenvironments(output_json_path, project, asset, task, app):
@click.option(
"--envgroup", help="Environment group (e.g. \"farm\")", default=None
)
def extractenvironments(output_json_path, project, asset, task, app, envgroup):
"""Extract environment variables for entered context to a json file.
Entered output filepath will be created if does not exists.
Expand All @@ -149,7 +152,7 @@ def extractenvironments(output_json_path, project, asset, task, app):
Context options are "project", "asset", "task", "app"
"""
PypeCommands.extractenvironments(
output_json_path, project, asset, task, app
output_json_path, project, asset, task, app, envgroup
)


Expand Down
2 changes: 1 addition & 1 deletion openpype/hooks/pre_global_host_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ def execute(self):
"log": self.log
})

prepare_host_environments(temp_data)
prepare_host_environments(temp_data, self.launch_context.env_group)
prepare_context_environments(temp_data)

temp_data.pop("log")
Expand Down
119 changes: 109 additions & 10 deletions openpype/lib/applications.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,97 @@

_logger = None

PLATFORM_NAMES = {"windows", "linux", "darwin"}
DEFAULT_ENV_SUBGROUP = "standard"


def parse_environments(env_data, env_group=None, platform_name=None):
"""Parse environment values from settings byt group and platfrom.
Data may contain up to 2 hierarchical levels of dictionaries. At the end
of the last level must be string or list. List is joined using platform
specific joiner (';' for windows and ':' for linux and mac).
Hierarchical levels can contain keys for subgroups and platform name.
Platform specific values must be always last level of dictionary. Platform
names are "windows" (MS Windows), "linux" (any linux distribution) and
"darwin" (any MacOS distribution).
Subgroups are helpers added mainly for standard and on farm usage. Farm
may require different environments for e.g. licence related values or
plugins. Default subgroup is "standard".
Examples:
```
{
# Unchanged value
"ENV_KEY1": "value",
# Empty values are kept (unset environment variable)
"ENV_KEY2": "",
# Join list values with ':' or ';'
"ENV_KEY3": ["value1", "value2"],
# Environment groups
"ENV_KEY4": {
"standard": "DEMO_SERVER_URL",
"farm": "LICENCE_SERVER_URL"
},
# Platform specific (and only for windows and mac)
"ENV_KEY5": {
"windows": "windows value",
"darwin": ["value 1", "value 2"]
},
# Environment groups and platform combination
"ENV_KEY6": {
"farm": "FARM_VALUE",
"standard": {
"windows": ["value1", "value2"],
"linux": "value1",
"darwin": ""
}
}
}
```
"""
output = {}
if not env_data:
return output

if not env_group:
env_group = DEFAULT_ENV_SUBGROUP

if not platform_name:
platform_name = platform.system().lower()

for key, value in env_data.items():
if isinstance(value, dict):
# Look if any key is platform key
# - expect that represents environment group if does not contain
# platform keys
if not PLATFORM_NAMES.intersection(set(value.keys())):
# Skip the key if group is not available
if env_group not in value:
continue
value = value[env_group]

# Check again if value is dictionary
# - this time there should be only platform keys
if isinstance(value, dict):
value = value.get(platform_name)

# Check if value is list and join it's values
# QUESTION Should empty values be skipped?
if isinstance(value, (list, tuple)):
value = os.pathsep.join(value)

# Set key to output if value is string
if isinstance(value, six.string_types):
output[key] = value
return output


def get_logger():
"""Global lib.applications logger getter."""
Expand Down Expand Up @@ -705,7 +796,7 @@ class ApplicationLaunchContext:
preparation to store objects usable in multiple places.
"""

def __init__(self, application, executable, **data):
def __init__(self, application, executable, env_group=None, **data):
from openpype.modules import ModulesManager

# Application object
Expand All @@ -719,6 +810,11 @@ def __init__(self, application, executable, **data):

self.executable = executable

if env_group is None:
env_group = DEFAULT_ENV_SUBGROUP

self.env_group = env_group

self.data = dict(data)

# subprocess.Popen launch arguments (first argument in constructor)
Expand Down Expand Up @@ -1052,7 +1148,7 @@ def __init__(self, data):


def get_app_environments_for_context(
project_name, asset_name, task_name, app_name, env=None
project_name, asset_name, task_name, app_name, env_group=None, env=None
):
"""Prepare environment variables by context.
Args:
Expand Down Expand Up @@ -1104,8 +1200,8 @@ def get_app_environments_for_context(
"env": env
})

prepare_host_environments(data)
prepare_context_environments(data)
prepare_host_environments(data, env_group)
prepare_context_environments(data, env_group)

# Discard avalon connection
dbcon.uninstall()
Expand All @@ -1125,7 +1221,7 @@ def _merge_env(env, current_env):
return result


def prepare_host_environments(data, implementation_envs=True):
def prepare_host_environments(data, env_group=None, implementation_envs=True):
"""Modify launch environments based on launched app and context.
Args:
Expand Down Expand Up @@ -1179,7 +1275,7 @@ def prepare_host_environments(data, implementation_envs=True):
continue

# Choose right platform
tool_env = acre.parse(_env_values)
tool_env = parse_environments(_env_values, env_group)
# Merge dictionaries
env_values = _merge_env(tool_env, env_values)

Expand Down Expand Up @@ -1211,7 +1307,9 @@ def prepare_host_environments(data, implementation_envs=True):
data["env"].pop(key, None)


def apply_project_environments_value(project_name, env, project_settings=None):
def apply_project_environments_value(
project_name, env, project_settings=None, env_group=None
):
"""Apply project specific environments on passed environments.
The enviornments are applied on passed `env` argument value so it is not
Expand Down Expand Up @@ -1239,14 +1337,15 @@ def apply_project_environments_value(project_name, env, project_settings=None):

env_value = project_settings["global"]["project_environments"]
if env_value:
parsed_value = parse_environments(env_value, env_group)
env.update(acre.compute(
_merge_env(acre.parse(env_value), env),
_merge_env(parsed_value, env),
cleanup=False
))
return env


def prepare_context_environments(data):
def prepare_context_environments(data, env_group=None):
"""Modify launch environemnts with context data for launched host.
Args:
Expand Down Expand Up @@ -1276,7 +1375,7 @@ def prepare_context_environments(data):
data["project_settings"] = project_settings
# Apply project specific environments on current env value
apply_project_environments_value(
project_name, data["env"], project_settings
project_name, data["env"], project_settings, env_group
)

app = data["app"]
Expand Down
9 changes: 6 additions & 3 deletions openpype/pype_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -305,13 +305,16 @@ def remotepublish(project, batch_path, user, targets=None):
log.info("Publish finished.")

@staticmethod
def extractenvironments(output_json_path, project, asset, task, app):
env = os.environ.copy()
def extractenvironments(
output_json_path, project, asset, task, app, env_group
):
if all((project, asset, task, app)):
from openpype.api import get_app_environments_for_context
env = get_app_environments_for_context(
project, asset, task, app, env
project, asset, task, app, env_group
)
else:
env = os.environ.copy()

output_dir = os.path.dirname(output_json_path)
if not os.path.exists(output_dir):
Expand Down
1 change: 1 addition & 0 deletions vendor/deadline/custom/plugins/GlobalJobPreLoad.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ def inject_openpype_environment(deadlinePlugin):
add_args['asset'] = job.GetJobEnvironmentKeyValue('AVALON_ASSET')
add_args['task'] = job.GetJobEnvironmentKeyValue('AVALON_TASK')
add_args['app'] = job.GetJobEnvironmentKeyValue('AVALON_APP_NAME')
add_args["envgroup"] = "farm"

if all(add_args.values()):
for key, value in add_args.items():
Expand Down

0 comments on commit fdcbd03

Please sign in to comment.