General: Environment variables groups

openpype/cli.py

@@ -138,7 +138,10 @@ def webpublisherwebserver(debug, executable, upload_dir, host=None, port=None):
 @click.option("--asset", help="Asset name", default=None)
 @click.option("--task", help="Task name", default=None)
 @click.option("--app", help="Application name", default=None)
-def extractenvironments(output_json_path, project, asset, task, app):
+@click.option(
+ "--envgroup", help="Environment group (e.g. \"farm\")", default=None
+)
+def extractenvironments(output_json_path, project, asset, task, app, envgroup):
  """Extract environment variables for entered context to a json file.
 
  Entered output filepath will be created if does not exists.
@@ -149,7 +152,7 @@ def extractenvironments(output_json_path, project, asset, task, app):
  Context options are "project", "asset", "task", "app"
  """
  PypeCommands.extractenvironments(
- output_json_path, project, asset, task, app
+ output_json_path, project, asset, task, app, envgroup
  )
 
 

openpype/hooks/pre_global_host_data.py

@@ -48,7 +48,7 @@ def execute(self):
  "log": self.log
  })
 
- prepare_host_environments(temp_data)
+ prepare_host_environments(temp_data, self.launch_context.env_group)
  prepare_context_environments(temp_data)
 
  temp_data.pop("log")

openpype/lib/applications.py

@@ -41,6 +41,97 @@
 
 _logger = None
 
+PLATFORM_NAMES = {"windows", "linux", "darwin"}
+DEFAULT_ENV_SUBGROUP = "standard"
+
+
+def parse_environments(env_data, env_group=None, platform_name=None):
+ """Parse environment values from settings byt group and platfrom.
+
+ Data may contain up to 2 hierarchical levels of dictionaries. At the end
+ of the last level must be string or list. List is joined using platform
+ specific joiner (';' for windows and ':' for linux and mac).
+
+ Hierarchical levels can contain keys for subgroups and platform name.
+ Platform specific values must be always last level of dictionary. Platform
+ names are "windows" (MS Windows), "linux" (any linux distribution) and
+ "darwin" (any MacOS distribution).
+
+ Subgroups are helpers added mainly for standard and on farm usage. Farm
+ may require different environments for e.g. licence related values or
+ plugins. Default subgroup is "standard".
+
+ Examples:
+ ```
+ {
+ # Unchanged value
+ "ENV_KEY1": "value",
+ # Empty values are kept (unset environment variable)
+ "ENV_KEY2": "",
+
+ # Join list values with ':' or ';'
+ "ENV_KEY3": ["value1", "value2"],
+
+ # Environment groups
+ "ENV_KEY4": {
+ "standard": "DEMO_SERVER_URL",
+ "farm": "LICENCE_SERVER_URL"
+ },
+
+ # Platform specific (and only for windows and mac)
+ "ENV_KEY5": {
+ "windows": "windows value",
+ "darwin": ["value 1", "value 2"]
+ },
+
+ # Environment groups and platform combination
+ "ENV_KEY6": {
+ "farm": "FARM_VALUE",
+ "standard": {
+ "windows": ["value1", "value2"],
+ "linux": "value1",
+ "darwin": ""
+ }
+ }
+ }
+ ```
+ """
+ output = {}
+ if not env_data:
+ return output
+
+ if not env_group:
+ env_group = DEFAULT_ENV_SUBGROUP
+
+ if not platform_name:
+ platform_name = platform.system().lower()
+
+ for key, value in env_data.items():
+ if isinstance(value, dict):
+ # Look if any key is platform key
+ # - expect that represents environment group if does not contain
+ # platform keys
+ if not PLATFORM_NAMES.intersection(set(value.keys())):
+ # Skip the key if group is not available
+ if env_group not in value:
+ continue
+ value = value[env_group]
+
+ # Check again if value is dictionary
+ # - this time there should be only platform keys
+ if isinstance(value, dict):
+ value = value.get(platform_name)
+
+ # Check if value is list and join it's values
+ # QUESTION Should empty values be skipped?
+ if isinstance(value, (list, tuple)):
+ value = os.pathsep.join(value)
+
+ # Set key to output if value is string
+ if isinstance(value, six.string_types):
+ output[key] = value
+ return output
+
 
 def get_logger():
  """Global lib.applications logger getter."""
@@ -701,7 +792,7 @@ class ApplicationLaunchContext:
  preparation to store objects usable in multiple places.
  """
 
- def __init__(self, application, executable, **data):
+ def __init__(self, application, executable, env_group=None, **data):
  # Application object
  self.application = application
 
@@ -711,6 +802,11 @@ def __init__(self, application, executable, **data):
 
  self.executable = executable
 
+ if env_group is None:
+ env_group = DEFAULT_ENV_SUBGROUP
+
+ self.env_group = env_group
+
  self.data = dict(data)
 
  # subprocess.Popen launch arguments (first argument in constructor)
@@ -1047,7 +1143,7 @@ def __init__(self, data):
 
 
 def get_app_environments_for_context(
- project_name, asset_name, task_name, app_name, env=None
+ project_name, asset_name, task_name, app_name, env_group=None, env=None
 ):
  """Prepare environment variables by context.
  Args:
@@ -1099,8 +1195,8 @@ def get_app_environments_for_context(
  "env": env
  })
 
- prepare_host_environments(data)
- prepare_context_environments(data)
+ prepare_host_environments(data, env_group)
+ prepare_context_environments(data, env_group)
 
  # Discard avalon connection
  dbcon.uninstall()
@@ -1120,7 +1216,7 @@ def _merge_env(env, current_env):
  return result
 
 
-def prepare_host_environments(data, implementation_envs=True):
+def prepare_host_environments(data, env_group=None, implementation_envs=True):
  """Modify launch environments based on launched app and context.
 
  Args:
@@ -1174,7 +1270,7 @@ def prepare_host_environments(data, implementation_envs=True):
  continue
 
  # Choose right platform
- tool_env = acre.parse(_env_values)
+ tool_env = parse_environments(_env_values, env_group)
  # Merge dictionaries
  env_values = _merge_env(tool_env, env_values)
 
@@ -1206,7 +1302,9 @@ def prepare_host_environments(data, implementation_envs=True):
  data["env"].pop(key, None)
 
 
-def apply_project_environments_value(project_name, env, project_settings=None):
+def apply_project_environments_value(
+ project_name, env, project_settings=None, env_group=None
+):
  """Apply project specific environments on passed environments.
 
  The enviornments are applied on passed `env` argument value so it is not
@@ -1234,14 +1332,15 @@ def apply_project_environments_value(project_name, env, project_settings=None):
 
  env_value = project_settings["global"]["project_environments"]
  if env_value:
+ parsed_value = parse_environments(env_value, env_group)
  env.update(acre.compute(
- _merge_env(acre.parse(env_value), env),
+ _merge_env(parsed_value, env),
  cleanup=False
  ))
  return env
 
 
-def prepare_context_environments(data):
+def prepare_context_environments(data, env_group=None):
  """Modify launch environemnts with context data for launched host.
 
  Args:
@@ -1271,7 +1370,7 @@ def prepare_context_environments(data):
  data["project_settings"] = project_settings
  # Apply project specific environments on current env value
  apply_project_environments_value(
- project_name, data["env"], project_settings
+ project_name, data["env"], project_settings, env_group
  )
 
  app = data["app"]

openpype/pype_commands.py

@@ -305,13 +305,16 @@ def remotepublish(project, batch_path, user, targets=None):
  log.info("Publish finished.")
 
  @staticmethod
- def extractenvironments(output_json_path, project, asset, task, app):
- env = os.environ.copy()
+ def extractenvironments(
+ output_json_path, project, asset, task, app, env_group
+ ):
  if all((project, asset, task, app)):
  from openpype.api import get_app_environments_for_context
  env = get_app_environments_for_context(
- project, asset, task, app, env
+ project, asset, task, app, env_group
  )
+ else:
+ env = os.environ.copy()
 
  output_dir = os.path.dirname(output_json_path)
  if not os.path.exists(output_dir):

vendor/deadline/custom/plugins/GlobalJobPreLoad.py

@@ -48,6 +48,7 @@ def inject_openpype_environment(deadlinePlugin):
  add_args['asset'] = job.GetJobEnvironmentKeyValue('AVALON_ASSET')
  add_args['task'] = job.GetJobEnvironmentKeyValue('AVALON_TASK')
  add_args['app'] = job.GetJobEnvironmentKeyValue('AVALON_APP_NAME')
+ add_args["envgroup"] = "farm"
 
  if all(add_args.values()):
  for key, value in add_args.items():