Use new site sync entity in settings and modify their loading

openpype/cli.py

@@ -346,3 +346,28 @@ def run(script):
 def runtests(folder, mark, pyargs):
  """Run all automatic tests after proper initialization via start.py"""
  PypeCommands().run_tests(folder, mark, pyargs)
+
+
+@main.command()
+@click.option("-d", "--debug",
+ is_flag=True, help=("Run process in debug mode"))
+@click.option("-a", "--active_site", required=True,
+ help="Name of active stie")
+def syncserver(debug, active_site):
+ """Run sync site server in background.
+
+ Some Site Sync use cases need to expose site to another one.
+ For example if majority of artists work in studio, they are not using
+ SS at all, but if you want to expose published assets to 'studio' site
+ to SFTP for only a couple of artists, some background process must
+ mark published assets to live on multiple sites (they might be
+ physically in same location - mounted shared disk).
+
+ Process mimics OP Tray with specific 'active_site' name, all
+ configuration for this "dummy" user comes from Setting or Local
+ Settings (configured by starting OP Tray with env
+ var OPENPYPE_LOCAL_ID set to 'active_site'.
+ """
+ if debug:
+ os.environ['OPENPYPE_DEBUG'] = '3'
+ PypeCommands().syncserver(active_site)

openpype/lib/local_settings.py

@@ -522,6 +522,11 @@ def get_local_site_id():
 
  Identifier is created if does not exists yet.
  """
+ # override local id from environment
+ # used for background syncing
+ if os.environ.get("OPENPYPE_LOCAL_ID"):
+ return os.environ["OPENPYPE_LOCAL_ID"]
+
  registry = OpenPypeSettingsRegistry()
  try:
  return registry.get_item("localId")

openpype/modules/default_modules/sync_server/providers/dropbox.py

@@ -24,25 +24,19 @@ def __init__(self, project_name, site_name, tree=None, presets=None):
  )
  return
 
- provider_presets = self.presets.get(self.CODE)
- if not provider_presets:
- msg = "Sync Server: No provider presets for {}".format(self.CODE)
- log.info(msg)
- return
-
- token = self.presets[self.CODE].get("token", "")
+ token = self.presets.get("token", "")
  if not token:
  msg = "Sync Server: No access token for dropbox provider"
  log.info(msg)
  return
 
- team_folder_name = self.presets[self.CODE].get("team_folder_name", "")
+ team_folder_name = self.presets.get("team_folder_name", "")
  if not team_folder_name:
  msg = "Sync Server: No team folder name for dropbox provider"
  log.info(msg)
  return
 
- acting_as_member = self.presets[self.CODE].get("acting_as_member", "")
+ acting_as_member = self.presets.get("acting_as_member", "")
  if not acting_as_member:
  msg = (
  "Sync Server: No acting member for dropbox provider"
@@ -51,13 +45,15 @@ def __init__(self, project_name, site_name, tree=None, presets=None):
  return
 
  self.dbx = None
- try:
- self.dbx = self._get_service(
- token, acting_as_member, team_folder_name
- )
- except Exception as e:
- log.info("Could not establish dropbox object: {}".format(e))
- return
+
+ if self.presets["enabled"]:
+ try:
+ self.dbx = self._get_service(
+ token, acting_as_member, team_folder_name
+ )
+ except Exception as e:
+ log.info("Could not establish dropbox object: {}".format(e))
+ return
 
  super(AbstractProvider, self).__init__()
 
@@ -106,7 +102,7 @@ def get_project_settings_schema(cls):
  "type": "dict-roots",
  "object_type": {
  "type": "path",
- "multiplatform": True,
+ "multiplatform": False,
  "multipath": False
  }
  }
@@ -169,7 +165,7 @@ def is_active(self):
  Returns:
  (boolean)
  """
- return self.dbx is not None
+ return self.presets["enabled"] and self.dbx is not None
 
  @classmethod
  def get_configurable_items(cls):
@@ -393,7 +389,7 @@ def get_roots_config(self, anatomy=None):
  {"root": {"root_ONE": "value", "root_TWO":"value}}
  Format is importing for usage of python's format ** approach
  """
- return self.presets['root']
+ return self.presets['roots']
 
  def resolve_path(self, path, root_config=None, anatomy=None):
  """

openpype/modules/default_modules/sync_server/providers/gdrive.py

@@ -73,32 +73,28 @@ def __init__(self, project_name, site_name, tree=None, presets=None):
  format(site_name))
  return
 
- provider_presets = self.presets.get(self.CODE)
- if not provider_presets:
- msg = "Sync Server: No provider presets for {}".format(self.CODE)
- log.info(msg)
- return
-
- cred_path = self.presets[self.CODE].get("credentials_url", {}).\
+ cred_path = self.presets.get("credentials_url", {}).\
  get(platform.system().lower()) or ''
  if not os.path.exists(cred_path):
  msg = "Sync Server: No credentials for gdrive provider " + \
  "for '{}' on path '{}'!".format(site_name, cred_path)
  log.info(msg)
  return
 
- self.service = self._get_gd_service(cred_path)
+ self.service = None
+ if self.presets["enabled"]:
+ self.service = self._get_gd_service(cred_path)
 
- self._tree = tree
- self.active = True
+  self._tree = tree
+  self.active = True
 
  def is_active(self):
  """
  Returns True if provider is activated, eg. has working credentials.
  Returns:
  (boolean)
  """
- return self.service is not None
+ return self.presets["enabled"] and self.service is not None
 
  @classmethod
  def get_system_settings_schema(cls):
@@ -125,9 +121,11 @@ def get_project_settings_schema(cls):
  editable = [
  # credentials could be overriden on Project or User level
  {
- 'key': "credentials_url",
- 'label': "Credentials url",
- 'type': 'text'
+ "type": "path",
+ "key": "credentials_url",
+ "label": "Credentials url",
+ "multiplatform": True,
+ "placeholder": "Credentials url"
  },
  # roots could be overriden only on Project leve, User cannot
  {
@@ -136,7 +134,7 @@ def get_project_settings_schema(cls):
  "type": "dict-roots",
  "object_type": {
  "type": "path",
- "multiplatform": True,
+ "multiplatform": False,
  "multipath": False
  }
  }
@@ -176,7 +174,7 @@ def get_roots_config(self, anatomy=None):
  Format is importing for usage of python's format ** approach
  """
  # GDrive roots cannot be locally overridden
- return self.presets['root']
+ return self.presets['roots']
 
  def get_tree(self):
  """

openpype/modules/default_modules/sync_server/providers/local_drive.py

@@ -89,6 +89,7 @@ def upload_file(self, source_path, target_path,
  if not os.path.isfile(source_path):
  raise FileNotFoundError("Source file {} doesn't exist."
  .format(source_path))
+
  if overwrite:
  thread = threading.Thread(target=self._copy,
  args=(source_path, target_path))
@@ -181,7 +182,10 @@ def get_configurable_items_for_site(self):
 
  def _copy(self, source_path, target_path):
  print("copying {}->{}".format(source_path, target_path))
- shutil.copy(source_path, target_path)
+ try:
+ shutil.copy(source_path, target_path)
+ except shutil.SameFileError:
+ print("same files, skipping")
 
  def _mark_progress(self, collection, file, representation, server, site,
  source_path, target_path, direction):

openpype/modules/default_modules/sync_server/providers/sftp.py

@@ -1,8 +1,6 @@
 import os
 import os.path
 import time
-import sys
-import six
 import threading
 import platform
 
@@ -14,6 +12,7 @@
 pysftp = None
 try:
  import pysftp
+ import paramiko
 except (ImportError, SyntaxError):
  pass
 
@@ -37,7 +36,6 @@ class SFTPHandler(AbstractProvider):
 
  def __init__(self, project_name, site_name, tree=None, presets=None):
  self.presets = None
- self.active = False
  self.project_name = project_name
  self.site_name = site_name
  self.root = None
@@ -49,22 +47,15 @@ def __init__(self, project_name, site_name, tree=None, presets=None):
  format(site_name))
  return
 
- provider_presets = self.presets.get(self.CODE)
- if not provider_presets:
- msg = "Sync Server: No provider presets for {}".format(self.CODE)
- log.warning(msg)
- return
-
  # store to instance for reconnect
- self.sftp_host = provider_presets["sftp_host"]
- self.sftp_port = provider_presets["sftp_port"]
- self.sftp_user = provider_presets["sftp_user"]
- self.sftp_pass = provider_presets["sftp_pass"]
- self.sftp_key = provider_presets["sftp_key"]
- self.sftp_key_pass = provider_presets["sftp_key_pass"]
+ self.sftp_host = presets["sftp_host"]
+ self.sftp_port = presets["sftp_port"]
+ self.sftp_user = presets["sftp_user"]
+ self.sftp_pass = presets["sftp_pass"]
+ self.sftp_key = presets["sftp_key"]
+ self.sftp_key_pass = presets["sftp_key_pass"]
 
  self._tree = None
- self.active = True
 
  @property
  def conn(self):
@@ -80,7 +71,7 @@ def is_active(self):
  Returns:
  (boolean)
  """
- return self.conn is not None
+ return self.presets["enabled"] and self.conn is not None
 
  @classmethod
  def get_system_settings_schema(cls):
@@ -108,7 +99,7 @@ def get_project_settings_schema(cls):
  editable = [
  # credentials could be overriden on Project or User level
  {
- 'key': "sftp_server",
+ 'key': "sftp_host",
  'label': "SFTP host name",
  'type': 'text'
  },
@@ -130,7 +121,8 @@ def get_project_settings_schema(cls):
  {
  'key': "sftp_key",
  'label': "SFTP user ssh key",
- 'type': 'path'
+ 'type': 'path',
+ "multiplatform": True
  },
  {
  'key': "sftp_key_pass",
@@ -144,7 +136,7 @@ def get_project_settings_schema(cls):
  "type": "dict-roots",
  "object_type": {
  "type": "path",
- "multiplatform": True,
+ "multiplatform": False,
  "multipath": False
  }
  }
@@ -176,7 +168,8 @@ def get_local_settings_schema(cls):
  {
  'key': "sftp_key",
  'label': "SFTP user ssh key",
- 'type': 'path'
+ 'type': 'path',
+ "multiplatform": True
  },
  {
  'key': "sftp_key_pass",
@@ -199,7 +192,7 @@ def get_roots_config(self, anatomy=None):
  Format is importing for usage of python's format ** approach
  """
  # roots cannot be locally overridden
- return self.presets['root']
+ return self.presets['roots']
 
  def get_tree(self):
  """
@@ -426,7 +419,10 @@ def _get_conn(self):
  if self.sftp_key_pass:
  conn_params['private_key_pass'] = self.sftp_key_pass
 
- return pysftp.Connection(**conn_params)
+ try:
+ return pysftp.Connection(**conn_params)
+ except paramiko.ssh_exception.SSHException:
+ log.warning("Couldn't connect", exc_info=True)
 
  def _mark_progress(self, collection, file, representation, server, site,
  source_path, target_path, direction):

openpype/modules/default_modules/sync_server/sync_server.py

@@ -80,6 +80,10 @@ async def upload(module, collection, file, representation, provider_name,
  remote_site_name,
  True
  )
+
+ module.handle_alternate_site(collection, representation, remote_site_name,
+ file["_id"], file_id)
+
  return file_id
 
 
@@ -131,6 +135,10 @@ async def download(module, collection, file, representation, provider_name,
  local_site,
  True
  )
+
+ module.handle_alternate_site(collection, representation, local_site,
+ file["_id"], file_id)
+
  return file_id
 
 
@@ -246,6 +254,7 @@ def run(self):
 
  asyncio.ensure_future(self.check_shutdown(), loop=self.loop)
  asyncio.ensure_future(self.sync_loop(), loop=self.loop)
+ log.info("Sync Server Started")
  self.loop.run_forever()
  except Exception:
  log.warning(