Add command line way of running site sync server

openpype/cli.py

@@ -317,3 +317,28 @@ def run(script):
 def runtests(folder, mark, pyargs):
  """Run all automatic tests after proper initialization via start.py"""
  PypeCommands().run_tests(folder, mark, pyargs)
+
+
+@main.command()
+@click.option("-d", "--debug",
+ is_flag=True, help=("Run process in debug mode"))
+@click.option("-a", "--active_site", required=True,
+ help="Name of active stie")
+def syncserver(debug, active_site):
+ """Run sync site server in background.
+
+ Some Site Sync use cases need to expose site to another one.
+ For example if majority of artists work in studio, they are not using
+ SS at all, but if you want to expose published assets to 'studio' site
+ to SFTP for only a couple of artists, some background process must
+ mark published assets to live on multiple sites (they might be
+ physically in same location - mounted shared disk).
+
+ Process mimics OP Tray with specific 'active_site' name, all
+ configuration for this "dummy" user comes from Setting or Local
+ Settings (configured by starting OP Tray with env
+ var SITE_SYNC_LOCAL_ID set to 'active_site'.
+ """
+ if debug:
+ os.environ['OPENPYPE_DEBUG'] = '3'
+ PypeCommands().syncserver(active_site)

openpype/lib/local_settings.py

@@ -522,6 +522,11 @@ def get_local_site_id():
 
  Identifier is created if does not exists yet.
  """
+ # override local id from environment
+ # used for background syncing
+ if os.environ.get("SITE_SYNC_LOCAL_ID"):
+ return os.environ["SITE_SYNC_LOCAL_ID"]
+
  registry = OpenPypeSettingsRegistry()
  try:
  return registry.get_item("localId")

openpype/modules/default_modules/sync_server/providers/local_drive.py

@@ -84,6 +84,7 @@ def upload_file(self, source_path, target_path,
  if not os.path.isfile(source_path):
  raise FileNotFoundError("Source file {} doesn't exist."
  .format(source_path))
+
  if overwrite:
  thread = threading.Thread(target=self._copy,
  args=(source_path, target_path))
@@ -176,7 +177,10 @@ def get_configurable_items_for_site(self):
 
  def _copy(self, source_path, target_path):
  print("copying {}->{}".format(source_path, target_path))
- shutil.copy(source_path, target_path)
+ try:
+ shutil.copy(source_path, target_path)
+ except shutil.SameFileError:
+ print("same files, skipping")
 
  def _mark_progress(self, collection, file, representation, server, site,
  source_path, target_path, direction):

openpype/modules/default_modules/sync_server/sync_server.py

@@ -246,6 +246,7 @@ def run(self):
 
  asyncio.ensure_future(self.check_shutdown(), loop=self.loop)
  asyncio.ensure_future(self.sync_loop(), loop=self.loop)
+ log.info("Sync Server Started")
  self.loop.run_forever()
  except Exception:
  log.warning(

openpype/modules/default_modules/sync_server/sync_server_module.py

@@ -146,9 +146,9 @@ def add_site(self, collection, representation_id, site_name=None,
  if not site_name:
  site_name = self.DEFAULT_SITE
 
- self.reset_provider_for_file(collection,
- representation_id,
- site_name=site_name, force=force)
+ self.reset_site_on_representation(collection,
+  representation_id,
+  site_name=site_name, force=force)
 
  # public facing API
  def remove_site(self, collection, representation_id, site_name,
@@ -170,10 +170,10 @@ def remove_site(self, collection, representation_id, site_name,
  if not self.get_sync_project_setting(collection):
  raise ValueError("Project not configured")
 
- self.reset_provider_for_file(collection,
- representation_id,
- site_name=site_name,
- remove=True)
+ self.reset_site_on_representation(collection,
+  representation_id,
+  site_name=site_name,
+  remove=True)
  if remove_local_files:
  self._remove_local_file(collection, representation_id, site_name)
 
@@ -209,8 +209,8 @@ def pause_representation(self, collection, representation_id, site_name):
  """
  log.info("Pausing SyncServer for {}".format(representation_id))
  self._paused_representations.add(representation_id)
- self.reset_provider_for_file(collection, representation_id,
- site_name=site_name, pause=True)
+ self.reset_site_on_representation(collection, representation_id,
+  site_name=site_name, pause=True)
 
  def unpause_representation(self, collection, representation_id, site_name):
  """
@@ -229,8 +229,8 @@ def unpause_representation(self, collection, representation_id, site_name):
  except KeyError:
  pass
  # self.paused_representations is not persistent
- self.reset_provider_for_file(collection, representation_id,
- site_name=site_name, pause=False)
+ self.reset_site_on_representation(collection, representation_id,
+  site_name=site_name, pause=False)
 
  def is_representation_paused(self, representation_id,
  check_parents=False, project_name=None):
@@ -694,12 +694,19 @@ def _get_enabled_sites_from_settings(self, sync_settings):
 
  def tray_init(self):
  """
- Actual initialization of Sync Server.
+ Actual initialization of Sync Server for Tray.
 
  Called when tray is initialized, it checks if module should be
  enabled. If not, no initialization necessary.
  """
- # import only in tray, because of Python2 hosts
+ self.server_init()
+
+ from .tray.app import SyncServerWindow
+ self.widget = SyncServerWindow(self)
+
+ def server_init(self):
+ """Actual initialization of Sync Server."""
+ # import only in tray or Python3, because of Python2 hosts
  from .sync_server import SyncServerThread
 
  if not self.enabled:
@@ -715,17 +722,15 @@ def tray_init(self):
  try:
  self.sync_server_thread = SyncServerThread(self)
 
- from .tray.app import SyncServerWindow
- self.widget = SyncServerWindow(self)
  except ValueError:
  log.info("No system setting for sync. Not syncing.", exc_info=True)
  self.enabled = False
  except KeyError:
  log.info((
- "There are not set presets for SyncServer OR "
- "Credentials provided are invalid, "
- "no syncing possible").
- format(str(self.sync_project_settings)), exc_info=True)
+  "There are not set presets for SyncServer OR "
+  "Credentials provided are invalid, "
+  "no syncing possible").
+  format(str(self.sync_project_settings)), exc_info=True)
  self.enabled = False
 
  def tray_start(self):
@@ -739,6 +744,9 @@ def tray_start(self):
  Returns:
  None
  """
+ self.server_start()
+
+ def server_start(self):
  if self.sync_project_settings and self.enabled:
  self.sync_server_thread.start()
  else:
@@ -751,6 +759,9 @@ def tray_exit(self):
 
  Called from Module Manager
  """
+ self.server_exit()
+
+ def server_exit(self):
  if not self.sync_server_thread:
  return
 
@@ -760,6 +771,7 @@ def tray_exit(self):
  log.info("Stopping sync server server")
  self.sync_server_thread.is_running = False
  self.sync_server_thread.stop()
+ log.info("Sync server stopped")
  except Exception:
  log.warning(
  "Error has happened during Killing sync server",
@@ -1229,9 +1241,9 @@ def _get_site_rec(self, sites, site_name):
 
  return -1, None
 
- def reset_provider_for_file(self, collection, representation_id,
- side=None, file_id=None, site_name=None,
- remove=False, pause=None, force=False):
+ def reset_site_on_representation(self, collection, representation_id,
+  side=None, file_id=None, site_name=None,
+  remove=False, pause=None, force=False):
  """
  Reset information about synchronization for particular 'file_id'
  and provider.

openpype/modules/default_modules/sync_server/tray/widgets.py

@@ -411,7 +411,7 @@ def _reset_site(self, selected_ids=None, site_name=None):
  format(check_progress))
  continue
 
- self.sync_server.reset_provider_for_file(
+ self.sync_server.reset_site_on_representation(
  self.model.project,
  representation_id,
  site_name=site_name,
@@ -786,7 +786,7 @@ def _reset_site(self, selected_ids=None, site_name=None):
  format(check_progress))
  continue
 
- self.sync_server.reset_provider_for_file(
+ self.sync_server.reset_site_on_representation(
  self.model.project,
  self.representation_id,
  site_name=site_name,

openpype/plugins/publish/integrate_new.py

@@ -1028,6 +1028,7 @@ def prepare_file_info(self, path, size=None, file_hash=None,
  """
  local_site = 'studio' # default
  remote_site = None
+ always_accesible = []
  sync_server_presets = None
 
  if (instance.context.data["system_settings"]
@@ -1042,6 +1043,8 @@ def prepare_file_info(self, path, size=None, file_hash=None,
  if sync_server_presets["enabled"]:
  local_site = sync_server_presets["config"].\
  get("active_site", "studio").strip()
+ always_accesible = sync_server_presets["config"].\
+ get("always_accessible_on", [])
  if local_site == 'local':
  local_site = local_site_id
 
@@ -1072,6 +1075,12 @@ def prepare_file_info(self, path, size=None, file_hash=None,
  meta = {"name": remote_site.strip()}
  rec["sites"].append(meta)
 
+ # add skeleton for site where it should be always synced to
+ for always_on_site in always_accesible:
+ if always_on_site not in [local_site, remote_site]:
+ meta = {"name": always_on_site.strip()}
+ rec["sites"].append(meta)
+
  return rec
 
  def handle_destination_files(self, integrated_file_sizes, mode):

openpype/pype_commands.py

@@ -3,7 +3,6 @@
 import os
 import sys
 import json
-from datetime import datetime
 import time
 
 from openpype.lib import PypeLogger
@@ -329,3 +328,28 @@ def run_tests(self, folder, mark, pyargs):
  cmd = "pytest {} {} {}".format(folder, mark_str, pyargs_str)
  print("Running {}".format(cmd))
  subprocess.run(cmd)
+
+ def syncserver(self, active_site):
+ """Start running sync_server in background."""
+ import signal
+ os.environ["SITE_SYNC_LOCAL_ID"] = active_site
+
+ def signal_handler(sig, frame):
+ print("You pressed Ctrl+C. Process ended.")
+ sync_server_module.server_exit()
+ sys.exit(0)
+
+ signal.signal(signal.SIGINT, signal_handler)
+ signal.signal(signal.SIGTERM, signal_handler)
+
+ from openpype.modules import ModulesManager
+
+ manager = ModulesManager()
+ sync_server_module = manager.modules_by_name["sync_server"]
+
+ sync_server_module.server_init()
+ sync_server_module.server_start()
+
+ import time
+ while True:
+ time.sleep(1.0)

openpype/settings/defaults/project_settings/global.json

@@ -319,6 +319,7 @@
  "config": {
  "retry_cnt": "3",
  "loop_delay": "60",
+ "always_accessible_on": [],
  "active_site": "studio",
  "remote_site": "studio"
  },

openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json

@@ -26,15 +26,24 @@
  "key": "loop_delay",
  "label": "Loop Delay"
  },
+ {
+ "type": "list",
+ "key": "always_accessible_on",
+ "label": "Always accessible on sites",
+ "object_type": "text"
+ },
+ {
+ "type": "splitter"
+ },
  {
  "type": "text",
  "key": "active_site",
- "label": "Active Site"
+ "label": "User Default Active Site"
  },
  {
  "type": "text",
  "key": "remote_site",
- "label": "Remote Site"
+ "label": "User Default Remote Site"
  }
  ]
  },

website/docs/module_site_sync.md

@@ -140,3 +140,42 @@ Beware that ssh key expects OpenSSH format (`.pem`) not a Putty format (`.ppk`)!
 
 If a studio needs to use other services for cloud storage, or want to implement totally different storage providers, they can do so by writing their own provider plugin. We're working on a developer documentation, however, for now we recommend looking at `abstract_provider.py`and `gdrive.py` inside `openpype/modules/sync_server/providers` and using it as a template.
 
+### Running Site Sync in background
+
+Site Sync server synchronizes new published files from artist machine into configured remote location by default.
+
+There might be a use case where you need to synchronize between "non-artist" sites, for example between studio site and cloud. In this case
+you need to run Site Sync as a background process from a command line (via service etc) 24/7.
+
+To configure all sites where all published files should be synced eventually you need to configure `project_settings/global/sync_server/config/always_accessible_on` property in Settins (per project) first.
+
+![Set another non artist remote site](assets/site_sync_always_on.png)
+
+This is an example of:
+- Site Sync is enabled for a project
+- default active and remote sites are set to `studio` - eg. standard process: everyone is working in a studio, publishing to shared location etc.
+- (but this also allows any of the artists to work remotely, they would change their active site in their own Local Settings to `local` and configure local root.
+ This would result in everything artist publishes is saved first onto his local folder AND synchronized to `studio` site eventually.)
+- everything exported must also be eventually uploaded to `sftp` site
+
+This eventual synchronization between `studio` and `sftp` sites must be physically handled by background process.
+
+As current implementation relies heavily on Settings and Local Settings, background process for a specific site ('studio' for example) must be configured via Tray first to `syncserver` command to work.
+
+To do this:
+
+- run OP `Tray` with environment variable SITE_SYNC_LOCAL_ID set to name of active (source) site. In most use cases it would be studio (for cases of backups of everything published to studio site to different cloud site etc.)
+- start `Tray`
+- check `Local ID` in information dialog after clicking on version number in the Tray
+- open `Local Settings` in the `Tray`
+- configure for each project necessary active site and remote site
+- close `Tray`
+- run OP from a command line with `syncserver` and `--active_site` arguments
+
+
+This is an example how to trigger background synching process where active (source) site is `studio`. 
+(It is expected that OP is installed on a machine, `openpype_console` is on PATH. If not, add full path to executable.
+)
+```shell
+openpype_console syncserver --active_site studio
+```