Merge pull request #1115 from pypeclub/sync_server_fix_local_drive

Sync server fix local drive

pype/lib/anatomy.py

@@ -77,7 +77,7 @@ class Anatomy:
  root_key_regex = re.compile(r"{(root?[^}]+)}")
  root_name_regex = re.compile(r"root\[([^]]+)\]")
 
- def __init__(self, project_name=None):
+ def __init__(self, project_name=None, site_name=None):
  if not project_name:
  project_name = os.environ.get("AVALON_PROJECT")
 
@@ -89,7 +89,7 @@ def __init__(self, project_name=None):
 
  self.project_name = project_name
 
- self._data = get_anatomy_settings(project_name)
+ self._data = get_anatomy_settings(project_name, site_name)
 
  self._templates_obj = Templates(self)
  self._roots_obj = Roots(self)

pype/modules/sync_server/providers/abstract_provider.py

@@ -71,3 +71,39 @@ def list_folder(self, folder_path):
  (list)
  """
  pass
+
+ @abstractmethod
+ def create_folder(self, folder_path):
+ """
+ Create all nonexistent folders and subfolders in 'path'.
+
+ Args:
+ path (string): absolute path
+
+ Returns:
+ (string) folder id of lowest subfolder from 'path'
+ """
+ pass
+
+ @abstractmethod
+ def get_tree(self):
+ """
+ Creates folder structure for providers which do not provide
+ tree folder structure (GDrive has no accessible tree structure,
+ only parents and their parents)
+ """
+ pass
+
+ @abstractmethod
+ def resolve_path(self, path, root_config, anatomy=None):
+ """
+ Replaces root placeholders with appropriate real value from
+ 'root_configs' (from Settings or Local Settings) or Anatomy
+ (mainly for 'studio' site)
+
+ Args:
+ path(string): path with '{root[work]}/...'
+ root_config(dict): from Settings or Local Settings
+ anatomy (Anatomy): prepared anatomy object for project
+ """
+ pass

pype/modules/sync_server/providers/gdrive.py

@@ -678,6 +678,16 @@ def get_presets(cls):
  return
  return provider_presets
 
+ def resolve_path(self, path, root_config, anatomy=None):
+ if not root_config.get("root"):
+ root_config = {"root": root_config}
+
+ try:
+ return path.format(**root_config)
+ except KeyError:
+ msg = "Error in resolving remote root, unknown key"
+ log.error(msg)
+
  def _handle_q(self, q, trashed=False):
  """ API list call contain trashed and hidden files/folder by default.
  Usually we dont want those, must be included in query explicitly.

pype/modules/sync_server/providers/local_drive.py

@@ -1,6 +1,8 @@
 from __future__ import print_function
 import os.path
 import shutil
+import threading
+import time
 
 from pype.api import Logger
 from .abstract_provider import AbstractProvider
@@ -13,29 +15,37 @@ class LocalDriveHandler(AbstractProvider):
  def is_active(self):
  return True
 
- def upload_file(self, source_path, target_path, overwrite=True):
+ def upload_file(self, source_path, target_path,
+ server, collection, file, representation, site,
+ overwrite=False, direction="Upload"):
  """
  Copies file from 'source_path' to 'target_path'
  """
- if os.path.exists(source_path):
- if overwrite:
- shutil.copy(source_path, target_path)
- else:
- if os.path.exists(target_path):
- raise ValueError("File {} exists, set overwrite".
- format(target_path))
+ if not os.path.isfile(source_path):
+ raise FileNotFoundError("Source file {} doesn't exist."
+ .format(source_path))
+ if overwrite:
+ thread = threading.Thread(target=self._copy,
+ args=(source_path, target_path))
+ thread.start()
+ self._mark_progress(collection, file, representation, server,
+ site, source_path, target_path, direction)
+ else:
+ if os.path.exists(target_path):
+ raise ValueError("File {} exists, set overwrite".
+ format(target_path))
 
- def download_file(self, source_path, local_path, overwrite=True):
+ return os.path.basename(target_path)
+
+ def download_file(self, source_path, local_path,
+ server, collection, file, representation, site,
+ overwrite=False):
  """
  Download a file form 'source_path' to 'local_path'
  """
- if os.path.exists(source_path):
- if overwrite:
- shutil.copy(source_path, local_path)
- else:
- if os.path.exists(local_path):
- raise ValueError("File {} exists, set overwrite".
- format(local_path))
+ return self.upload_file(source_path, local_path,
+ server, collection, file, representation, site,
+ overwrite, direction="Download")
 
  def delete_file(self, path):
  """
@@ -57,3 +67,69 @@ def list_folder(self, folder_path):
  lst.append(os.path.join(dir_path, name))
 
  return lst
+
+ def create_folder(self, folder_path):
+ """
+ Creates 'folder_path' on local system
+
+ Args:
+ folder_path (string): absolute path on local (and mounted) disk
+
+ Returns:
+ (string) - sends back folder_path to denote folder(s) was
+ created
+ """
+ os.makedirs(folder_path, exist_ok=True)
+ return folder_path
+
+ def get_tree(self):
+ return
+
+ def resolve_path(self, path, root_config, anatomy=None):
+ if root_config and not root_config.get("root"):
+ root_config = {"root": root_config}
+
+ try:
+ if not root_config:
+ raise KeyError
+
+ path = path.format(**root_config)
+ except KeyError:
+ try:
+ path = anatomy.fill_root(path)
+ except KeyError:
+ msg = "Error in resolving local root from anatomy"
+ log.error(msg)
+ raise ValueError(msg)
+
+ return path
+
+ def _copy(self, source_path, target_path):
+ print("copying {}->{}".format(source_path, target_path))
+ shutil.copy(source_path, target_path)
+
+ def _mark_progress(self, collection, file, representation, server, site,
+ source_path, target_path, direction):
+ """
+ Updates progress field in DB by values 0-1.
+
+ Compares file sizes of source and target.
+ """
+ source_file_size = os.path.getsize(source_path)
+ target_file_size = 0
+ last_tick = status_val = None
+ while source_file_size != target_file_size:
+ if not last_tick or \
+ time.time() - last_tick >= server.LOG_PROGRESS_SEC:
+ status_val = target_file_size / source_file_size
+ last_tick = time.time()
+ log.debug(direction + "ed %d%%." % int(status_val * 100))
+ server.update_db(collection=collection,
+ new_file_id=None,
+ file=file,
+ representation=representation,
+ site=site,
+ progress=status_val
+ )
+ target_file_size = os.path.getsize(target_path)
+ time.sleep(0.5)