From 96c2cc0f9099474fc1664d85710b95e36193ac52 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 14:00:16 +0200 Subject: [PATCH 01/24] OP-1920 - create command for background running of Site Sync server WIP --- openpype/cli.py | 22 +++++++++++++++++++ .../sync_server/sync_server_module.py | 20 ++++++++++++----- openpype/pype_commands.py | 13 +++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index c69407e2955..a98ba8d1773 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -298,3 +298,25 @@ def run(script): def runtests(folder, mark, pyargs): """Run all automatic tests after proper initialization via start.py""" PypeCommands().run_tests(folder, mark, pyargs) + + +@main.command() +@click.option("-d", "--debug", + is_flag=True, help=("Run process in debug mode")) +@click.option("-a", "--active_site", required=True, + help="Name of active stie") +@click.option("-r", "--remote_site", required=True, + help="Name of remote site") +def syncsiteserver(debug, active_site, remote_site): + """Run sync site server in background. + + Some Site Sync use cases need to expose site to another one. + For example if majority of artists work in studio, they are not using + SS at all, but if you want to expose published assets to 'studio' site + to SFTP for only a couple of artists, some background process must + mark published assets to live on multiple sites (they might be + physically in same location - mounted shared disk). + """ + if debug: + os.environ['OPENPYPE_DEBUG'] = '3' + PypeCommands().syncsiteserver(active_site, remote_site) \ No newline at end of file diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index f2e92375426..4bec6267449 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -699,7 +699,11 @@ def tray_init(self): Called when tray is initialized, it checks if module should be enabled. If not, no initialization necessary. """ - # import only in tray, because of Python2 hosts + self.server_init() + + def server_init(self): + """Actual initialization of Sync Server.""" + # import only in tray or Python3, because of Python2 hosts from .sync_server import SyncServerThread if not self.enabled: @@ -722,10 +726,10 @@ def tray_init(self): self.enabled = False except KeyError: log.info(( - "There are not set presets for SyncServer OR " - "Credentials provided are invalid, " - "no syncing possible"). - format(str(self.sync_project_settings)), exc_info=True) + "There are not set presets for SyncServer OR " + "Credentials provided are invalid, " + "no syncing possible"). + format(str(self.sync_project_settings)), exc_info=True) self.enabled = False def tray_start(self): @@ -739,6 +743,9 @@ def tray_start(self): Returns: None """ + self.server_start() + + def server_start(self): if self.sync_project_settings and self.enabled: self.sync_server_thread.start() else: @@ -751,6 +758,9 @@ def tray_exit(self): Called from Module Manager """ + self.server_exit() + + def server_exit(self): if not self.sync_server_thread: return diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 5288749e8bd..0a897e43e41 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -284,3 +284,16 @@ def run_tests(self, folder, mark, pyargs): cmd = "pytest {} {} {}".format(folder, mark_str, pyargs_str) print("Running {}".format(cmd)) subprocess.run(cmd) + + def syncsiteserver(self, active_site, remote_site): + from openpype.modules import ModulesManager + + manager = ModulesManager() + sync_server_module = manager.modules_by_name["sync_server"] + + sync_server_module.init_server() + sync_server_module.start_server() + + import time + while True: + time.sleep(1.0) From 19b5d47b2494498d19701e1441ed1b0a05aa34a8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 16:46:45 +0200 Subject: [PATCH 02/24] OP-1920 - skip upload/download for same files --- .../default_modules/sync_server/providers/local_drive.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/sync_server/providers/local_drive.py b/openpype/modules/default_modules/sync_server/providers/local_drive.py index 8e5f170bc9f..2961a07cdde 100644 --- a/openpype/modules/default_modules/sync_server/providers/local_drive.py +++ b/openpype/modules/default_modules/sync_server/providers/local_drive.py @@ -84,6 +84,7 @@ def upload_file(self, source_path, target_path, if not os.path.isfile(source_path): raise FileNotFoundError("Source file {} doesn't exist." .format(source_path)) + if overwrite: thread = threading.Thread(target=self._copy, args=(source_path, target_path)) @@ -176,7 +177,10 @@ def get_configurable_items_for_site(self): def _copy(self, source_path, target_path): print("copying {}->{}".format(source_path, target_path)) - shutil.copy(source_path, target_path) + try: + shutil.copy(source_path, target_path) + except shutil.SameFileError: + print("same files, skipping") def _mark_progress(self, collection, file, representation, server, site, source_path, target_path, direction): From 421ac7716ea004c3738aa1c5ceb00518d05aa94d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 16:48:04 +0200 Subject: [PATCH 03/24] OP-1920 - override get_local_site_id from env var --- openpype/lib/local_settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openpype/lib/local_settings.py b/openpype/lib/local_settings.py index 66dad279de8..af8c3cdbc83 100644 --- a/openpype/lib/local_settings.py +++ b/openpype/lib/local_settings.py @@ -522,6 +522,11 @@ def get_local_site_id(): Identifier is created if does not exists yet. """ + # override local id from environment + # used for background syncing + if os.environ.get("SITE_SYNC_LOCAL_ID"): + return os.environ["SITE_SYNC_LOCAL_ID"] + registry = OpenPypeSettingsRegistry() try: return registry.get_item("localId") From 02b78dc7f8ec7a5c55e270156fef31f0f33ec8ff Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 16:55:11 +0200 Subject: [PATCH 04/24] OP-1920 - added syncserver command --- openpype/cli.py | 11 +++++++---- openpype/pype_commands.py | 7 +++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index a98ba8d1773..583fd6daacf 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -305,9 +305,7 @@ def runtests(folder, mark, pyargs): is_flag=True, help=("Run process in debug mode")) @click.option("-a", "--active_site", required=True, help="Name of active stie") -@click.option("-r", "--remote_site", required=True, - help="Name of remote site") -def syncsiteserver(debug, active_site, remote_site): +def syncserver(debug, active_site): """Run sync site server in background. Some Site Sync use cases need to expose site to another one. @@ -316,7 +314,12 @@ def syncsiteserver(debug, active_site, remote_site): to SFTP for only a couple of artists, some background process must mark published assets to live on multiple sites (they might be physically in same location - mounted shared disk). + + Process mimics OP Tray with specific 'active_site' name, all + configuration for this "dummy" user comes from Setting or Local + Settings (configured by starting OP Tray with env + var SITE_SYNC_LOCAL_ID set to 'active_site'. """ if debug: os.environ['OPENPYPE_DEBUG'] = '3' - PypeCommands().syncsiteserver(active_site, remote_site) \ No newline at end of file + PypeCommands().syncserver(active_site) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 0a897e43e41..ed3fc5996bf 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -285,13 +285,16 @@ def run_tests(self, folder, mark, pyargs): print("Running {}".format(cmd)) subprocess.run(cmd) - def syncsiteserver(self, active_site, remote_site): + def syncserver(self, active_site): + """Start running sync_server in background.""" + os.environ["SITE_SYNC_LOCAL_ID"] = active_site + from openpype.modules import ModulesManager manager = ModulesManager() sync_server_module = manager.modules_by_name["sync_server"] - sync_server_module.init_server() + sync_server_module.server_init() sync_server_module.start_server() import time From a96b0b8d989d52a9a138859f1960c4f485c9c0af Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 17:44:39 +0200 Subject: [PATCH 05/24] OP-1920 - fixes of names, tray not triggering --- .../default_modules/sync_server/sync_server_module.py | 7 ++++--- openpype/pype_commands.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index 4bec6267449..d8a69b3b07a 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -694,13 +694,16 @@ def _get_enabled_sites_from_settings(self, sync_settings): def tray_init(self): """ - Actual initialization of Sync Server. + Actual initialization of Sync Server for Tray. Called when tray is initialized, it checks if module should be enabled. If not, no initialization necessary. """ self.server_init() + from .tray.app import SyncServerWindow + self.widget = SyncServerWindow(self) + def server_init(self): """Actual initialization of Sync Server.""" # import only in tray or Python3, because of Python2 hosts @@ -719,8 +722,6 @@ def server_init(self): try: self.sync_server_thread = SyncServerThread(self) - from .tray.app import SyncServerWindow - self.widget = SyncServerWindow(self) except ValueError: log.info("No system setting for sync. Not syncing.", exc_info=True) self.enabled = False diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index ed3fc5996bf..bb7ad152dcc 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -295,7 +295,7 @@ def syncserver(self, active_site): sync_server_module = manager.modules_by_name["sync_server"] sync_server_module.server_init() - sync_server_module.start_server() + sync_server_module.server_start() import time while True: From be36900ffa037b4c98df70b87f68f4864e34a9aa Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 13:55:47 +0200 Subject: [PATCH 06/24] OP-1920 - added always_accessible_on to Settings --- .../settings/defaults/project_settings/global.json | 1 + .../projects_schema/schema_project_syncserver.json | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 45c1a59d17e..46e5574eb30 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -319,6 +319,7 @@ "config": { "retry_cnt": "3", "loop_delay": "60", + "always_accessible_on": [], "active_site": "studio", "remote_site": "studio" }, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json b/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json index 3211babd439..88ef2ed0c32 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json @@ -26,15 +26,24 @@ "key": "loop_delay", "label": "Loop Delay" }, + { + "type": "list", + "key": "always_accessible_on", + "label": "Always accessible on sites", + "object_type": "text" + }, + { + "type": "splitter" + }, { "type": "text", "key": "active_site", - "label": "Active Site" + "label": "User Default Active Site" }, { "type": "text", "key": "remote_site", - "label": "Remote Site" + "label": "User Default Remote Site" } ] }, From c69c15310fe1a55ffa9daf1410ad189ff125adb7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 14:02:24 +0200 Subject: [PATCH 07/24] OP-1920 - implemented always_accessible_on Needed when new representation is created to map wherever it needs to be synched in the end --- openpype/plugins/publish/integrate_new.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index 451ea1d80d8..fe780480c2b 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1028,6 +1028,7 @@ def prepare_file_info(self, path, size=None, file_hash=None, """ local_site = 'studio' # default remote_site = None + always_accesible = [] sync_server_presets = None if (instance.context.data["system_settings"] @@ -1042,6 +1043,8 @@ def prepare_file_info(self, path, size=None, file_hash=None, if sync_server_presets["enabled"]: local_site = sync_server_presets["config"].\ get("active_site", "studio").strip() + always_accesible = sync_server_presets["config"].\ + get("always_accessible_on", []) if local_site == 'local': local_site = local_site_id @@ -1072,6 +1075,12 @@ def prepare_file_info(self, path, size=None, file_hash=None, meta = {"name": remote_site.strip()} rec["sites"].append(meta) + # add skeleton for site where it should be always synced to + for always_on_site in always_accesible: + if always_on_site not in [local_site, remote_site]: + meta = {"name": always_on_site.strip()} + rec["sites"].append(meta) + return rec def handle_destination_files(self, integrated_file_sizes, mode): From ba87f9bc09a14c30a0d9082c295d0803ba7f6219 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 14:47:08 +0200 Subject: [PATCH 08/24] OP-1920 - added documentation --- website/docs/assets/site_sync_always_on.png | Bin 0 -> 27817 bytes website/docs/module_site_sync.md | 39 ++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 website/docs/assets/site_sync_always_on.png diff --git a/website/docs/assets/site_sync_always_on.png b/website/docs/assets/site_sync_always_on.png new file mode 100644 index 0000000000000000000000000000000000000000..712adf173bc6049a2f55b7ff24d4d6cf09928ad3 GIT binary patch literal 27817 zcmb@t2{@bWwmz=&r>go@(NgrQmX@Nesb;OVilT;4GgV{6JjT=solsTOTr)vpiYY?T z)=BJ5z0cX#-e+I?oa^^buNvZgpJBc0S@*r}wesqLz82?E{-bPc zY@FKnZX2<&9l)`%?LYFzVc?xaHAE%wZJ)1^)-5)8@7a0amxInX^=`7URm5@Z+8+Xb zXZN~i?#sq@yp8pLU%O|~V>UK-HSOCsO@eGFM2_@RGa1XwWiP|SUs7D1PTl=PI)8hp z!%>6%!lzrZUtI1UXky$>*g(|p_CP!N8QxQh&fJ(_$@;TX@Z{Wxs(508%Sf`zV>zi; zH#rwuS6cr#&2yla)9dho-b4G_^k!`hu*RxH)<0v_tEy|k&LM>h#S4Y763dL`-At$w zV(>3uiEM0rV9iB?-#_WT0Pp>#0Ng#w|#d8b0E-R$RcsL(&QVk0_<6(UM%tB@RPX^c3nPr<88?Qq zdt;w3sQ5il3hfB?yQg)WJk_SP?x&d{1Rin9VrSj=`E#=A40I-z2%Rsp#_xo`Svyu} zKX+nAk&Dw?sBLv9ICbsxUtuaRW8pZp>^M!a>YuF`msZ!@#^bEJ*`5z$!Z&45AMgrI zE1aky^^a>!OATZbD=T-)M-)P>K{I0mL0r=Z_EjHNC;(S?bMcLSmi8blVDeHJbl`?u zwXjJ~beX)BZaYRL+u=_82C`C{wwf|i6MbR9b^sG@3pUH^E4^r;g6@DjTB2=eqsu)0 zvON}+55od(*mi?Uhn$-$G*qn;r!5r7^B^Qln1tqH z+8f(N-twv1e7P&T4q_1z>A2$FN&Rx6C6%wiA9&*i%t+}a&6ctt$FI&hZBdGG*-N>; zTURLyi{%*oR?dtv{m=BMkbTW7F&AN_*7=1BwRl0=GxnkJm89J=~VdFiE|GR01NxvX?LrN9V2=>?J7n0RKws=oS2fq z+?RLyoyVB+`*y1TT99pP6^u}XTJ6Ls6UMwKD(QBR{Q(Qx-4@9D0@K8-nB%YSq&9uN zFs_hsZgPot$#D;HuWK`AMF+iiVb_~)Vkmfb9ijRu)r*(z17!|yy((^eC-hAnL zShnn{sn;a+@Zsd`g-hEaet8UQL7~8=Z!>8yimn@%V1E<{Lskd7nW*R8BE|V0xBCeD zC5_%pO_h&?-M8IT)J)yjT?1!q&XKB%n+nXg`93!39bwRJP=Y6up$g49JdDndFq$p} zc7s_i?-Y96@b&7@W)?F$FgI(8yg85TLXyMt&yKz;}!v8JT0Fz;cE4eF>^C7;Mqv(fpz>?Cn`aBW^K$ zKh2Gh2%2%N9VE}JzE?S$TT?lIF#5({coX=V-`p2(x$50J(cL&7PC47}Gb&L*GYf<1 zR|7|1RuO0NawHnASqu$0@ny^?xaA#Mdgm5+wpQ>w<)AKg+(CJaR@E!;=NV;+N@0F$ zjT!8s_Qghj-xxQ|L~#l6fpwS;cG#Z+vL&rs%MlOHjHf?Gj0I;Zj1kAwe;G}mc$H~6 zx42^tqG{szuDrA~Fj>{#w9sRE7R^=JFgem+3f1Ep8G2~Lb@OFKCxJviV1MTdKf1^F z9uPBwQmx1@)B;*J*u@?OD(&0(@&{@1@qC=YjZQr0a9e&O9L`YkgZBz4wZbp<59Hj7 ziv82*())%CyZX;jV>$CMSak~usaGcY5MC*qK7msrZle`3DfZkB!I9khmRR1-Mr(}u z#@%=|)W{}ev!v|J!NG=M^HtOwQSr8K>&_#w4yBgH4XV6dA$=_KLU36D^8pU*75zEn zjZwC57&v00om#^>0&^1K*`{?s;Rgp+Cg|m!t~cmi8HC!O0Z=rX|Dsojwxjp?Hpdm^{?7ql{f|ZYq@9J`Kj{d z)MBUR>BBp4Hzk&@oU_?g9bTc@?Fyw>Isz;HU|`kgTkJghBHFT zEP}GbW&Phf@tzuS&VLmBh{?EUDJkEW-JL2YBEDWzG@Rw#%{iV3yDISJXoip*_p!Dm zbWWvC@unWyocv|Ha?VBY$Jqr1t)5*XEDFyXdEdN=t5-d0zIlcd?J5mdJ~(dq_EKuo zL*4Z93H}}29zHxtbYHE!y-!&>W4S)Od_Ceus9e2)~b!q~pD&(H$ z()OE7II-l{$Ry3;t0I3Yw86_@jA~^TE>9D(6LV*lemfNeBEK*aaYCh(e`lW>=hd2V z-iSHmlg0fyl&cyST`PoMRLf1HC5IPGd|#gu&QlQ73pR-lf9TzWBKoaFFlWP%$5l&n??i79*S`Ck-~9Q! z5J9?{4iW7b2KjMqiLITby%3nTG`aJR#V4Mq#SfSN`m>O;zgvVQkX%HkEq z!+MMQP1^)($F1K{r?|RRjx{dZJ*gw@o_kaDHJ^a*q4ggxz4lN_L*wt1;k?z}6(>$i z^0V%F^5hSID*>;AAIz-yXG8vW+mo)}L+<@MFV-cml z9515fHd1b{$X&|m3!hEzsZYb%MLi=`PPX~(?6S6RSAOp_To0*$*|vpWCuQntwj`W+ z-(K#;nz-W-`(WCcqnYHxt-s0CbkvJUrR1n*VlpRe_Dpn6;p)<9)}{Y#BiXH1|1Bu} zhi!PamPni@xQ6&4zJ6kf?rUuiV$QY0juSwv8#H>mu2>~ce43FkbB$!yF%OHKcY;~z zE=6eQc2O2CuM{w-dbZfIUWaXtsdY2;1{?0hq-w~P7HOMVWJVEZ4Pp1I>zow^C-}Vt zN$S7Y5L`?RYIJS+6wL}j~*MO4Y6_BU#t5|~c zd1%_&R=*^zD~=HjKdY>7HcqbHIR9gBU$n@^p95Zn*0HBm7mlG$eKjddj>B>2MwD{a zbUKLltM}Job=QAD39`!`Wkus4`#vWn8yy z6$h!cd0NZEZluedi{MdkGRshGbPz3(Oe5lRQySb{cY~IoWfo}dL8(knT!wq_%6*7A z<)D+;;|ZgM7tXZ<^m2LSn?YFX32dqp_e;p3G0x#$-z56&!HF$6Zqp@!mnft<9a>O8 z7N*FP9@fH*u;k5c;$~0x;)3+tVvnaE=koZP0R0>0^Q5kMXvFgPkW8&4e zH#fyZEelim5img;Xhs3Hae-6Nmib}0$YLUnYigyd9bD?Pz>AuAomlIdN*q#X5S2w? zj}!G>0*!I>5%>zfLXCM3iULEd90)s2EZn)LIwruH<*L_XUU+WGk1;ElcK|P(6H}Vx zWQ~ySeLt*H)eDx@{Ix`oEbxmx(YpCl~3ThoJ%aV;&0tb`4Tfu_LtO6RD8kc+{t zOINI`3-8JiZ>=4MjH_s7Sq^M0I8C^UDOZKjFZ;kokQf`WN}w1=j5&^$37G3K`WN!txix!nbolk<=CF)*7UC5^lHYvmqIo*A{vlmUl;pH)J*+zaWFDwaQ;4lYY{^X{cMblby|4Pv697*oAk#DbDP z?Sd5nK~}Qtp%JM&j3&3=L4G~LoC7zR&o#G0+Qpa1Q@O(|>eaM=$M8O;V}%cq`M1z7 zrn&6Qv1ceYslNflzeaO*FA%iQrB;my342bx*@>x|qup-@S7VzRARb3QU-pg0yu`TGknix@LV;#PUA2CpPz( zC3W}f7(VwrRm$%!r(hLzk?UhhWxHR&crqpa2|F$;akzBoZiLt%=KOr^<*ekf=}t%5 zlyL1R^5JaBz3}xPxdD>(&FCppsxy-zev;d-SyI&x2*iYfQk|g)TVWZbS_HFOy1Q04 zj<}J>4>4AlU*twala7^Q9GL^^2*@TwabXaLOT*CY5*GM3s>tol&TMQqHlTv=T%7v# zn<1w0sI-3CH)Gn&UBwm}v}e&RKz_(&hJW5yzYY#58|Xe-YnT>}p9A zO9_vR`BFmI7-W`jDa)n*M%7ttp#(H7ciWL3EroO5_0J7PE>R|v>*~Lo&ht{!^Hl+h zQcL{|lv=!}Vx2E2&uR`R^hm$kbZ#a8yxwN#IVg+bMXRm^`zI$-F^SVwr#8NO0lkbT!~eSMPH*xu*e+wH9D4A)Kk?|=3SCJ*Ro-grfiV|4V+fhW!QX; z7_ZzOz24K0`y~i*ZPL*3%UrrPI4?rW%6&Mx(|w^{6up&x+g75Mo{f_Wt6*;5V_>w; zj|PRQhI^RB;i%H#AccwAUgA<_vrw#g^TPAxg==BTH`RR*maUlVHE6ro?%0K-brtJQ zwW|5*cAKf&Su-9GMDGe*4;GQMOUAXW7%hAyNu(^&s{lWTN=Q(xmWg#nTx!2wQnx@c zvgE2%<4Oa8h?NiW$kbo%wcAhYi*&LDYbb$`)cQ*(s+|^d|DTfPAqrD4xuuPZOYeF^ zvRFVTr*gHjXSZ^B_m+PMa#Ll260(-7H{L{!C{0^7FH_yT{B6upy?AJVyQ+qu0HW>==#=*^bm2|2ZE>tUq?j1sL6Akf%VT*LWfB%s z^O%aS#%m^<<|1O#HHH=`nd8nf73UC$b_@kh#2YJ(H^F@pljIf2RTN3daoVgIIbnTj z7jTMq65|fre34ahn=~{SEV@m-S^XhvDi2ECUeNA~0K$yM@`jRCmI6xW zjz7|dM;SkEfvv&MJhlL@r!v39S)4-egkZEHf3f|Hi0_!=dozorDOg6jT5W?NXIOw9 zaV7>|03a2-Xk!EMZAsvoEeMph{>(P=)hLFrSyy(_v0bUmCoXZJ4t;`-?~$Gk=tm!) z%85<*5MiTA-q=a*tzTbny3@==G{IG;zJVKS@%aGUY27lW?GKX@S@f4BSg|a;=f3eJs8vV6;wtc-|+PX5l(b-jW%5Sa8|+yIQ=%BnSLQD5u=}jDRWH-vz|Sh z7GBmqehGwCsq)VG76Udi-o30vO<3?f9nVnTfqz`Pzsgwe%Ut*6iJwb9S-r8f{M;{& zPmGX-xMaQ!MENIg9Z$o*`$uN(%vKLE0`JxuC$e8NCf?FUY~6bLK+6`4Jn$vADCkDRNtQh~@I0H3f3baGc)p`R5pC1RPQce&0#DAUGc?AJSKXBi*CdmzfzSP#apSq=z$c zxZ;)Pq6mHnTyKaZ?yat^7LDF=R<~z%j%R(KXsemM>b;z>_I8K*R06wV)VWEL(DziA z0CFAb^<%gVXumK+(ArwfGd`);t%}h7z>6@>b&&D3mHzee&6#gku`fP~N14sIjysx# zElEF4Tdz}I?2E_?Hn72WKk!5>hh+JNUxPQ@oJ{bs(&~cgJX@=@9Ee}pN!E`PBT{=_ zG7g206KM=Ny-jFpc5@Ay?>f^;v?*2~(_FHr!NxZIVp*_gTCQ-;A*MSdh$x_RR;{oa zJO>pHQ>)??y}q5ECz642;=djq<{IkJvm>*azTGggeYPHfLEKf|_ds}YJwT`5;3vjD zin-`a&O&@(9fuDs!J}Csp2kX-|7@TZ(~&4b#DfmWnuB= z6u*PnpdG*(bA`jWYKo_Cn|VB{)8vEP>3s?B`Z@<&+?tiDvK@5Xsmmt>P6mJy2L7}G z0xjdVvA@xvhuG)58y>9;8D*yz+cQ2^$fh~l=ptJU?kqE59dD^WU7J07H)uxw({zQF z2Mp&H!JlAW{n$KQe6XF8QR1V zYv{IO?n$RWsEs<~4jh>OHlvFa7-*N4fg#UprMc zIJEY%K^YDfZ30nG;MD$U_DQ=Lx1Gw!C_B zmj(j4ChGB9Ppe((iMaG=Rnm03-dKca`(mX&ms*4N;4X<6UNJZe+DXVTo6ruIvW5mt z*<^7#p;PzFw6 z>xqwX7w8d9g9*LPGCKpbv4+VJ_S%3P#kMjh<$XBy<{`Qt^Q;pM6TZqYZdC7N5L2do z6X(*QDwqpQwU@&vjCYN?EgRbp6J3DA&|dj9y_T?MZ+?AYmJHhek&o`L`TV{#!rpYe zIcOBBeqb&E$7A1wcdc<7_-Gh%Zh-(Mv=3~#&-Ri9#8xH{>`GuHd0*l$7 zqn=eMpOE^R&(Pcs!Vj9 zQ!3XiRO@n$0>^-yKcX~A$a_`emSFq zogw-#&7SN*K)9tp+!MPryv&c`68vA4oyJjK!Vlx-Yr{Whm`s{}vYbF*Z2}2ul%R>* zW9LEo3u+&B8%J%v;+H)=b<=jLT<+f^>8gwK={LGs82~ zrUF|dH{equ5gUxHdB!1kvqts@pLkL0Ob^HwzlTw?~wU(DTkG#_}=Nw7o z=Barza4?bk1t4vBojfL}*zoH}fa0Uaj7z8wU8nUjA>#eBne!3DgVN2}IGYen0{vC% z8?pKKzHidKtP6295!3Q9slT+tR?|ZTJ)l~i6s75!#QT{uHOQvsPKX5K%N0;+A+{+y z>DqWD`(krw3wa{Uovz`0m%^u5z(dq}OvvV1PYMg)F(2RDjgJPA)@L%p1& zy3W9hf=`|`3uUauXGz1i0|OObcAd#GG`C09z(Z`e9i8VF8xP>Emg#>QWV>24`emzu z4D+s$BJL)KEu8Q97#=Sky*PmCiAL2ZLkd)(mPHG0lC&>>)TXswj|hq1^z`s!NV>pg zZ2KJJNs)5Z-l$Zi^b83SISH(ud*(0F$7IEJS%O4V(MnrILzlW~N4i6%Q@CpduOdm- ziD?OwiQrt%A}=-v6NIQEfH&Vj#tN!FN`90vv|K-qs`!SMR=O^0`8D_YYv9DQas7#wUk2;h;-l3LWx5hIo%9hMWp2ZxMdxZ zvR;Woj5_sJfc%%(l}jAQh0ooJbZ$^52rNDY+@sGkW5|m#&Ib_{dULEtzUp~kkNkK{ zy)^#53a{LUo#PR+zo^4=Efb3SlnbP(=K3$0VbkE2G16H5JqM9C97NdZyzEl_VOpYN z?SxY@RLsObTlvD+i+=q8^?T-o+(CF3K09l12m0Ze`wh!Qw;R8d@0wC7(M){O!H&_>z5BAw4 zF>5p@X=~#k=2}9MHQs-)=DP_Dr_?zzF^dpzs@p;ua||_Y98$G*&@4aGs~Yjl?2zZj z5#-`Q>ImI2F6+l>%L&s-6Ot8W4RDA)_4Eb=a1Mjbv_M(p;gh9LZ(ay04|vh)!-2l8 zSxIr96upPi<2H#8?fvqH`bg>za8%e%+h-Y`GrdHo<&9nqNShSZdE_9NKFg|DB3?uJr8x& zExk_t1M1V2E7HwJ&h}dGBn1lBfRh&>k!OJw(k+|Wacn1US85Q9n!mgHw|cJUIcg7i zgcFDgDibD6>SfmviaGhS>ck(GS2tiq%l&Yih0JzJN?teQjX-K0P->cR2SqHQsg9f5 z`C%H}ekxAKOp;i9Bw4LgGT_FxsswtdX|f5PJ!1)jf46%{ZTQGwNs^E_r%mq~v>JhP zIdH25(7FJaxMa?W^KCXMs}Lo(Oq_n z8oy!Ss#hgTF^zh)x5I|-5>lLj_?eeKAH1^G0m%P-c_aDO+tOT9nGJ8gYyKp6ZS z65|;4*tT#sBqu5+4MtsEfwriKK}S6{zo<$(sr!z3>T;z(R_Iy?F-!TKuWPn%iao># zM^0M-k2)tKhBJU=_9y*Es8(qqi9J^#zRf-j+^LFx1qK5`hFhB--GkD9-x@oJ^VK_z zsOUi<30IOa{p;QyB@TtDX}WSA&#W^PP<$YWdTAQ&R{5UTc>xz)6`8h^XdF3EWIy4t zaf2oV@!a%EL6CfEmE&$FyiS?K!-lCnn}l|-K@~({U&Mx}9tx{QedPLvcoLl9=4Pbl za<^!2to2f^sK^7SJ5KLOyMDOn#NV>z;Uj>;dVL_RYva3_N2TrnRzSK|7ubr%l46iu zv-RscGk+_}?FYe?d3T!wI6vFPGY6P?d-UYyo4-}%!e0Els$Spa<2V826dXZ*9u^=c zi4KmH?(p3!MBID(_mZCDH7f$_S4{+V7$wyoaU-sb95@Pz4YQ^&y>VRFpG!;y0254V z$6PIu{iB^^hjti(5ccyTG&a|A;+s_1uTK*JuL9h4SFKpS!nVK5X9mv>Z%!7SV5X}1 zVkYjDFj0cfCZwXe z>)8uI`Z-M*51J|L*qEIoh@Rm4sqg3N_r^b%JmW<NU!xfgx--)~qO!DX13i7%#@y{*q32n)h&NcqbW?wBE^ z%0Z8*@z^f!XH6Mi>R9v4Ql>|AB*A*$G5y3f|L;8t3rUA&YW|Asla-X;33s)4_NT0B z$Mg+gEvK(ms$B`4K-CyEdOqKjGXZy>a;^``x$A~huu>*K=aeJH)m@!7?kH6!AC?=V>)h0Rvp|9KzSU@%z5ZZZvgRm0^Ug z%C_S0%#;v&V@bg`(1Ib?4S*h^5^0m0J5q9P&sz0smxT0PA{q*iU=(igwYyAPt3xah z6`nku{_&I^aM@IN+*j}eJGthhVDJSRREn*Bp($EOpVnbwn9b);5a4%Us`gjj0qf1h z%#IsV4uZMA5RXzh@OYa~#aJ`2)B5SvXENB$*Ud!p4B9-LBC4{3^>&tXVwIo{XM~@H z_n0}~(KDJ=I~k_R_rwahxJg(vDt$~TKE;DSlw7P6NDM^Z@OT(h6#C54f9dGYG8ouv zplZ;g4S|{x!*Vom)6Bgu66m zSRVBQhQsq1%$XZ|=o{;%DKr*-x+y%cPk&m$ZeaM4?MTMpFUVBG82jO0_s4 zXIUp*N$eK)s6iN)4O*eLA4Dd6l)ZFPpa09Ul|Qws=$(8kX+U`Rh~YL+3RY!`f+1L z6-QqGF%XTsqi5lZIVqr&aRRmb2qI6`&1laZ0O@|u55|EF6S(i|rL6^dh2@Y$jbV4N z0}iwG)Y2sQcEHdHict*xhj50P-VOP7_^^=J15~^BreFv-XL3xtxBoDC-ZQi|nY7pN zc=|pKaJB8A7&-f(ZSk^)^C})0#TBIc9|9=DnJ_~27-vO_&ohdq0d&IAl9^{ybQxR6 zzI0eCLB83|GcOS;S9tE4mP$6?!EzaIvBmKh;15iQyGG6=5ve z@-QR^k3JgG@OKMU%35x9o>2H6h)4-{DiOUdapTGTtCp-_d$>w1Q&J`E{uUB6-7)3S zgS0%5reHa^rw6lIcG=DqwHVjOhGroF+kV1&+WlJ4+XK(jb3T`>4$N`wl|(fH>0S?6 z!?Cf|NB%Qr{sa!x8O2eml|iPJQ*s_xs^#KI;6g9A={_^>J>Ym}`S<3Se=CUmXGP|J z8sZ>NYQa~9i6YA;S-rmqr*eIoJgyP=J`kdW)`x5huB)m6>$rZm?(n;+rI0y$)G3pC z-qc@bs1L0`k87O`iWU~qk2lnZGk~kYfNh;Cbd;!RU{Ib#4^;hCUe}-+12gv!K9iEK0*E^$pOC)v? zJG*&Z(x%}-7VLD+H%Q3PDH_ z=AAXPcDwwP_~6^h{bH!)f!V)|{l6o#`rr2LgTOtFXPyc86CKAHOKsK5XW|>0J?_uY zIU3xd4-sI+WZ%4Y$Hj^E7AU~0|NX(Rh2Z?={-kbO~>plro zi&A7#K>eS{PZ~DUS3XGo5}o9dn8@#u>y-GM$N`TXOAstxRhk?Yv=@9Cu^S*a)GpW@ zjXPN!q;)IY03lcZ7y-}}yK_MMnMT~gMuBn(?x6L;anGg)k*}^ldc0M2#?%A$gAA?n z92JgQQh~8J1pzAGr`pNb-foYuNo%;cA!MUVQBo!MM$I-ST|mf5j)sQ;CxC8d85-_NdN8no51Ql(w+!5ybrQ`?EmV{gb>*DD~8w z%ws1m>hl#D%gP5J;B<6Aj&KEu(0>{NkbC>uO`>U9u0-!afVw~V060*`AA2nRn3Kux zp13&?z%Z=W{(sw3_n*T?@YnoFebiaQv*H!HoqETTUE4PyEY^R`b@dd*-MxHohGYM+ zFXO+$5YS9=PDCuE=GU8~9;B`j#UY}am0)}>_Wt!b9ChmN8N|PKg8cum7hs4v-G3m_ ztQdL#;2j#h4a?cl6r-ygdRuy{49AHw{(LYAOW-8h38SPb;1a#Cl&ZUOn z(z15A_FHun#8iQ=2Y{91Tltu_hIS zIx>(U+MNSzr=hc9fqL|uUqrJW@n6bRy@ENlS~WFguDkc7O<4BP?h7tszQB^No}1=L zj#8%h-1aW|B6U&WnpJ@yp>{9q=XDxa0|E5&Xzk+B4zVG@Ur4hEnZ?qE%U7Q`B zTa!Z)#FBx^>7a)O0;?X!cSVixV>zL2Fi#lvG3*nFHETfQ37Dy%;Q!MQXTNQC|!X z<7bb7r0NWlmBf-Nxwzkx49X=;f#|uJBKaEq$a|M0=g~JMjV0X8ieUWC+jK?^8yeu+ z$=0;C*Z$X~CChO9eVsR0dB(_p8S35O{m;EZ+!wO?rX+0oh69T27C;s(0^$e*rDa|V z%lLA_c|X!u#kadXokiky|n#FsZI!k5n2}RC=?LNZwlWy+g@b+|Av2Rj+3FJ|lOK)SV*fiN3B`l zw3EM4(GAvvJ;AqWIu9H%)6mT!Tf_IbGG}K**br-|3zSgEvo09Kci`moN&J-vO%7vwwxr1pKkn=lxYK23frMAK)*FJ zRjmOyyjleG>K*%Iw4RDLgP*Bs9UVDg{c0>f8FCIRnf1f;3Xl;)cxpiYoT4$jMWu}) zpJC}spc{0C1-`L4i1!pPtg@TKJ;rkPwFi25)9GQYuvJDhdvNx#-fNZ1!?h=)dQ3!0$u%UjpbzXZ-yT0Q(^IBw(>uS4P17JH-{5PPRL_%sAkEas)_$B1_yYuhLe z>r_@)wgG4P_7sYxmZ|7-g=L*$@BOQP191K4+(3NO|L*Y>Kjr{F@M*2X?|kKh$&@9` zW{o`6`Mh;oJY_0HQpu$@<5*{J1(rSbuD$(&eb1|2^^86NSn32Qo2Bn(kQy|LW9FV@ zI-J}y-wojGO_wenCU@`U#O;<3cxzh9@9{V;eRh)@3Pdo5JvL#;N!P)KXL7WeV^8*d(7!wNbq$po|wtB#Wh78t&r|k$JvD`gzbqIaHZ~ z!w*PN4GtVSq%VDUtlkN#8Zsvx{{zPYAKCvV(-Ujw?#Seo&*X;fp|{ihS9n5#nNt;{ z`y+hv`w_-~eR5?FJ9JP?DSqLlTBs)~^(Pm%_~0?_gM>g=Uo;a>Fsa>s@-5V@z{#Nb zo=kzk@W`KrRUqYCJbcC7iuBEIwI;Y~t!4S+hBMV1ay`U};Pk+xyC2oSH7;w6^F3Q% zF6gc7bnR4)6Kb?1dqU)@A62$ac8>cN@VPQ)?|!}Qz}$@u!ptktg?g{i{Lwp^o5$F} zIzc(A(fenRaeXiU+)S;`yTPn^b`U3K8XYE>we=#*W7;ZnI70sG68D;gKz`*xTvJav zd8EoDY0fH*U#2nB(j*JkynAptLhCg-ej31~Cr>9w`b7dR^V)trbWVgm4-`{s;fSe) zfaN`mW4+BLraiM?rZbVAN&d!pkVUO-0-rT_^7a8MGJm$a`!@kpZBNkue~Z+t4>T-Q zfB*Aq_DR%-u`gaJ0qzq{IJRTF38Du5z26R_SXl7+)Bn0C{sn0Jz3x-#%rOyQ3Iyv#LkujI~1cU0IDBO95fJyzg}_3d)4OW zzIcLy=rAT6x-2xMZVDc^9fFoyA7T1vw6Dc3bXSO&DAj1RMXezE9*gsh6C#5+?DJFl^S29?;xhYI zG_~?u?Uf{$$B;|Q<;r5l0Jd6`-pOhJ&dN^QdCJ6CQ6sEmW9AMY`0m^jd0cJ%)Afe$ zQUf39p2cvinyN!*9F8m{GxMU2T)=xTMqqtQcf*LWmzVN;Ph+5-1fdPTuRk598XlAI z(32ZHuS28{xI#>(-jx@;5QF)3NTCU#BOZVO=`9txT8lDdC--9=3~#u8tToaIC#GMnV55@Sqx;|Bt z5`a78LNt$NEDL#^?7fCc8hIyUF!+nR&=nK?Y8mAnP3;YQ=JcBYxK;ExeOw2C5q|-N z_I{oTK|R2wx_Q3cLw=ZC8w^lyAD7tv8JoNsq;Hbu^sIJm-QU0kQZ^`rW=agq%$}y8 z(jO??v=1o<2=v@Mzh&kImHJh<{TD_KYMCc_XF}U1xA+gSMyV2loGLUz>wbjc`_29~ zq#VzJZrf6h7&}E^@-zCR{hI=Yfa%Zc5)Kd1Ksb z(%`<4W6osj<+#9y(OOCvLFHSK!8Rd9KhFoE=?0!f1!a8#4XUk*49~1#4goHg?)6tn zA?{5Ic`8aTnG1IyV~g(U5ie4wqJxveZ)N%PwonGu)gvATlyxn`?{XetZAUM<0()2nOQ=SUZTvojAyPOiN-W@N!klvNV>-EbaOMjCq~mm zap|So`z8bH<(XMk5hGh&-!Y8dz#j@f)Dg+@AvlM{mm;)ZX!sh2BZXR`p`~_A?E|3E zJ?3g_-FNKLcaP>CsjoW29_e$7L+|1!I!Wo}>fA8vLiLXo=%!y*o;k2IAd8Z1rOFbxBxASL0AB0V-@`IC-YkFO|a6Y3_gIRT&lv&fdQ}x?{wzH-iEDEy8kn?>a4PMn6)1Ds{cFK{uxl1ePe8bTym^c3&Emn-C`I z@wb4uNXb9-Y8`IFCHJlQ2eGR#ZiI2p?23r2-B6+d^WcdAQxMj4n*9Tu{6fL&cU zggMWXfBL4!V<<#|&ySNxrOG&9&f6cAqqL?zDqE{B1r z#8AaW#|aIZ7sd=^t47PMKg`rW=3lw$s;x8Rth5hhSGO7)s3V;9E^69!u4vGYma^UJ z8x5p5L%8hRV1%+ER!nU&Vxl^wx~rS9Tg@1}|4c}g-89VdG1Du^2S+vb4Am3perz!) zH6&Zx6a2BWa?v+uBS-~jKa&jQ_DT6`QhGbgZ%1k&G%-_HuG7>wSAJP~;H|;^RZ(4; zFw(`X@r76sepgYF_zi0ok$1Aj!3xZ-^!FAh&1ByFF zNsHI3!fRIJ-g=3h0DIn7*)}V{)!a;w4GHig07+FObqtqUj<{Rg9U)C;?=iB{WrmcZ z$%daA1AH0%+^LBr3)e|X$Yum{&1rUm6KE-*=~uG%Y>7*;o#~uG{Cw@ylm(r8_ZPo6 z?JCt-DR%w^cW)f|$5Rod$FLuYouPc2R*^(6f|l*dH0Fwj(`2vT$?}$tETqZ;`TH({ zOLueQ5zu)#q`e8U0+Q7;XD&@b`(872%1%*bkML~Ay6g|w&k(+mbGunsY+e=MCumJT zN9NhfYmYf~w#sp*vU2MGO)~v2Y~-Jl;|KpqdE2XIKmU8-`QMKB=#~s%n|$`3e<(?^ zm=$z+uWZQ5)mX3p7oF{&m;R3*f9+dN1Wy8mUtX#=D^-^PrlN5It(T8)0xe(y;q>-U z5FT<9B=xbR9FO$h;D&~NU4u!~wp%>#paR!^E(VnPCr?CgpZMu;Gy1>VB|kW_l(pX)wEb5lEVL}v zN7CwCng4oOB7DLy7g}jqH!nIiYTO#)Z$_14gN`hyVZpC|Kp^=G;ZjbhhF0qcq{_(y`h?ir4K%Ni|2;?@?HFfyF;$C959>n8@ zn(-ai`&PX35oAm6Ng-e$F(7~Z{OQ#1>Z_?VyK`QCkARNJgSqkhf-i-a2zU!(7I4`O zIhLv?ss4Z}|4<-Q1dND!&Rl3Y^O)39(+Bb`LmC zmRcjp>qmSOs&kx=NCUl|!Q3B_K#`TGY0aCY(1l_plo}p#Lq`zuTCgAS3=VADJ%3=f z^?iVo95mS(;D!!&pXbd4H&4;T^(05~)$0=Cu;8+kyi%Z3PGB_YYf58=*RqiE05ej& zJ2X!TxOe~0k~+4>FAzT$Nat-S8#S3>PH!@9v&6;GR-pRy`N`uZO@c`&<+&a3o|g`g z9=-c>Pl!9+{WmqNPYc*9>vixyrxE|w`~SIVU@(?KSmb}w@(8F{<3~sA_GUQ!pEDaE zsC4`Re3=)JcYk^G%{Q0n4#OG%&!=USh!>%`OxO_k-vFeLZ9RTBKKGAdAlYVvKp;RTO15F{`?$Lz z|Evhsk@z+40$;A_us-5vm46=PHbh_glRm48dT&xO8@f?7keZ19xK)$?HnH0rmS5G- zo3{~Dxc)t6XVF4FKVqqZS(c7aHXSc;4bXENNDWyUfvoTgrK@{M| zlXq-iglU>wN8N&jSfY-n95$XoYvXK`bd$T)%ruDsg8^tK-kFm1gaTr4=%belso%)D zZ`#RkU)c^J-{^o=UmZ#fo#up4nZ~~3=ZP!Z*I)$N(=oV+D0!8v1KKHZMk-cLia-E_ z$8I1Z43kr*Zuhp-D7S-S$1%3G-^c6MLzfVW80`zQ&54kRA+Tp4y=pMr>#dZzS3!VE z)lHN;(Bv>TVMLd@52WnxJ_A={ zaEUy<%QQ7&C3URu6zFnwTQxl`d<}{g7u5^cxz?Lz0&l=k!uxezo8-wOFPzPxyqLTL za-(Oc>QA2?IH8z4N4ls+9sjtK?>1jwSlt3Y!1iPeh`hjqflDmyCM|gh>9!B~9dgtg zuIJf+wlyzrqb;yJAQvOpV}eC4Zg&US!{A8_cv+9b?Yvr=5#ly71iw+MEV-s54l1rw z3sE{^yIvnUlw*oIZ4xwpz@ow@(W4lj$7kx(_W#xPcWHFlHpXF*8}lGGlnJ>Avshd48Yw z_xU}~`+47Y{`KK|&Go&m^E}SuJkA3IFos)!VZyJ^k?ou&jaMr-EqW|>2P!ndKJa3p zdodrqF1H8PIa=dy`bX78-G|{4+7hjP>)KbduU^=@X3FiliAVf-5u~mb=)X_Xkf?Sq zSb2g#m9XywxWo^O`)lTw2lM1nZ6fs!23XtI$D#f*FI#u6)3K1Qs?vb3Rb2(v71Tb> zf&;kovTM0J%za};=!e6jYvq6MEi(v({0vDHBzVD)k@OsrRg(1(ia~%-nC^0iE?Nn@ht%XRI{*=*2b9 zkx1QZ3msQyn%d2-)pRe>mx42V2RMPmA2%_cxXvBax_5J=zQfgHw$SK`mx^ns=^^7T zo=WK>nXba8r7xBndj}J~>uE&>*5RsiN-a<{9q(=If!*#f?@a$oHEFrA{$A-0YXdT6 z-KYdKp6)+SC#ZzO*((!>3t1*Z!E04-)a)u>O07ksdB|#~ISw~BsKvc&)CAoU<^s0m zIk?D^fAt$amLN*xBAUybdFnOWfKLRcVO4+0tZy%?xQ8cRY#d5P=zT^jfEQ!~j(7_g z$j~SEjvBh~haPR=S=!jRg21S+q;TSt-v%{P6r79>Z|2XL%&a>{N)P%hPW^p%c-xK4 z1wD{oYfRrNoz%56+9^|_NK-j=Q2!#31?DMxfB7~1yz?1%i+2B)AQ!EBY&_y?&j>RN zB6kiZv%s-X{euImcK%br@E`Q-{(Ki({uzcZk)n18J+vpHH$tVL!G@9A+C4$v!{9L& z04V(XN62$hLc%U7z4el%+12+%unc<(8=Jx-$B)Zs`5DSEsUCLDV8y`nBtg&bH=rqg zxG5WlGvgqjVBPbbuK$`>zA020Im=um^U-Ruxfq zuawXNXTnl+vv>E_AlO~9)dM=L;2SZlWIx5K4fW~k)b+xE=69>CJz&EnBVrH2R<_g} zR8b{Q5=2j)IAzgn#QT}nT2gnFSAf3Pp{LhJA)Lfj_h2Rp!BY+|I)&y(R|hPef~oTC z#c+a<%X!;crYucRlMB60QM&x2)^)B?ckvP3UAZrRq3wFe*n=p@Bvxa3&}5XW+MaQY zDG9l%;h@nz;6iNg8q47Xdr&SdYnjK@<}lZzwY}q6D{K4eh>*FwyFW7xnX9}tctkR8 zCXSfjY@t}|_QahvoDFk9TzaXr+O(>@_6!CZ(){z7;qW4m>#iI*-lK%M!C-q@t_sF_ zzmOZXdex1tC!L4Z!WD+E%@Q+ft+Cp>n>Gr*F5pQHNo>8O70p;;j6xk;u!(Mc_aJZX zWT}%J2vNp~OouCaiA|QvT#rDYf_Cw+^)JFliSixt$4#2+;Nzo^K05%vwW5GxMl(nd z9S_tkWde(iio)}Wrti$kTV0uhM5MFXEK$0=Fu(j#j*3gryEa@QMN$%eL2F%eZmADL z>{UufDabSH#vJt6yvg)fU$fv3GP_8NPRy!_WKOgov%;L`8 zv)6MpdWrAP|7~|UX8h>AlDsckSE}$&Oz#t~?WSK*xjb>IoJydye0d6|@DSr^kT?AU z4*5~lws#Y7y|s2V%-pgo5D1Vs?K9{5G~)cma3eFk*bXup`~w zPDkTVMbTXXjQuqz>B>VQj}5o6gqw3$8z-cfMh%W{^5Cuo?G zm!ixjzh1sq42_KCf@nO`PY_cO-F-Q9!=*Afz?k;xO;g6(%$1=8+YJoSwE!PAbPomO zSngdD)@DJ$K(>4j(=SEDH}PE+MH18HpOPVd_Zl`j=_+$)hZ+p{Zcf$Xa2yy5j+ z%aoP+20I4rYlm%LfMtbX%wHR6^dO6d*znF@Rbo)jwBq4G0Z*mvy{&H(;eudm3Q}@* zmNBR1G-&x;{Dq<^dn8B0I9sHQGm>Efx1sIuf5OUf zvHiK%T%TQxIdB&Y>**^EqHmF7rF9d|*hqqiwEyr=;7Idr2PqwV{)3nb;t}}#(*x+g zL)+OJ=eL(l3~s4@kNLNXZ>A{O6qbx$wwzi+u9kIQMXpoV;%QJd)PM${{?GlpdVTQFtzvcST|&UPOo$p<-z}-{LLHgk@v6P0`+L4)Uv-LOSBZ0D8Mfp)RB{I4 zaxUlN_}w%cEBr`(B;%P{dyn4PmF!q)sC(8xspk?*b-r?)^<7=@l*z^lIKfDT;bw|m zh|yF-#?gQ&o6J8FQ~ZZLc;(a*>*b*z~9j^;x_0UjPxKk$v()4G$BgG*bu zX*FjtgjAo5wfnTxNtnO>UDoC0^-Z_g}+$J`5oGa{!w7MOLNPupI<@mJ8yZrVoJXz(t=5>6P$ zQDP)~GTfnuf9&k!VQz2fz3E9H#ioV^`qQC=hx>;R&WVRtgiaYUav~6iib;~f>t1MJ$qu(s}lNf_0JKKGbqV)OF06j-$ z&>3D+bRlKH419xDU{NS)T?c!12~m4LypieS<4EDOU6Pyhyc2B3efaWL%h2`=FH5cX zMj4ca33=W!L-)v%xIAhA89c#=O~RIa)JblwfP>(v4pQNt&bbLCmObt4yrZCM@jJ7IeewL4q_XnJt@5qB9DefvR0 zw=CALb36Gz<|{)UCzKmmwxfDeDyj{7B{CufE9I1Jk&njELlR^Nk|AyM(;-IT{uzU# z^prDsq|B$75wK?6(*>3N8CB52QKL9{;Bi3F0o&RRyn+VvS_y%Y-5}Fk0T4icGwb;e zubfU{ys$w%&1Y6P>GRr`^@23mL>O!%l!2rl#97M`?;AXul(9ycCJlr^gCt%KBqIm& z2r%c!W9{aTd3oj>H+d|$QdmT`DQD%EkLVc!{kX;g>ldmW-R;#szqXQz1s-4|h>785Ro$VQx$xw!1Ml0X8Y^aQ{Ok23MSiK;?=*`|8 zev^|4U;K+aw+M0dKuMv0aT^Vc#{U+{`C=v&Jm(mdAZ#jnapT7Z-&Ajf%Er{S8ZUfJ z^+915SB7=-VZ@UT=bUmR-lKELt7wvwUDqsB>?v0r9Z-0`_>0|3WQCOlyRQy;w&-Fj zB_SVL?w(&Zwhm^%XhoJ@bJaRtv@=>G!GhVYYC~~6(qTg2;B%8e4F!xJVumy@LqCu# z5NB#O+?G?_Qd}=3rCoZc`z5tn;gtN1#vFsd*6tM4@rpK)zWt(Gy&Z69|HtK>mGhKA zRn~2oDF;+p0s?txhd9%- zkn+iJSxrMXUuJ00lIAWwyb+^2v>>UtOxajES!15!6I6A8;jHl4m`4o6RZ|)5OD{dY z1m%*Pdp&w#CPOzruu*v9ZVa%j^7?uS3>H`>rm3EIutmt{oq$=Nid;IQ;kNSjn_9fM zr0x(?!Pap}O0jm4^~LZ`jEp7^!o1?s9#EH-b{V47Q0zG}u-C)06+3u~0)90P#518q z1L*qV@r>&}1xjmU7l^W(%p0y;ze#fC2E^=(Csa#8J)@7~NFmRQ*{(4%g&uo+)aC|J zjM-!xU_F8ZQh$Dn=SYFCJ;lclsakrheXCG5`4-VfXulKD=i8VTe8r8M6{nL6C9gxM z2`?jMHtx|@V+hluae72W7#G7%9rzg1+_ERMW^rHX@8Fp2cLDBYhqgfo2qd~=eQ=Ef6%8gG58oO^uKw*2fc9$_TFpfbGWUMynm zlP}GWfg{R7)a-d9yElsWqH8V^*48gj(`+0U>k#Z;_?&0;P5$c?56;yYX-uEc!_ZH{ zj16wlPPpggbuneZ%*)@wG~b)gvi^Fwe8*KxU)|cfSQ#h=Tq9(G*8;42LZPIkg;pP# zQP90$G<8p0FSYD;#wB&0v{tQovmOnjP#+230bp&rMX}l2b-2NN6XXwU{JzMa`{eIw zD*nk)pP%^}u*XiC_xMgd=fS$=MA>fcH)y=C$M>1!EAzGq-8wDM%l`5cjZVe><;FY!AC4i67Md|T{ZVBTo| zhlCSc)BXE_QyO^Wq5c0Kq^19MaS)-R7teI?U8Lij+dASc>h4h8j(E zl;{BT1q8l8-hBuq4qJ-fDV?}JkstpDOD)-^mrKs{wXs{yDDWoldWS0!40-Cy+}x)t z7$m(Vbbp@K4Viqp8Ma^7)2>=j7lO}E>FTzm`rxqwIk;L}bK=H0q^M&Pu4?I_(p#)% z!#}*}HUrbUNC1Y;rM?X^$6}Zmh3j_2N)*+{Qzp)$AMRF2s=YAoosoZX;|AuWa@<>7 zov;0HRUmIvhWGXP*1`1SNqd(~;m;b3pin-4OxRU56OB{%t6?h&O7MmPoXkBJ~$b0V-g-`?s{sBZ^VZKE@# zaL8Hr(;(tQwp)_lpCszNQiMHMZjK=}F^ibCd6=WS=rXWmp z$5W*(50@u(vn$(;UvuFfzi}TV!2%As4}U^gJFeeD;l8Dk0%Iua%l=D)s9K}3q3_sF zG|cDYi{l=zd4AF=RA<<=KpXc{UCz)rH>aCxGMjSSdl$R$lnTj1~3+p zOBezR0K8btDr@Nl<&15!PcPPB*wmJJb)DO#d-2Eut4}c>`;!n0nyw!#mcvyuYHYiQ z@*vBp>XW|KSCb|IK~`rQB3;fKF0~DrO;+;d=?vyws8F*9v>r*&xlD8k3*{ED#S?}t@MyC#UGhcJaxfsk@$%4TpXUwTH&1V-gOif6AMC!PwUu0NZhqv9UyZ)k+ zYt?NHPzw95AG;#xEbul;qU#LHOk{p#NauK0p-opxar$pUhJ%#}@bio^_q%nlHvQFu zW-=AV5C${&h>HxakG#q&MqAo_h80H_0qfn4Hy|5zq|97oOloJpd2R3tdaBAmuMAH0 za__uLr`L_qOkmWBQ!f3>Th-QbMqKul!CQ`~9jh&VkLJ^_F}`&8EZeI=&MUKKJ9bZK zTLnzL3zQsEt1EEPtSi#3***0Drmh@wn0{`!0*tQ5U^pnKJufzhcveECni+I?c(=@eWIT}+-*Ui z&G>9XS(nXBM@I~xT2H%Dy?*N#A#I+E2s4VXN{=ks*C-~|-ZQh!+jV7Ycq6CD3VKK9 z3j~4|dTOz5{>vIkkm>2yn1T_#vYFou${S-S_iOe!Q&__F??+Qnw=?ywkA8*}1*O3L zTGhGA`Xab&uOGG?xjVDNdR z-E3-@`M^Rzq<5nQ{xxePGf@97{q~EgoGD{NagpVh4`;;li4;~OjpSVuNuuS9Y!7g} zZ_|@^XTN=SRA2u=*P7{8*+Op-tO zH;CKKI*~S*L2Z4haxC@+bVkG7f5FspHE9mC5OAxAI(qc=DUb+IfSUQ`lSM&Bo@Tp@ z{7~MT`lDt2bfdLciT)#MSD;RfWOBc$5>b5!(asjXYv5&@8zDy}yx>~3v{0rw{I>Mh z8P5m5Gc`6UXZTHsW>>`!ETfj-Dv8@xb29B^cFO~?RNk6!X?P9#j?YHwudTusXxGZB zKmp2LI@nUbjoE_=LA@w%B;m^lMG|d{*Q^S9V5<`jg%8=P-KixUG<-ns8m0JK#pO6K zrwSkb(c&Ic*Go(vAThFfoZAshJI*;^g~3?N1`qS%O=y{k$PaLAIoB+*CTx&%yI}xb zmH*FKQ!dI@==)0{hdG=0pA;CLm5yWKoSq7VvDtBJjI4P?{i@L{AfvwA)o}1J#1bc| z<$^W=Xp!OofkL=<(CQ!khIK;LOvmSfF~`GApmnj|$wu=zDZUJha7WCd+S6Z2ZrL!zRU(71(pXmTeN!%ohYJ!Wbh&?qGVO|7%NK zg0Y9o!3ydPEqv4C5nJl%7JQcI^~q{o$COsaYd!5dG^ewEt($rY6_YF-5+&3=v85on z({>TiY@@6c=b5aYvw}KE(0aZi@a+}82J$GUR6#f$^JZAul3!-oDafNopQL3^hV{LR z-f758uVl|OZxj=DAMB>!=LR=I+Z<}*mY=+`%Tx7eATYv658wfgZAV=~b|F@S8-&*` znX4mw=3{qWef#;|gk&YDWIrrugnpg^s6va2bVUYmZ`h?0|A`R?&^FN@8V4S~0s9)Vf&} z;*|C*HB`zJ+4I!gl1kjeb4V@fT&6AMf8DaFh27pV(k`HAt){ zCtUs=)wH3;*t(K(EZFh*CvCl>z2ld7r}<~TuIB5yyht9wAlpXLjg5qlk+-?Y9E}e` zGLpM`tC7>YXtj$@aH~Ws8V81R^8(F()AWT(?3*TkVBO+?I~zC17Z{=jnxs^sq~Tq; z)cr~T@i_|WeXD03SBK9Syc1_cJzWiMfd!`e*!$RIE&LAWx$+5xQEu%KMMQq8{Ek*X zan$kd1VTA8Pz$TQXi)K(n)hah5pWC5Bz#as+fh$L3>_s(y+R{Qh2q@VbLB36@QYJ+ zESZDro(dh)bY*%oBdbFjXM58-MeUldQFkN!Mr&|Q=92*!8OXKRK3{_^Q?S4+<%YSEbsA`4iB|Y1L`hGpsW48 zBEn>AW3_zcK;^dZMoX8yXFJh%E*DPn%9DJgQNLn)y^@ z6haKD2&DFgwde+RmYTP=<{!}bfCoTU_Z&bM4r&H3>K?jmp15l+KD8P7#ka@Mk@eb8 zue?5A&8l-mSo63|+T^`1O|bJqQ^DLKNuJarlvM$a9Nj<=hk~N!aYusFc+xu8BY_5c+{xE`=o>Z z>14W~^i#t0=;n&Ke@148jKyZs88z2k2gsM(-ugJNfS%JpTRtI%Jq532_k?n0`W>cRG+9FcqQ z=hn(K(085;b!SYiscO1h@R${yWpSR0vo^)W>kI72pEexOb4^GK#CD1dnVCJsH08^e z^V;|~dh+gaK{63{xY!Z zyAFDvPh#fIxQ^RAsV=sr(<#;TGKo8+d(_sk+aC46J9qt>`dERidw&VPy3o6!#XZAe zm|d%7OEC4ETS`aUbHF^--LmAmUIg8`EM|%28(SJ2y$rdUYe> z`=6$0xZf@}oJ?6#Va!{~Ajlk|$4(EXKTV%2L6PPWDYD;+h{eUPC<$_4FdQu6;Sg){UO)z^`D zuOO}P4%Cf@D7~oC-a5#spMt2x_C3XM+ literal 0 HcmV?d00001 diff --git a/website/docs/module_site_sync.md b/website/docs/module_site_sync.md index b0604ed3cf1..31854e27297 100644 --- a/website/docs/module_site_sync.md +++ b/website/docs/module_site_sync.md @@ -140,3 +140,42 @@ Beware that ssh key expects OpenSSH format (`.pem`) not a Putty format (`.ppk`)! If a studio needs to use other services for cloud storage, or want to implement totally different storage providers, they can do so by writing their own provider plugin. We're working on a developer documentation, however, for now we recommend looking at `abstract_provider.py`and `gdrive.py` inside `openpype/modules/sync_server/providers` and using it as a template. +### Running Site Sync in background + +Site Sync server synchronizes new published files from artist machine into configured remote location by default. + +There might be a use case where you need to synchronize between "non-artist" sites, for example between studio site and cloud. In this case +you need to run Site Sync as a background process from a command line (via service etc) 24/7. + +To configure all sites where all published files should be synced eventually you need to configure `project_settings/global/sync_server/config/always_accessible_on` property in Settins (per project) first. + +![Set another non artist remote site](assets/site_sync_always_on.png) + +This is an example of: +- Site Sync is enabled for a project +- default active and remote sites are set to `studio` - eg. standard process: everyone is working in a studio, publishing to shared location etc. +- (but this also allows any of the artists to work remotely, they would change their active site in their own Local Settings to `local` and configure local root. + This would result in everything artist publishes is saved first onto his local folder AND synchronized to `studio` site eventually.) +- everything exported must also be eventually uploaded to `sftp` site + +This eventual synchronization between `studio` and `sftp` sites must be physically handled by background process. + +As current implementation relies heavily on Settings and Local Settings, background process for a specific site ('studio' for example) must be configured via Tray first to `syncserver` command to work. + +To do this: + +- run OP `Tray` with environment variable SITE_SYNC_LOCAL_ID set to name of active (source) site. In most use cases it would be studio (for cases of backups of everything published to studio site to different cloud site etc.) +- start `Tray` +- check `Local ID` in information dialog after clicking on version number in the Tray +- open `Local Settings` in the `Tray` +- configure for each project necessary active site and remote site +- close `Tray` +- run OP from a command line with `syncserver` and `--active_site` arguments + + +This is an example how to trigger background synching process where active (source) site is `studio`. +(It is expected that OP is installed on a machine, `openpype_console` is on PATH. If not, add full path to executable. +) +```shell +openpype_console syncserver --active_site studio +``` \ No newline at end of file From 6f3944d01aa54d3f15c0d88ce7f0c8d617f96f50 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 14:48:25 +0200 Subject: [PATCH 09/24] Hound --- .../default_modules/sync_server/sync_server_module.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index d8a69b3b07a..82c1dc178ae 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -727,9 +727,9 @@ def server_init(self): self.enabled = False except KeyError: log.info(( - "There are not set presets for SyncServer OR " - "Credentials provided are invalid, " - "no syncing possible"). + "There are not set presets for SyncServer OR " + "Credentials provided are invalid, " + "no syncing possible"). format(str(self.sync_project_settings)), exc_info=True) self.enabled = False From de8cfeff7f996ee2d33ca0b235ea3c60ccc66ebb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 14:52:12 +0200 Subject: [PATCH 10/24] OP-1920 - renamed reset site method --- .../sync_server/sync_server_module.py | 28 +++++++++---------- .../sync_server/tray/widgets.py | 4 +-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index 82c1dc178ae..1fee0b46764 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -146,9 +146,9 @@ def add_site(self, collection, representation_id, site_name=None, if not site_name: site_name = self.DEFAULT_SITE - self.reset_provider_for_file(collection, - representation_id, - site_name=site_name, force=force) + self.reset_site_on_representation(collection, + representation_id, + site_name=site_name, force=force) # public facing API def remove_site(self, collection, representation_id, site_name, @@ -170,10 +170,10 @@ def remove_site(self, collection, representation_id, site_name, if not self.get_sync_project_setting(collection): raise ValueError("Project not configured") - self.reset_provider_for_file(collection, - representation_id, - site_name=site_name, - remove=True) + self.reset_site_on_representation(collection, + representation_id, + site_name=site_name, + remove=True) if remove_local_files: self._remove_local_file(collection, representation_id, site_name) @@ -209,8 +209,8 @@ def pause_representation(self, collection, representation_id, site_name): """ log.info("Pausing SyncServer for {}".format(representation_id)) self._paused_representations.add(representation_id) - self.reset_provider_for_file(collection, representation_id, - site_name=site_name, pause=True) + self.reset_site_on_representation(collection, representation_id, + site_name=site_name, pause=True) def unpause_representation(self, collection, representation_id, site_name): """ @@ -229,8 +229,8 @@ def unpause_representation(self, collection, representation_id, site_name): except KeyError: pass # self.paused_representations is not persistent - self.reset_provider_for_file(collection, representation_id, - site_name=site_name, pause=False) + self.reset_site_on_representation(collection, representation_id, + site_name=site_name, pause=False) def is_representation_paused(self, representation_id, check_parents=False, project_name=None): @@ -1240,9 +1240,9 @@ def _get_site_rec(self, sites, site_name): return -1, None - def reset_provider_for_file(self, collection, representation_id, - side=None, file_id=None, site_name=None, - remove=False, pause=None, force=False): + def reset_site_on_representation(self, collection, representation_id, + side=None, file_id=None, site_name=None, + remove=False, pause=None, force=False): """ Reset information about synchronization for particular 'file_id' and provider. diff --git a/openpype/modules/default_modules/sync_server/tray/widgets.py b/openpype/modules/default_modules/sync_server/tray/widgets.py index 45537c1c2e9..b401411db5c 100644 --- a/openpype/modules/default_modules/sync_server/tray/widgets.py +++ b/openpype/modules/default_modules/sync_server/tray/widgets.py @@ -411,7 +411,7 @@ def _reset_site(self, selected_ids=None, site_name=None): format(check_progress)) continue - self.sync_server.reset_provider_for_file( + self.sync_server.reset_site_on_representation( self.model.project, representation_id, site_name=site_name, @@ -786,7 +786,7 @@ def _reset_site(self, selected_ids=None, site_name=None): format(check_progress)) continue - self.sync_server.reset_provider_for_file( + self.sync_server.reset_site_on_representation( self.model.project, self.representation_id, site_name=site_name, From 1772e7bf8d887c059d28ca923448e5fd925285b8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 17:15:47 +0200 Subject: [PATCH 11/24] OP-1905 - implemented exit on key interrupt --- .../modules/default_modules/sync_server/sync_server.py | 1 + .../default_modules/sync_server/sync_server_module.py | 1 + openpype/pype_commands.py | 10 +++++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server.py b/openpype/modules/default_modules/sync_server/sync_server.py index 2227ec9366d..48df5aad1b5 100644 --- a/openpype/modules/default_modules/sync_server/sync_server.py +++ b/openpype/modules/default_modules/sync_server/sync_server.py @@ -246,6 +246,7 @@ def run(self): asyncio.ensure_future(self.check_shutdown(), loop=self.loop) asyncio.ensure_future(self.sync_loop(), loop=self.loop) + log.info("Sync Server Started") self.loop.run_forever() except Exception: log.warning( diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index 1fee0b46764..281491eedf6 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -771,6 +771,7 @@ def server_exit(self): log.info("Stopping sync server server") self.sync_server_thread.is_running = False self.sync_server_thread.stop() + log.info("Sync server stopped") except Exception: log.warning( "Error has happened during Killing sync server", diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index cfa16012d02..e160db0f150 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -3,7 +3,6 @@ import os import sys import json -from datetime import datetime import time from openpype.lib import PypeLogger @@ -332,8 +331,17 @@ def run_tests(self, folder, mark, pyargs): def syncserver(self, active_site): """Start running sync_server in background.""" + import signal os.environ["SITE_SYNC_LOCAL_ID"] = active_site + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sync_server_module.server_exit() + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + from openpype.modules import ModulesManager manager = ModulesManager() From 8a37b9065e340be28a1a7c85a4be68ac8308e9a1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 Nov 2021 16:22:20 +0100 Subject: [PATCH 12/24] OP-1937 - added alternative_sites to System Setting for a site --- openpype/settings/entities/dict_conditional.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/openpype/settings/entities/dict_conditional.py b/openpype/settings/entities/dict_conditional.py index 6f277605700..3621c1319a1 100644 --- a/openpype/settings/entities/dict_conditional.py +++ b/openpype/settings/entities/dict_conditional.py @@ -762,6 +762,17 @@ def _get_enum_children(self): enum_children = [] for provider_code, configurables in system_settings_schema.items(): + # any site could be exposed or vendorized by different site + # eg studio site content could be mapped on sftp site, single file + # accessible via 2 different protocols (sites) + configurables.append( + { + "type": "list", + "key": "alternative_sites", + "label": "Alternative sites", + "object_type": "text" + } + ) label = provider_code_to_label.get(provider_code) or provider_code enum_children.append({ From 26b9bbaedae7e21b6f4346b403160a495eb53d83 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 13:03:57 +0100 Subject: [PATCH 13/24] OP-1937 - added alternative_sites integrate_new --- openpype/plugins/publish/integrate_new.py | 72 +++++++++++++++-------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index fe780480c2b..2f90cd7d667 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1029,31 +1029,25 @@ def prepare_file_info(self, path, size=None, file_hash=None, local_site = 'studio' # default remote_site = None always_accesible = [] - sync_server_presets = None - - if (instance.context.data["system_settings"] - ["modules"] - ["sync_server"] - ["enabled"]): - sync_server_presets = (instance.context.data["project_settings"] - ["global"] - ["sync_server"]) - - local_site_id = openpype.api.get_local_site_id() - if sync_server_presets["enabled"]: - local_site = sync_server_presets["config"].\ - get("active_site", "studio").strip() - always_accesible = sync_server_presets["config"].\ - get("always_accessible_on", []) - if local_site == 'local': - local_site = local_site_id - - remote_site = sync_server_presets["config"].get("remote_site") - if remote_site == local_site: - remote_site = None - - if remote_site == 'local': - remote_site = local_site_id + alternate_sites = set() + system_sync_server_presets = instance.context.data["system_settings"]\ + ["modules"]\ + ["sync_server"] + log.debug("system_sett:: {}".format(system_sync_server_presets)) + if system_sync_server_presets["enabled"]: + sync_project_presets = (instance.context.data["project_settings"] + ["global"] + ["sync_server"]) + + if sync_project_presets["enabled"]: + local_site, remote_site = self._get_sites(sync_project_presets) + + sites = system_sync_server_presets.get("sites", {}) + log.debug("sites:: {}".format(sites)) + for site_name, site_info in sites.items(): + for added_site in [local_site, remote_site]: + if added_site in site_info.get("alternative_sites",[]): + alternate_sites.add(site_name) rec = { "_id": io.ObjectId(), @@ -1068,21 +1062,49 @@ def prepare_file_info(self, path, size=None, file_hash=None, if sites: rec["sites"] = sites else: + already_attached_sites = set() meta = {"name": local_site, "created_dt": datetime.now()} rec["sites"] = [meta] + already_attached_sites.add(meta["name"]) if remote_site: meta = {"name": remote_site.strip()} rec["sites"].append(meta) + already_attached_sites.add(meta["name"]) # add skeleton for site where it should be always synced to for always_on_site in always_accesible: if always_on_site not in [local_site, remote_site]: meta = {"name": always_on_site.strip()} rec["sites"].append(meta) + already_attached_sites.add(meta["name"]) + + log.debug("alternate_sites:: {}".format(alternate_sites)) + for alt_site in alternate_sites: + if alt_site not in already_attached_sites: + meta = {"name": local_site, "created_dt": datetime.now()} + rec["sites"].append(meta) return rec + def _get_sites(self, sync_project_presets): + local_site_id = openpype.api.get_local_site_id() + local_site = sync_project_presets["config"]. \ + get("active_site", "studio").strip() + always_accesible = sync_project_presets["config"]. \ + get("always_accessible_on", []) + if local_site == 'local': + local_site = local_site_id + + remote_site = sync_project_presets["config"].get("remote_site") + if remote_site == local_site: + remote_site = None + + if remote_site == 'local': + remote_site = local_site_id + + return local_site, remote_site + def handle_destination_files(self, integrated_file_sizes, mode): """ Clean destination files Called when error happened during integrating to DB or to disk From 691faaf70d7abbd191d5ce7c22d188256c577066 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 13:04:20 +0100 Subject: [PATCH 14/24] OP-1937 - added alternative_sites to upload/download --- .../sync_server/sync_server.py | 8 +++ .../sync_server/sync_server_module.py | 53 ++++++++++++++++--- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server.py b/openpype/modules/default_modules/sync_server/sync_server.py index 66d4e46db7c..6eaede048c4 100644 --- a/openpype/modules/default_modules/sync_server/sync_server.py +++ b/openpype/modules/default_modules/sync_server/sync_server.py @@ -80,6 +80,10 @@ async def upload(module, collection, file, representation, provider_name, remote_site_name, True ) + + module.handle_alternate_site(collection, representation, remote_site_name, + file["_id"]) + return file_id @@ -131,6 +135,10 @@ async def download(module, collection, file, representation, provider_name, local_site, True ) + + module.handle_alternate_site(collection, representation, remote_site_name, + file["_id"]) + return file_id diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index a2cfd6f6b9d..af672e7a6f2 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -109,6 +109,7 @@ def initialize(self, module_settings): # some parts of code need to run sequentially, not in async self.lock = None + self._sync_system_settings = None # settings for all enabled projects for sync self._sync_project_settings = None self.sync_server_thread = None # asyncio requires new thread @@ -769,6 +770,38 @@ def get_enabled_projects(self): enabled_projects.append(project_name) return enabled_projects + + def handle_alternate_site(self, collection, representation, processed_site, + file_id): + """ + For special use cases where one site vendors another. + + Current use case is sftp site vendoring (exposing) same data as + regular site (studio). Each site is accessible for different + audience. 'studio' for artists in a studio, 'sftp' for externals. + + Change of file status on one site actually means same change on + 'alternate' site. (eg. artists publish to 'studio', 'sftp' is using + same location >> file is accesible on 'sftp' site right away. + + Args: + collection (str): name of project + representation (dict) + processed_site (str): real site_name of published/uploaded file + file_id (ObjectId): DB id of file handled + """ + sites = self.sync_system_settings.get("sites", {}) + for site_name, site_info in sites.items(): + if processed_site in site_info.get("alternative_sites", []): + query = { + "_id": representation["_id"] + } + elem = {"name": "sftp", "created_dt": datetime.now()} + self.log.debug("Adding alternate {} to {}".format( + site_name, representation["_id"])) + self._add_site(collection, query, + [representation], elem, + site_name, file_id=file_id, force=True) """ End of Public API """ def get_local_file_path(self, collection, site_name, file_path): @@ -919,6 +952,14 @@ def connection(self): return self._connection + @property + def sync_system_settings(self): + if self._sync_system_settings is None: + self._sync_system_settings = get_system_settings()["modules"].\ + get("sync_server") + + return self._sync_system_settings + @property def sync_project_settings(self): if self._sync_project_settings is None: @@ -1004,9 +1045,7 @@ def get_all_site_configs(self, project_name=None): (dict): {'studio': {'provider':'local_drive'...}, 'MY_LOCAL': {'provider':....}} """ - sys_sett = get_system_settings() - sync_sett = sys_sett["modules"].get("sync_server") - + sync_sett = self.sync_system_settings project_enabled = True if project_name: project_enabled = project_name in self.get_enabled_projects() @@ -1064,8 +1103,7 @@ def get_provider_for_site(self, project_name=None, site=None): if provider: return provider - sys_sett = get_system_settings() - sync_sett = sys_sett["modules"].get("sync_server") + sync_sett = self.sync_system_settings for site, detail in sync_sett.get("sites", {}).items(): sites[site] = detail.get("provider") @@ -1434,9 +1472,12 @@ def _reset_site_for_file(self, collection, query, update = { "$set": {"files.$[f].sites.$[s]": elem} } + if not isinstance(file_id, ObjectId): + file_id = ObjectId(file_id) + arr_filter = [ {'s.name': site_name}, - {'f._id': ObjectId(file_id)} + {'f._id': file_id} ] self._update_site(collection, query, update, arr_filter) From f19b39185d47b4666a8065bb155823dbcaefd4dd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 13:04:54 +0100 Subject: [PATCH 15/24] OP-1937 - fixes to stabilize sftp provider if wrong settings --- .../sync_server/providers/sftp.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index 07450265e2b..8549c1c981e 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -1,8 +1,7 @@ import os import os.path import time -import sys -import six +import paramiko import threading import platform @@ -37,7 +36,6 @@ class SFTPHandler(AbstractProvider): def __init__(self, project_name, site_name, tree=None, presets=None): self.presets = None - self.active = False self.project_name = project_name self.site_name = site_name self.root = None @@ -64,7 +62,6 @@ def __init__(self, project_name, site_name, tree=None, presets=None): self.sftp_key_pass = provider_presets["sftp_key_pass"] self._tree = None - self.active = True @property def conn(self): @@ -80,7 +77,9 @@ def is_active(self): Returns: (boolean) """ - return self.conn is not None + return self.presets.get(self.CODE) and \ + self.presets[self.CODE].get("sftp_host") and \ + self.conn is not None @classmethod def get_system_settings_schema(cls): @@ -108,7 +107,7 @@ def get_project_settings_schema(cls): editable = [ # credentials could be overriden on Project or User level { - 'key': "sftp_server", + 'key': "sftp_host", 'label': "SFTP host name", 'type': 'text' }, @@ -421,7 +420,10 @@ def _get_conn(self): if self.sftp_key_pass: conn_params['private_key_pass'] = self.sftp_key_pass - return pysftp.Connection(**conn_params) + try: + return pysftp.Connection(**conn_params) + except paramiko.ssh_exception.SSHException: + log.warning("Couldn't connect", exc_info=True) def _mark_progress(self, collection, file, representation, server, site, source_path, target_path, direction): From ab36fdfe711400059e103e6475c43f60a6de754c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 14:00:03 +0100 Subject: [PATCH 16/24] OP-1937 - added documentation --- .../docs/assets/site_sync_system_sites.png | Bin 0 -> 13118 bytes website/docs/module_site_sync.md | 47 ++++++++++++------ 2 files changed, 32 insertions(+), 15 deletions(-) create mode 100644 website/docs/assets/site_sync_system_sites.png diff --git a/website/docs/assets/site_sync_system_sites.png b/website/docs/assets/site_sync_system_sites.png new file mode 100644 index 0000000000000000000000000000000000000000..e9f895c74347ac8f311e81ad3f1dd6a388723695 GIT binary patch literal 13118 zcma*OcU)6V_b&>fB2p9)QK}#yf`HO{5vBJk9Yl~08fxgGpi~i2>Ai*o1VSKmP>|jU z20{d+210Lvz}@kA?>+Z-@AG-j`9p`xcZa}e7Uvo@&$qU$moZ1 zAH#c&Q%KO#8GqhnP)N@ntgm&?xNS5IC=~DRn4dLGpk>!vBPcus1qEF_Jxv`P z@_9|P4V3U7ZhirYU=4Bv@9&rrYIh3PBr*f z&{FE?=t!^{hvM~4Ds#cl~NyTJB;1#?u(1ypd z&XW4!PozkV9X%bXdHRNe)RNegBLa$?SLEgVzMST}d^JVxki%sJ&kb z_evkRSiAbUmwFdvmaylyMPm(2yq)r0ynOk&QEXb%!X{qDRR*jHy{lKcK1K!4l;g}) zSP?8P5G@{mMTiR}3`bjR4F}=1NkW!#!Mumi=(5qf%;V^h#q6XNqGH2y^;*C_ z`RR7GN&~!jnELO+3{`_?bD-lw+vw9C&7F3oG@*XBoEARe;qM>qL)9y_iqKUe<8v3H zC3QJA_3%H@()1PEFe~Kfx_T5mjbY#1#U({^wd=y;;4bMI6Q7QO!ZJBq6fH@Ox>mC} zg-{(q7Y~~bEcHe6nR_#aQa(BOlAHNw@_QEX>}ej6%D1_{hF2D^f0C8GIO^Q&lAbj= zJB}O2PSmQ1n%QRk3D>7TX=Kw=r*1|XYk0g!?>x~N5!gQt>`j2cn>u{#&S?kCw_x_a z9$8>(ZFaiqJVcLv*;N)25t{8OHQTpYlP4;n&Dm{T$m4bijlqm@to-Q z!mK6zQIQhDtdn-X1vkRL#yerPPe5fe6S+mEOmtNz(wDOG;v)9FjAbEkaEEo;OmnWB zQ<{^h`>5ZPoxguSDlJ#+v@+wJ%>~)-MMQ5l|ACPq{j7l?uEre0)FSEG_uqG2jni!2 zg6g9l7!?PRKaXf;US}SVK?P6ffGt>Re!r@4AIR{VD%#)7dZXLy_LP{S^(88EDGb{x zO^;;(gk*vrwkumeEJV=-%{C8~-OfEq)2)@jt`jbx>b)(l}_KCKMFoK*Xr2 zy|3*XC2Es7{{D3)qYq9KcFWm5I&_i0W$ye^G11rhp}?$Q<2*~}jP!=C|AN1ZN?g{v zOs$suVgV;t^jAQ4t=3>9zt0%D3{WS<&Vs*dMz8pJ?eWW!R{T|*?CaCLO%Arh))BdS z0;V-n2@?xfaSdq%Cg*8^P~xB~oJk}Dp6J?EXLbT=8HK~B{u%(JtqjY5vgput))qAvWEzC5R=Q#o{F*WXt?YYeu)t&^OdW6JXtIVT<@%P{R?q@Jw&U|Qs1wTt~+VKKk zukFuR1k<8%o#Fv)bHDELw}bfx76=Tjx~>vURJI9M6{77c6i3$S;zEU_d-C8qkrD8|B|*&ICx2i@97*lwr`^WN{~G&$MpmG$MDuVDu1;btIOyY(5le%N|Th_nlKcwBY>3nHQhG>pb~H#Lw; zXUrel+=H)ZW~(3QxXXUkV0FC#YIg)qUAUm@r2^|DDl2{pY`(ng95J9 zXy&&%y1KV)CwlzGuAfLCg3dw_eysN$qN3 zLBF7o1CzL+_@h8VXUQ{@=S%e?#=oF8k@!6FPKIx1EjF+fYU z8+uRb6v!*cltixnzvJ4C{}S@hL+#IIfXo80udKg8P@$a^*2rSLN&mNUv67m~XS(VG z>UyYf^2kpqsf9jX@{BdLwPmIC8h7>B&0-NGX17BPEQ!1~|NQw=>XqGmkAa2;1YJz) zho^6lC;)GR=q_IAsf}MV_AA{`Rgd;0pc-`m-J#fhQZU{h9#z##AD$F!8CVxRav?}~-JGe|qg zHhvD19S`yX-hHqD@3jVte&S0hBmq;oj zD=$eDVM~+jgTjsEjIIban|EM}m~AFL9-Fx(PQ~ecD^fg>9N9iLHo?Vw{L`0eV&HlaDZKoX5x%IX?UtZv-#oO0)Zt);oy354Dymiw{A=M!7j*z8L zm`UEPMB(wcYX~Y}s+HZ=0qdkqD=`u^1?{+4!o$7@rm*vX(a<+ z4uj}i4_sP!SG7cAtd*6ZWj2JS&M(pUTw5~*gRp>Zx6L5VS;z&OdF;Me`H-o2$?9nf zN|;4Ea%NAd&Rzr6e@2<&c^cHRM!83rn@(k3Fk4D3F$}@3?{{x@J^bAeo3j; zIY&g`)+IpBI&b8Ia_6SciicHV>l`&FU5ej&r{h%pf%2mlJPjSx6MUdb9w~(Xz_XFYy z4aE@q`d|B>7ktye2-LeUVh|E!kiz8qh@VLzkIDfp=ZEvDVPlv}b$Kc@B)|5C z_Qap+MJjCr^IyCs6|d$axe|5`puz5$7!v1QooAB_Yi_VSBloj50in_td-cRB-)Dx>sfJ%hXy`Ii8NEwK=7(iUcnP6u%yYL?Q>Zb|9nHGd-Ws_Fs>iRc=IuTLR!GLAjB=RF#M|v-1fiEUbSk^_{8b{gH^OB%%E5gM@B0|utwq%w-NW2sgB~j6vn&*KY zciYhw$_@oc#q^f>-pLXtkAAm@#rB~GzhC;pA3>y;^x^zgG{tC1_L*cFlU32GCPJV# zx2(?LGrv$7H<^#hgWI2-JLk2;^S`G6+%qV*6nnWGkG4yol z3{g9AmUMkxMwSqAXLYh5rMkIzZ}EfQ$DJuK*304T!O<^tuHwXA1h)Iqsj2)DNgK-{ z*F=_n&(lI`+*b~W3$~J4$)}GQI~0bavdbrmREoAX_AfA>)I0V}&Gv)_K3yAj@9)D(8-xn-1j1)|? zU1(Pmn}r4q!lvAQ4+!lt#}oxm7@?mf;-4Zy6RQ;}blu7tvCp@!`-l5OlY8Ot9`S8z z!2g$HdtNk;jv^oe2$4=AEvyY&aDl^-t zH*>@3%wyMfZX8(deGOv*YDhDgC2#KqU&kJz_ZMvJRs6g)+V>~4>y>kLD3jFIXyj$Z z6XKWlM~@|Iyng$Xwi($LGDbAOg7(YWrcMKY!0|zm|79#nt^X ztJFxRI7Yh1^z=xS%6QYx`l>xG7JrDRL!hA8J3m9R>a(hF#xA`lZnU8&=a!TO4#f61 zU(W`}IV?OG%X_jjh#Zjd_%-=MX70lfwpJz6%75tS)E{t6;8>xk$ZN#X^bhXQr;Rn3 zlw;7IVmF$*HHS(`mu6+*36K6A$Ypv8#k$?;l3 zXnOS_%G_z7VOAz(qNP+)C%sJi1|2Xmcq5ltRRo8;4Pcg>&`o#qK8Q1S@JA7ONv3Z>tUY!jMoIHM1ae@`Y zX+bAdKZGYreZZ66U!{DA2k%Y;akeENdHct*bv{lsd&8Q@Gv9CjVmp<1VR=@hSW^=4 z*xq^Bhza-3EH8BGD|qa$Ff2UJ2ihtdISuXJ4OjO1m{%_SW?+;11ZmWoc4F;&^Cz%$ zll=`T%52~@S!Vri;HSr!(1VQ4-1ph9RO ziJR@r-#i~FgJNJD1Dyh3Quxe0WrmNExQ@Z#HV#uUalPnvFx?Ec&b;1sH=3*PalWFb zxW&%%lN3Y3ddRQRSiC}zWmWWnA8y*Rdk3Z6J^%F2&w=)c?h^({1gSvwM` zf8?9gdWS&3_eFvv)=oG`_n9-w1C}-POSTQm03RNQ2t{ysgeQbzv+{f%BJIxE=#(z zJ>C+g_`iaXn79P4^)!-Sh`n^0Q+hS|SJyfz6&_-{Ct1!nUHE0uWhjSKe$-4wUZkrG zoA*8WuJQj1dOm;g#1B%y98!L&^y=TDbIKT1w?kKgpW8??A<}bw+R>>9!l!5kRZA4m z@sPowpN;lc#?izN_83MSNPdoVagkXQ3~Ca^Sy-3^{a*W0@+8@8-;sC(m-g(P%*QHJ z>oy3b=lwEL)qvyR+mego%{+wm%oP6pE8DEYQzpVLM!(%C6Rt>A=_pR6+$=wIBYUck_ z#r_u>|Mp6g#p3Do;zVTBA}z=bBN}R7XqoH>4d{aUVmI4a9rt?mP2JK)(yZ<8rj3<# z7CEsgf>rP8iX#DZA3*c(G13wC`$yi{|B`t)GNrC(1X3&&?8P#TCpp2_WqH`VfO=JM zv{7leILx*%Q*$&zFQK%)ew>YDJ+}+n+g_;JkXeyv4=8?NBcl|_ za=hBnl#OhE`pP5}CuW^KF9DS&tc&7&O4fP_pz3_TCeza*)h{8Zewe3g9~_A6U>3vmU&9x*pLMq@WED~O(7I~$sC9PBjOw>Nu2_xI^b>kPxpKjS!9b;br| z3$7?`5A0n}-0$6qdOp>nrcg$E;eCcXTWEjqK3JGLARU$PW!>k}m23Ox6CxmoQXTa* zx*Ib@9mI`$ruD<4TJ67l=8E#=#CW)tswc@O0;!a0uQ zFHl}NZn`e55;fUEFLTrwnT|N3CCqVEv}HpE<0Qm6hSkJjFIuIl)Hu)u-cO@FKL0d@ z`Qsh)iR_VRXaxIvW@4HRY0uPv+7?-BuAF1LKrug-%IlaR=XT!MCsJfu)BP5)%BSkv ze6CtU=scKwZSpLf)mpXEKcc|&%!dNjm+bdJ`Xf`9Dh{(%JEB= zxX?h_SmF5k9EOh+5joX%b{SH`B{dXVzH9~Y%U;=<*OL*#kTBa!DL*u0hG2!eW=Zt# z-=;*d2ls!z8?H;_5gYp{I19R*5uo6OQVUvh+JYy^Un6OY%WAWj@dtnx9nV!ILp3Yb z4?SB_l1ppm$cHIjn@!wBw2bH#uly-ZSnZjBk9wh)Uh7@HOtV16k;U-pCr4L4r^=%p zDnE~{AhH-OEfxFrK9x&-ci&y@T3RPWo(pW#4uHMiIVx(?r*BNKN(pNogOR)z`7Mfz zR|0=RDQTCn!*>+sB^&d8LIdNM`YET;^|F}B0w4xBmDq?ozVOn6KJunBn@HW#mdh0qOWDJx2M7oZW@?$R=sL&v1y`no zaiOgFA$tQ@-6i$XFF6;!9xyyFD2CW#wT~s)bzJw_pH%Cz%H}5*F>&#v`g0nWJ>?Z^ z{2oj)8E#|EsK{8;a(_;sLq?_aj|aEjr0OR1!59Cniu^a1`5(COpI7At&3{9H(rV7% z{rRtFGJC+(UD-gS$u z1WT#xAxDi zwE6)BY;!`xv-mngo(mfgRnIfobhyhIW2o?>^DWW#E&eBeyyC~n%$S#59-M2URkl9{ zNQ7r`FSNHg=qW5r4AgGoy^wIJ3+>8E$5BCcVQL*Q=h|A?M1RsgkvYO)SFKYhQP|+4 zlCZTe2`f5(zZf{@6N~9}0Wu%yn>W*EMuKKDuc1Dzbl&yc`mAW$eF<)(92j}Y=(zF$*$yedUVz4&cDW=k5rb_yS#VxYXiYX zeR9`La;Md88j)>3L@zq_Gb*n2+2VsbnSQ^V==NPMU>=#gfuN*QIoiiS}v$ z86`bXQp?h9v@BhpgO069d@A#h=cDakU0nedNwb+dSv_3Y#$%_Y>ZGAp+O>vAte-L+ z{qx89e@o&1?cDgkn4u|vug^&!*u})WzFLwr;{ENvY=#>QA0y5;?2c%O1q_inK6k1E z9Ax+yup6%x=w&M~i}Tyk9xD#7jJ_BfnV#DRs1;m{VkD;$DdV#}z7kx{(4}{c23UzE zunno~qOOUH1?SCo>DK?Hq5@E9?R)q@znWUFv!5P&LKGKih6*&2 zO^oHxDFDxz7;iFeB`-Qk$(ciGaS4f4I4X2q6CmyNqmqk^joHLl`vDo*Z?6BOdh~SR zGesrFM$MLG?j%(Es&G5`hx2u0Qvs#FLv?X1EFi$@E`b#oY6IRzz%Q4+8_?GRF~+Nz|6HP@pj&u zxSPpd6BK)3$T}<+-4Ny6{HhDJ+re?)-_l;wbMzz6s3Syt6;gAUwPZ5YJpJG}>(gS5 z@D+4HO6u9gHI*0Mb@MhID|BqsM?w=CQnkCA+axnF=h?Tl2hq$_+wCylk49w>48Z*);IZ#q82~inlg6pB?oXo zz$ylk>BA&+BWv;dLG*D^9%%MuT0VZ$=Js02wTaUc08Fjl8{v|)&3$~KP;Gp3g0sOf z6|{3uqTua$9(2OK9^A&pVN*dLN^5N(GSV=kgFZn{?U#q$|K@+>CfQsX^?I`vaK|6x z0aF;~OF>d~7amRRb|#C%8Z(omb2*hK?AhFYPS{O}zzm2J2K*(PrUEfyjrNGu{>mIO zkY@)y8z`MV=9SokHRd8{`It8XV(GeHro50QAuyH?P&VY4B~vdOdh=&Bf%nRWeBS(# zH1nE7At!T2m~q!W!SYz*SS7ZK`c+K}(prAe3IYC%QS;$CVP2bCWcBHl5wtYs!EC?F zGKY5x?Ra`Ol)vitTktcN`jOIAu|2WUgKbQb26EI4wKo(wcPFhOQ5R9Nb~PKEl`65Y z;WzKq2G|vzpHrID&t21aV&8N%w4&pyB?wcS8u=kdtt@Jg^J%k2A?S;%YUQ0=Oh#na zT;}N2%Z%03kQyoc4zLrruF{3G&wfyN`b~%8MZ>IObs>~qp0+3{8>(z$cWJ=n_IwC5E0J-|nGY zW)rLO-ik)eV!h0fS_F0ipXh3AYg?Sj5&Sy=pZ0treylYE@L@wNR@Y~9<_IjiU#d?v zaW=&Gw6wIE;@SxbbTufdr0#i5jOrIY0RgSZs_MN+$KJ8YvE_j^D=yqXpEl~lw3e7% ztCHF4ti|Qwd{k}6znUduBV5R4@0OQ zvoP-p3xq9eI_s+f)_S|boBmYMn5X$(L2&kvj`pK!#D z`My+9!ftY+waW$_rGrp|_WWJ#u@?C=Vb`5gU@71V8#cNdp~vJ+2iT1=9F0QlgK zg9GUHLB0FhWF3B0O*9B^<#F60OTZA$Uh!~tm!*4ukP#{(ArV6CP1`O{oL*O^QhLMU z{T&vz7t;1pWXBM;ie__KPMEFKyub*=h)~8{4=_3$k4u?tm6QIdeH9P$t!OHvO6IK> zv=zZ6s4cX6@t^$&@?9#wh1~guN!vO!?B{za3v8@#&t=ji!!bGOl7=)EmXt64@29qT zs__FUC6$%BmJ$>HT%`f7M@T)`V1AHzeO-S`TYh&_JN{Mr@K~sE{@id9Vqv~amTq{} zv*J}Qi^3m4EvJ-#sYGX2<=i5nhmL3EPkP%CO$&fKvU-*NIUwxC`N6^$0B^Fb6@Yi+ zo4!W3oCa4nSF!yv$nhju zOz$+hyWjf?do4)>Fgs17jr~WF_kcZ2CNm_^5fLc+Es)CGpSHVyyfpbPhD5ZhR@cqc zm-EAz*~$`d1pp2?Rk5VzbrWeKO=;K)D38n^}sF&Zve_E)dc$Ue_+>v&VRC(+C$qlDFZ)(w zR;8PE0W~3{VW5!~)3`g-IQ3nweNwKpPwqo&pR7m+H4cZX*@?U|J#3xaha@LkPOxF2 zi%T!kL#3G6K)X0><}~N3|8SgXoE^o&!Rv})3z`F;SV_n@W1%h`py74S$>0z7QDIJH^GC2ON$|)TS9bqM>v*af=oic3%`$g2e-2vP9 zfKsJ*ZeFf^M&~E|@Wc-_G2S}1!1ALtoY!SqE1A=nqu{sWxUYj&oI2j9Q3OHS1O0p7 zM0>c#MNOu;h);Doq(O6iuWm2|ayZ>wu~d#c*JA&=u(%kP)nZvdq@^W%Nm2AE{FXZV z7W-9FucbWHyu#R|eq{Nh!9l!asBV0GJa`>$+W=&s$YNA9#e8+jnM!r@tLTX)LRnfm z^ZL~L{^Zwv4Hx_FJn2u4tP(jc@j43y6G|NoqObk<_!BwW*r{SO!^14)og<*E2il!9 z5YJLzpr?tOoV~p!RDVwXg6E{}U3tnPWmX!7Sy~^^Oys=JvqHC41UA9BFZx>-WBp7l zpZ^90$|y4**Xsu%Xq?fT&7$ac($)yIXXx$y>$Tr^J6o9_NXF)-rCcQs1ja zx!8IiGqZ`(9)IP$)9I&wxa5D8t~unfLu5AFetm;%T~VcKGozqX2JV3Em+~cz+oID; zRo@)ukr0mlwR#YI{mRC&KklqIL57GU_e zu$jUq2lt1{UWAp?R#f0j=jBs<@4y}P_(yV1ULr#m*3)W7Xi~dPPYbLN2#P1t%q0f= z@TJs3!ww7#CZZC1B7>&oi78dxvs5e1X}5&(jci6?$`JNN)To--E9lbJgJ$c6B3Oct zvA-EkhVuUjaUC?mVomQBos{dIPh|Qy(HZ@|=M3CtYk$H(7rrBGc7dTVGdA03&mD1p zhF9l4{$#7LA#wT5zO&nJxrWE}D%jMRd&twGJFqO*b$9QD%t=IGV$_!uzJ}Xs#}EwL zv5@sBXUL|<*;h#$>9W~TnZW06(Z1Q%W_hW1DO0K1F>pNGOx%8i8?gXH*Gz%oxUMf3 zv8f}1=Pv+v5rHJ``8^hN3_Zc=67Wj6@1d-jS6cJFxN9UJQA+j9Q9ab`pXVKT6h{e#3UQ!5h z9&XO|8g2dMsU9En5Z!ziGXCIY8jBOCL9;1mHnA%aJIpk*#e9?!$_9Gn!&MGcw=bd& zQPL%x+oJPLMfrOpUn#9I)lI>3ypohY3&(z@W?WzWX}_w)_>}|{v4aDOWCA$rLvBZ; z`d=4|#Ad4VfwYdq-&8y$RHdEe4*Fxq~c}Dvg0zziH-f~p) zE6Xzgh`?-K>IWKbqb^}(6}JkzrX?ZK(%d0vo^kR{M3KH*!Qtb|f6Kg*DDW{twcm6q!MockT zVRO1?yB-G%k=-6229f>FdoiQyZu4ludXc_!QNwZeqq)C3uF=k$PX0a*W12p2OsbQw zcmotyw!72%u)=F>;&peL_69lG?q%SPH&c^`SBnQ$H~7Y=<>q|0bG`WuZ29bwBv5A^ z&!*yYu8OX8$+b&a^&vA2M4OjTORI6>?)NM(^ftCocp8inu^(eku-WQ9%6yu_{`Z{P{G+ALFH zsE3u|WoXjn%d5s5IAEvWdrRVl(zh)x@4Jui#fwx|xm%MIQa5LQaN+Jn6n-ZGG6k^^ zelRxvqyb`E^2pzP0cxL|;HKT7DE7GI5ZS#Brjiq9sVQu>a*W7&&jK@YA*O>xw|2gh zlQEIalC?hZ>Z0Uu*4M4=v=*I7d2A1+m#M2(3~fKZheO$N;NJzW*jxGfbD3ghT@UW< z8gq%hBz*I&4JJ~X^kn0D`bVQ&vm}L!>DVo6EBMP1cA1|AE2#0LBS%fWNAVMKw@izV z?(^QEP^xGfjR`;h+vQ1Vg`O#gJb}33b}HG7*F-LsrYMe_G&I9Lf4*UM?PEFe4uB7k zDV-Yvt}JGC7O9d-01W7i%j{YU&)K0Kcou8=g!ZCp)#^+$F~r(;eB6^7$f1E97QUlt zk^-dl;y`9@Hu4Z~qNOiF3v7wcpf<;Yw*N$|Vvex#k@pLlq|!i%>p}S?b+RbNJ)vyq z2?r@fufdO-jZcp`V|CrLu0Nj@xkb*o?z@RhpRfIGUuPlAl5;V zpOGvab{jxrl%$qJ2>ZC=_Y29Ng1UKXhZTUlcungk5c;$onsZ^g9QZhS<+?lFAE%%pdH&*rXLJM_vL>qHgc~ z2Lq!KAe|ij&!0>DPuPoR&*jNHlPOZ%1r_7}_h%U4S?~V2Qu;p!{*Rwr{Et_^lTH;a Y)~G@38KWNnZe$uybRWYW*}eY10Psf-VgLXD literal 0 HcmV?d00001 diff --git a/website/docs/module_site_sync.md b/website/docs/module_site_sync.md index 31854e27297..4e3c8e4ed91 100644 --- a/website/docs/module_site_sync.md +++ b/website/docs/module_site_sync.md @@ -27,6 +27,38 @@ To use synchronization, *Site Sync* needs to be enabled globally in **OpenPype S ![Configure module](assets/site_sync_system.png) +### Sites + +By default there are two sites created for each OpenPype installation: +- **studio** - default site - usually a centralized mounted disk accessible to all artists. Studio site is used if Site Sync is disabled. +- **local** - each workstation or server running OpenPype Tray receives its own with unique site name. Workstation refers to itself as "local"however all other sites will see it under it's unique ID. + +Artists can explore their site ID by opening OpenPype Info tool by clicking on a version number in the tray app. + +Many different sites can be created and configured on the system level, and some or all can be assigned to each project. + +Each OpenPype Tray app works with two sites at one time. (Sites can be the same, and no synching is done in this setup). + +Sites could be configured differently per project basis. + +Each new site needs to be created first in `System Settings`. Most important feature of site is its Provider, select one from already prepared Providers. + +#### Alternative sites + +This attribute is meant for special use cases only. + +One of the use cases is sftp site vendoring (exposing) same data as regular site (studio). Each site is accessible for different audience. 'studio' for artists in a studio via shared disk, 'sftp' for externals via sftp server with mounted 'studio' drive. + +Change of file status on one site actually means same change on 'alternate' site occured too. (eg. artists publish to 'studio', 'sftp' is using +same location >> file is accessible on 'sftp' site right away, no need to sync it anyhow.) + +##### Example +![Configure module](assets/site_sync_system_sites.png) +Admin created new `sftp` site which is handled by `SFTP` provider. Somewhere in the studio SFTP server is deployed on a machine that has access to `studio` drive. + +Alternative sites work both way: +- everything published to `studio` is accessible on a `sftp` site too +- everything published to `sftp` (most probably via artist's local disk - artists publishes locally, representation is marked to be synced to `sftp`. Immediately after it is synced, it is marked to be available on `studio` too for artists in the studio to use.) ## Project Settings @@ -45,21 +77,6 @@ Artists can also override which site they use as active and remote if need be. ![Local overrides](assets/site_sync_local_setting.png) -## Sites - -By default there are two sites created for each OpenPype installation: -- **studio** - default site - usually a centralized mounted disk accessible to all artists. Studio site is used if Site Sync is disabled. -- **local** - each workstation or server running OpenPype Tray receives its own with unique site name. Workstation refers to itself as "local"however all other sites will see it under it's unique ID. - -Artists can explore their site ID by opening OpenPype Info tool by clicking on a version number in the tray app. - -Many different sites can be created and configured on the system level, and some or all can be assigned to each project. - -Each OpenPype Tray app works with two sites at one time. (Sites can be the same, and no synching is done in this setup). - -Sites could be configured differently per project basis. - - ## Providers Each site implements a so called `provider` which handles most common operations (list files, copy files etc.) and provides interface with a particular type of storage. (disk, gdrive, aws, etc.) From db02c03394f1d5d159152e0873059c6c7146b387 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 14:01:51 +0100 Subject: [PATCH 17/24] OP-1937 - fix broken import in Python2 Not imported exception in that case shouldnt happen, as sync process is not running in Python2. --- openpype/modules/default_modules/sync_server/providers/sftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index 8549c1c981e..267e23f8fb6 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -1,7 +1,6 @@ import os import os.path import time -import paramiko import threading import platform @@ -13,6 +12,7 @@ pysftp = None try: import pysftp + import paramiko except (ImportError, SyntaxError): pass From 1e1b9b0416b804e815785fdfa83e7acf38e8128b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 16:19:04 +0100 Subject: [PATCH 18/24] OP-1937 - added file id to DB --- .../modules/default_modules/sync_server/sync_server.py | 4 ++-- .../default_modules/sync_server/sync_server_module.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server.py b/openpype/modules/default_modules/sync_server/sync_server.py index 6eaede048c4..8518c4a301f 100644 --- a/openpype/modules/default_modules/sync_server/sync_server.py +++ b/openpype/modules/default_modules/sync_server/sync_server.py @@ -82,7 +82,7 @@ async def upload(module, collection, file, representation, provider_name, ) module.handle_alternate_site(collection, representation, remote_site_name, - file["_id"]) + file["_id"], file_id) return file_id @@ -137,7 +137,7 @@ async def download(module, collection, file, representation, provider_name, ) module.handle_alternate_site(collection, representation, remote_site_name, - file["_id"]) + file["_id"], file_id) return file_id diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index af672e7a6f2..9ff706d6ddd 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -772,7 +772,7 @@ def get_enabled_projects(self): return enabled_projects def handle_alternate_site(self, collection, representation, processed_site, - file_id): + file_id, synced_file_id): """ For special use cases where one site vendors another. @@ -789,6 +789,8 @@ def handle_alternate_site(self, collection, representation, processed_site, representation (dict) processed_site (str): real site_name of published/uploaded file file_id (ObjectId): DB id of file handled + synced_file_id (str): id of the created file returned + by provider """ sites = self.sync_system_settings.get("sites", {}) for site_name, site_info in sites.items(): @@ -796,7 +798,10 @@ def handle_alternate_site(self, collection, representation, processed_site, query = { "_id": representation["_id"] } - elem = {"name": "sftp", "created_dt": datetime.now()} + elem = {"name": "sftp", + "created_dt": datetime.now(), + "id": synced_file_id} + self.log.debug("Adding alternate {} to {}".format( site_name, representation["_id"])) self._add_site(collection, query, From 8ba0e606711c0bc440dbaf43750bbfeb3b685e9f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 16:19:23 +0100 Subject: [PATCH 19/24] OP-1937 - fix - integrate_new --- openpype/plugins/publish/integrate_new.py | 98 ++++++++++++++--------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index ab79d95fb7a..c0a90ee78e0 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1030,25 +1030,6 @@ def prepare_file_info(self, path, size=None, file_hash=None, local_site = 'studio' # default remote_site = None always_accesible = [] - alternate_sites = set() - system_sync_server_presets = instance.context.data["system_settings"]\ - ["modules"]\ - ["sync_server"] - log.debug("system_sett:: {}".format(system_sync_server_presets)) - if system_sync_server_presets["enabled"]: - sync_project_presets = (instance.context.data["project_settings"] - ["global"] - ["sync_server"]) - - if sync_project_presets["enabled"]: - local_site, remote_site = self._get_sites(sync_project_presets) - - sites = system_sync_server_presets.get("sites", {}) - log.debug("sites:: {}".format(sites)) - for site_name, site_info in sites.items(): - for added_site in [local_site, remote_site]: - if added_site in site_info.get("alternative_sites",[]): - alternate_sites.add(site_name) rec = { "_id": io.ObjectId(), @@ -1063,28 +1044,48 @@ def prepare_file_info(self, path, size=None, file_hash=None, if sites: rec["sites"] = sites else: - already_attached_sites = set() + system_sync_server_presets = ( + instance.context.data["system_settings"] + ["modules"] + ["sync_server"]) + log.debug("system_sett:: {}".format(system_sync_server_presets)) + + if system_sync_server_presets["enabled"]: + sync_project_presets = ( + instance.context.data["project_settings"] + ["global"] + ["sync_server"]) + + if sync_project_presets and sync_project_presets["enabled"]: + local_site, remote_site = self._get_sites(sync_project_presets) + + always_accesible = sync_project_presets["config"]. \ + get("always_accessible_on", []) + + already_attached_sites = {} meta = {"name": local_site, "created_dt": datetime.now()} rec["sites"] = [meta] - already_attached_sites.add(meta["name"]) + already_attached_sites[meta["name"]] = meta["created_dt"] - if remote_site: + if sync_project_presets and sync_project_presets["enabled"]: + # add remote meta = {"name": remote_site.strip()} rec["sites"].append(meta) - already_attached_sites.add(meta["name"]) + already_attached_sites[meta["name"]] = None - # add skeleton for site where it should be always synced to - for always_on_site in always_accesible: - if always_on_site not in [local_site, remote_site]: - meta = {"name": always_on_site.strip()} - rec["sites"].append(meta) - already_attached_sites.add(meta["name"]) + # add skeleton for site where it should be always synced to + for always_on_site in always_accesible: + if always_on_site not in already_attached_sites.keys(): + meta = {"name": always_on_site.strip()} + rec["sites"].append(meta) + already_attached_sites[meta["name"]] = None - log.debug("alternate_sites:: {}".format(alternate_sites)) - for alt_site in alternate_sites: - if alt_site not in already_attached_sites: - meta = {"name": local_site, "created_dt": datetime.now()} - rec["sites"].append(meta) + # add alternative sites + rec = self._add_alternative_sites(system_sync_server_presets, + already_attached_sites, + rec) + + log.debug("final sites:: {}".format(rec["sites"])) return rec @@ -1092,8 +1093,7 @@ def _get_sites(self, sync_project_presets): local_site_id = openpype.api.get_local_site_id() local_site = sync_project_presets["config"]. \ get("active_site", "studio").strip() - always_accesible = sync_project_presets["config"]. \ - get("always_accessible_on", []) + if local_site == 'local': local_site = local_site_id @@ -1106,6 +1106,32 @@ def _get_sites(self, sync_project_presets): return local_site, remote_site + def _add_alternative_sites(self, + system_sync_server_presets, + already_attached_sites, + rec): + """Loop through all configured sites and add alternatives. + + See SyncServerModule.handle_alternate_site + """ + conf_sites = system_sync_server_presets.get("sites", {}) + + for site_name, site_info in conf_sites.items(): + alt_sites = set(site_info.get("alternative_sites", [])) + for added_site in already_attached_sites.keys(): + if added_site in alt_sites: + if site_name in already_attached_sites.keys(): + continue + meta = {"name": site_name} + real_created = already_attached_sites[added_site] + # alt site inherits state of 'created_dt' + if real_created: + meta["created_dt"] = real_created + rec["sites"].append(meta) + already_attached_sites[meta["name"]] = real_created + + return rec + def handle_destination_files(self, integrated_file_sizes, mode): """ Clean destination files Called when error happened during integrating to DB or to disk From 828c601d144dc0d24cfacdbe1a4ba29157349e99 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 16:38:29 +0100 Subject: [PATCH 20/24] OP-1937 - fix - failure when Site Sync is not enabled --- openpype/plugins/publish/integrate_new.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index c0a90ee78e0..6c51f640fbf 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1030,6 +1030,7 @@ def prepare_file_info(self, path, size=None, file_hash=None, local_site = 'studio' # default remote_site = None always_accesible = [] + sync_project_presets = None rec = { "_id": io.ObjectId(), @@ -1090,6 +1091,7 @@ def prepare_file_info(self, path, size=None, file_hash=None, return rec def _get_sites(self, sync_project_presets): + """Returns tuple (local_site, remote_site)""" local_site_id = openpype.api.get_local_site_id() local_site = sync_project_presets["config"]. \ get("active_site", "studio").strip() From 199717a0df976d21e23c5f2566a843ed3562771e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 17:10:19 +0100 Subject: [PATCH 21/24] OP-1937 - fix - wrong icon used Icon of provider from last configured site was used --- .../modules/default_modules/sync_server/sync_server_module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index 9ff706d6ddd..bfc7c75c83a 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -1109,8 +1109,8 @@ def get_provider_for_site(self, project_name=None, site=None): return provider sync_sett = self.sync_system_settings - for site, detail in sync_sett.get("sites", {}).items(): - sites[site] = detail.get("provider") + for conf_site, detail in sync_sett.get("sites", {}).items(): + sites[conf_site] = detail.get("provider") return sites.get(site, 'N/A') From 8d00883a04d22bb19454e6e18d57e8afe3e9bd91 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 17:26:34 +0100 Subject: [PATCH 22/24] Hound --- .../sync_server/tray/widgets.py | 2 +- openpype/tests/mongo_performance.py | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/tray/widgets.py b/openpype/modules/default_modules/sync_server/tray/widgets.py index 01cc0d46d2f..15da1b5d196 100644 --- a/openpype/modules/default_modules/sync_server/tray/widgets.py +++ b/openpype/modules/default_modules/sync_server/tray/widgets.py @@ -140,7 +140,7 @@ def refresh(self): selected_index.isValid() and \ not self._selection_changed: mode = QtCore.QItemSelectionModel.Select | \ - QtCore.QItemSelectionModel.Rows + QtCore.QItemSelectionModel.Rows self.project_list.selectionModel().select(selected_index, mode) if self.current_project: diff --git a/openpype/tests/mongo_performance.py b/openpype/tests/mongo_performance.py index 9220c6c7304..2df3363f4b5 100644 --- a/openpype/tests/mongo_performance.py +++ b/openpype/tests/mongo_performance.py @@ -104,8 +104,8 @@ def prepare(self, no_of_records=100000, create_files=False): "name": "mb", "parent": {"oid": '{}'.format(id)}, "data": { - "path": "C:\\projects\\test_performance\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\{}\\{}".format(version_str, file_name), # noqa - "template": "{root[work]}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}" # noqa + "path": "C:\\projects\\test_performance\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\{}\\{}".format(version_str, file_name), # noqa: E501 + "template": "{root[work]}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}" # noqa: E501 }, "type": "representation", "schema": "openpype:representation-2.0" @@ -188,21 +188,21 @@ def get_files_array(self, i, file_id, file_id2, file_id3, create_files=False): ret = [ { - "path": "{root[work]}" + "{root[work]}/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_A_workfileLookdev_v{:03d}.dat".format(i, i), #noqa + "path": "{root[work]}" + "{root[work]}/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_A_workfileLookdev_v{:03d}.dat".format(i, i), # noqa: E501 "_id": '{}'.format(file_id), "hash": "temphash", "sites": self.get_sites(self.MAX_NUMBER_OF_SITES), "size": random.randint(0, self.MAX_FILE_SIZE_B) }, { - "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_B_workfileLookdev_v{:03d}.dat".format(i, i), #noqa + "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_B_workfileLookdev_v{:03d}.dat".format(i, i), # noqa: E501 "_id": '{}'.format(file_id2), "hash": "temphash", "sites": self.get_sites(self.MAX_NUMBER_OF_SITES), "size": random.randint(0, self.MAX_FILE_SIZE_B) }, { - "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_C_workfileLookdev_v{:03d}.dat".format(i, i), #noqa + "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_C_workfileLookdev_v{:03d}.dat".format(i, i), # noqa: E501 "_id": '{}'.format(file_id3), "hash": "temphash", "sites": self.get_sites(self.MAX_NUMBER_OF_SITES), @@ -223,8 +223,8 @@ def get_files_doc(self, i, file_id, file_id2, file_id3): ret = {} ret['{}'.format(file_id)] = { "path": "{root[work]}" + - "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" #noqa - "v{:03d}/test_CylinderA_workfileLookdev_v{:03d}.mb".format(i, i), # noqa + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" # noqa: E501 + "v{:03d}/test_CylinderA_workfileLookdev_v{:03d}.mb".format(i, i), # noqa: E501 "hash": "temphash", "sites": ["studio"], "size": 87236 @@ -232,16 +232,16 @@ def get_files_doc(self, i, file_id, file_id2, file_id3): ret['{}'.format(file_id2)] = { "path": "{root[work]}" + - "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" #noqa - "v{:03d}/test_CylinderB_workfileLookdev_v{:03d}.mb".format(i, i), # noqa + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" # noqa: E501 + "v{:03d}/test_CylinderB_workfileLookdev_v{:03d}.mb".format(i, i), # noqa: E501 "hash": "temphash", "sites": ["studio"], "size": 87236 } ret['{}'.format(file_id3)] = { "path": "{root[work]}" + - "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" #noqa - "v{:03d}/test_CylinderC_workfileLookdev_v{:03d}.mb".format(i, i), # noqa + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" # noqa: E501 + "v{:03d}/test_CylinderC_workfileLookdev_v{:03d}.mb".format(i, i), # noqa: E501 "hash": "temphash", "sites": ["studio"], "size": 87236 From 24c1aed73939a8641c5224e664a88f6b039abb90 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 17:42:19 +0100 Subject: [PATCH 23/24] OP-1920 - renamed env var SITE_SYNC_LOCAL_ID to OPENPYPE_LOCAL_ID --- openpype/cli.py | 2 +- openpype/lib/local_settings.py | 4 ++-- openpype/pype_commands.py | 2 +- website/docs/module_site_sync.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index 2a7e0c173ba..f937d5818ef 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -337,7 +337,7 @@ def syncserver(debug, active_site): Process mimics OP Tray with specific 'active_site' name, all configuration for this "dummy" user comes from Setting or Local Settings (configured by starting OP Tray with env - var SITE_SYNC_LOCAL_ID set to 'active_site'. + var OPENPYPE_LOCAL_ID set to 'active_site'. """ if debug: os.environ['OPENPYPE_DEBUG'] = '3' diff --git a/openpype/lib/local_settings.py b/openpype/lib/local_settings.py index af8c3cdbc83..97e99b4b5ae 100644 --- a/openpype/lib/local_settings.py +++ b/openpype/lib/local_settings.py @@ -524,8 +524,8 @@ def get_local_site_id(): """ # override local id from environment # used for background syncing - if os.environ.get("SITE_SYNC_LOCAL_ID"): - return os.environ["SITE_SYNC_LOCAL_ID"] + if os.environ.get("OPENPYPE_LOCAL_ID"): + return os.environ["OPENPYPE_LOCAL_ID"] registry = OpenPypeSettingsRegistry() try: diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index e160db0f150..2ccb4c8a0b4 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -332,7 +332,7 @@ def run_tests(self, folder, mark, pyargs): def syncserver(self, active_site): """Start running sync_server in background.""" import signal - os.environ["SITE_SYNC_LOCAL_ID"] = active_site + os.environ["OPENPYPE_LOCAL_ID"] = active_site def signal_handler(sig, frame): print("You pressed Ctrl+C. Process ended.") diff --git a/website/docs/module_site_sync.md b/website/docs/module_site_sync.md index 31854e27297..d9b53e32fb2 100644 --- a/website/docs/module_site_sync.md +++ b/website/docs/module_site_sync.md @@ -164,7 +164,7 @@ As current implementation relies heavily on Settings and Local Settings, backgro To do this: -- run OP `Tray` with environment variable SITE_SYNC_LOCAL_ID set to name of active (source) site. In most use cases it would be studio (for cases of backups of everything published to studio site to different cloud site etc.) +- run OP `Tray` with environment variable OPENPYPE_LOCAL_ID set to name of active (source) site. In most use cases it would be studio (for cases of backups of everything published to studio site to different cloud site etc.) - start `Tray` - check `Local ID` in information dialog after clicking on version number in the Tray - open `Local Settings` in the `Tray` From 87191d8d300338defa81c3894af849d98562e5d3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Nov 2021 12:08:04 +0100 Subject: [PATCH 24/24] OP-1937 - fix alternate sites for default studio site --- .../sync_server/sync_server_module.py | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index ca322b9f64a..c5649afec43 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -793,20 +793,34 @@ def handle_alternate_site(self, collection, representation, processed_site, by provider """ sites = self.sync_system_settings.get("sites", {}) + sites[self.DEFAULT_SITE] = {"provider": "local_drive", + "alternative_sites": []} + + alternate_sites = [] for site_name, site_info in sites.items(): - if processed_site in site_info.get("alternative_sites", []): - query = { - "_id": representation["_id"] - } - elem = {"name": "sftp", - "created_dt": datetime.now(), - "id": synced_file_id} - - self.log.debug("Adding alternate {} to {}".format( - site_name, representation["_id"])) - self._add_site(collection, query, - [representation], elem, - site_name, file_id=file_id, force=True) + conf_alternative_sites = site_info.get("alternative_sites", []) + if processed_site in conf_alternative_sites: + alternate_sites.append(site_name) + continue + if processed_site == site_name and conf_alternative_sites: + alternate_sites.extend(conf_alternative_sites) + continue + + alternate_sites = set(alternate_sites) + + for alt_site in alternate_sites: + query = { + "_id": representation["_id"] + } + elem = {"name": alt_site, + "created_dt": datetime.now(), + "id": synced_file_id} + + self.log.debug("Adding alternate {} to {}".format( + alt_site, representation["_id"])) + self._add_site(collection, query, + [representation], elem, + site_name, file_id=file_id, force=True) """ End of Public API """ def get_local_file_path(self, collection, site_name, file_path):