-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathpzm_locking.py
186 lines (164 loc) · 8.76 KB
/
pzm_locking.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
#!/usr/bin/env -S python3 -u
import pzm_common
from pzm_common import log, log_debug, execute_command, execute_readonly_command
import os
import socket
import random
import time
remoteSyncLock = "/var/lib/pve-zsync/manager_sync.lock"
locked = False
remote_locked_here = False
local_locked_here = False
#Check if the local lock (=file "remoteSyncLock") is lockable
#the file on remote and local is the same, in order to be able to do local and remote sync one by one.
def can_get_local_lock():
if os.path.exists(remoteSyncLock):
with open(remoteSyncLock, 'r') as lockfile:
output = lockfile.read()
log("Local lock is held by " + output + ", have to wait...")
return False
else:
log_debug ("Local lockfile does not exist, can proceed...")
return True
#Check if the remote lock (=file "remoteSyncLock") is lockable
#the file on remote and local is the same, in order to be able to do local and remote sync one by one.
def can_get_remote_lock(hostname):
lockvalue = socket.gethostname().lower() + "-" + str(os.getpid())
rc, stdout, stderr = execute_readonly_command(['ssh', '-o', 'BatchMode yes', 'root@' + hostname, 'cat', remoteSyncLock])
if rc == 1: #rc 1 means, file not found - which means no lock is held on remote side
log_debug ("Remote lockfile does not exist, can proceed...")
return True
################# REMOVED: It doesn't seem to be needed, as there is no possibility that the remote side could be locked by "us" (=hostname and pid identical) at this point in the execution
################# The local lock, which would be the exact same file as the remote lock in a local sync, get's set AFTER the remote file was placed
#elif rc == 0 and lockvalue in stdout.lower(): #rc 0 - file found, if my hostname is the same as in the file, proceed
# #Needed for local syncs/backups
# log_debug ("Remote lock is held by this host and this process, can proceed...")
# return True
elif rc == 0: #rc 0 file found, but content is not my hostname, can't proceed
log ("Remote lock is held by " + stdout + ", have to wait...")
return False
else: #rc not 1 or 0 - must be another error - can't proceed
log ("(SSH) Error while checking lock availability (Maybe host is down or network issue) " + stderr)
return False
#Gather the local lock. Return true if we got it
def lock_local():
global local_locked_here
log_debug ("Locking locally")
lockvalue = socket.gethostname().lower() + "-" + str(os.getpid())
if not os.path.exists(remoteSyncLock): #In case it was locked by "remote lock" if it's a local sync, do nothing
with open(remoteSyncLock, 'w') as lockfile:
log_debug("Writing local lockfile")
lockfile.write(lockvalue)
local_locked_here = True
execute_command(['chattr', '+i', remoteSyncLock]) #Make file immuteable with chattr
else:
with open(remoteSyncLock, 'r') as lockfile:
output = lockfile.read()
if lockvalue in output.lower(): #If read value is the same as this hostname-pid then it was us who locked the file. Most likely the remote lock part in a local sync
local_locked_here = False
log_debug ("Was already locked locally")
else:
local_locked_here = False
log ("Couldn't get local lock as it's held by " + output + "!")
return False
log_debug ("Locally locked")
return True
#Gather the remote lock. Return true if we got it. Also return true if the remote lock is held by this host and this PID. This will happen in a local sync situation
def lock_remote(hostname):
global remote_locked_here
log_debug("Locking remotly")
log_debug("Trying to write remote lockfile")
lockvalue = socket.gethostname().lower() + "-" + str(os.getpid())
rc, stdout, stderr, pid = execute_command(['ssh', '-o', 'BatchMode yes', 'root@' + hostname,
"echo -n " + lockvalue + " > " + remoteSyncLock + " && chattr +i " + remoteSyncLock])
if rc == 1: #Operation not permitteed or File no Found in chattr
################# REMOVED: It doesn't seem to be needed, as there is no possibility that the remote side could be locked by "us" (=hostname and pid identical) at this point in the execution
################# The local lock, which would be the exact same file as the remote lock in a local sync, get's set AFTER the remote file was placed
#rc, stdout, stderr_1 = execute_readonly_command(['ssh', '-o', 'BatchMode yes', 'root@' + hostname, 'cat', remoteSyncLock])
#if lockvalue in stdout.lower():
# #Wasn't able to get lock, due to chattr imutability, but if lockvalue is ours, it count's as if.
# log_debug ("Was already remotely locked")
# remote_locked_here = False
# log_debug("Remotely locked")
# return True
#else:
# log ("Wasn't able to get the remote lock! " + stderr)
# return False
log ("Wasn't able to get the remote lock! " + stderr)
return False
elif rc == 0: #Worked fine
remote_locked_here = True
log_debug("Remotely locked")
return True
else:
log ("(SSH) Error in putting lock on remote side, trying again. " + stderr)
return False
#Release the remote lock
def unlock_remote(hostname):
global locked
global remote_locked_here
if remote_locked_here: #Only delete if it was remote locked here
while locked: #Make sure we safely delete the lock
#chattr: make file muteable again
log_debug("Removing remote lockfile")
rc, stdout, stderr, pid = execute_command(['ssh', '-o', 'BatchMode yes', 'root@' + hostname, 'chattr -i ' + remoteSyncLock + ' ; rm ' + remoteSyncLock])
if rc == 0:
locked = False
elif rc == 1:
log_debug ("(SSH) Odd, remote lockfile doesn't exist anymore???")
locked = False
else:
log ("(SSH) Error while deleting the remote lock, trying again " + stderr)
time.sleep(30)
else:
log_debug("Not removing remote lockfile as it wasn't created in lock_remote (was previously locked)")
locked = False
log_debug("Remotely unlocked")
#Release to local lock
def unlock_local():
global locked
global local_locked_here
if local_locked_here: #Only delete if it was remote locked here
log_debug("Removing local lockfile")
if os.path.exists(remoteSyncLock):
execute_command(['chattr', '-i', remoteSyncLock]) #Make file mutable again
os.remove(remoteSyncLock)
else:
if socket.gethostname().lower() not in hostname.lower() and "localhost" not in hostname.lower():
#AKA if my hostname is not the same as the hostname where the remotefile was previously removed. = Non-local Backup
log_debug ("Odd, local lockfile doesn't exist anymore???")
#It would be normal that the lockfile doesn't exist anymore at this point, if the destination was localhost
else:
log_debug("Not removing local lockfile as it wasn't created in lock_local (was previously locked)")
log_debug ("Locally unlocked")
if remote_locked_here or local_locked_here or locked:
log ("Locks released")
#Check if both locks are available, then lock both. If anything goes wrong, reset and start over.
def lock(hostname):
global locked
presleep = random.uniform(0,60)
if not pzm_common.test:
log ("Waiting for " + str(presleep) + "s before starting...")
time.sleep(presleep) #Random Delay to minimize possibility of simultanious locking...
log ("Aquiring locks")
while not locked: #Make sure lock was established successfully on remote side. If not, check again if possible
while not (can_get_remote_lock(hostname) and can_get_local_lock()):
sleeptime = random.uniform(30,60)
log_debug ("Lock is held... sleeping " + str(sleeptime) + "s")
time.sleep(sleeptime)
if lock_remote(hostname):
if not lock_local():
log("Local lock couldn't get aquired, even if the prechecks said it would be. Unlocking remote, and trying again...")
unlock_remote(hostname) #If for any reason we weren't able to get the local lock but did get the remote lock - Unlock the remote lock again
else:
locked = True #breaks the while loop
log ("Locks aquired")
#Unlock remote and lock lock
def unlock(hostname):
global locked
global remote_locked_here
global local_locked_here
if remote_locked_here or local_locked_here or locked:
log("Releasing locks")
unlock_remote(hostname)
unlock_local()