|
| 1 | +import logging |
| 2 | +import os |
| 3 | +import time |
| 4 | +from Directory import Directory |
| 5 | +from File import File |
| 6 | +from talk_to_ftp import TalkToFTP |
| 7 | +import threading as th |
| 8 | + |
| 9 | +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') |
| 10 | + |
| 11 | + |
| 12 | +class DirectoryManager: |
| 13 | + def __init__(self, ftp_website, directory, depth, excluded_extensions): |
| 14 | + self.root_directory = directory |
| 15 | + self.depth = depth |
| 16 | + # list of the extensions to exclude during synchronization |
| 17 | + self.excluded_extensions = excluded_extensions |
| 18 | + # dictionary to remember the instance of File / Directory saved on the FTP |
| 19 | + self.synchronize_dict = {} |
| 20 | + self.os_separator_count = len(directory.split(os.path.sep)) |
| 21 | + # list of the path explored for each synchronization |
| 22 | + self.paths_explored = [] |
| 23 | + # list of the File / Directory to removed from the dictionary at the end |
| 24 | + # of the synchronization |
| 25 | + self.to_remove_from_dict = [] |
| 26 | + # FTP instance |
| 27 | + self.ftp = TalkToFTP(ftp_website) |
| 28 | + # create the directory on the FTP if not already existing |
| 29 | + self.ftp.connect() |
| 30 | + if self.ftp.directory.count(os.path.sep) == 0: |
| 31 | + # want to create folder at the root of the server |
| 32 | + directory_split = "" |
| 33 | + else: |
| 34 | + directory_split = self.ftp.directory.rsplit(os.path.sep, 1)[0] |
| 35 | + if not self.ftp.if_exist(self.ftp.directory, self.ftp.get_folder_content(directory_split)): |
| 36 | + self.ftp.create_folder(self.ftp.directory) |
| 37 | + self.ftp.disconnect() |
| 38 | + |
| 39 | + def synchronize_directory(self, frequency): |
| 40 | + while True: |
| 41 | + # init the path explored to an empty list before each synchronization |
| 42 | + self.paths_explored = [] |
| 43 | + |
| 44 | + # init to an empty list for each synchronization |
| 45 | + self.to_remove_from_dict = [] |
| 46 | + |
| 47 | + # search for an eventual updates of files in the root directory |
| 48 | + self.ftp.connect() |
| 49 | + self.search_updates(self.root_directory) |
| 50 | + |
| 51 | + # look for any removals of files / directories |
| 52 | + self.any_removals() |
| 53 | + self.ftp.disconnect() |
| 54 | + |
| 55 | + # wait before next synchronization |
| 56 | + time.sleep(frequency) |
| 57 | + |
| 58 | + def create_dir(self, srv_full_path): |
| 59 | + self.ftp.create_folder(srv_full_path) |
| 60 | + |
| 61 | + def launch_files(self, file_path, path_file, file_name): |
| 62 | + # get depth of the current file by the count of the os separator in a path |
| 63 | + # and compare it with the count of the root directory |
| 64 | + if self.is_superior_max_depth(file_path) is False and \ |
| 65 | + (self.contain_excluded_extensions(file_path) is False): |
| 66 | + |
| 67 | + self.paths_explored.append(file_path) |
| 68 | + # try if already in the dictionary |
| 69 | + if file_path in self.synchronize_dict.keys(): |
| 70 | + |
| 71 | + # if yes and he get updated, we update this file on the FTP server |
| 72 | + if self.synchronize_dict[file_path].update_instance() == 1: |
| 73 | + # file get updates |
| 74 | + split_path = file_path.split(self.root_directory) |
| 75 | + srv_full_path = '{}{}'.format(self.ftp.directory, split_path[1]) |
| 76 | + self.ftp.remove_file(srv_full_path) |
| 77 | + # update this file on the FTP server |
| 78 | + self.ftp.file_transfer(path_file, srv_full_path, file_name) |
| 79 | + |
| 80 | + else: |
| 81 | + |
| 82 | + # file get created |
| 83 | + self.synchronize_dict[file_path] = File(file_path) |
| 84 | + split_path = file_path.split(self.root_directory) |
| 85 | + srv_full_path = '{}{}'.format(self.ftp.directory, split_path[1]) |
| 86 | + # add this file on the FTP server |
| 87 | + self.ftp.file_transfer(path_file, srv_full_path, file_name) |
| 88 | + |
| 89 | + def search_updates(self, directory): |
| 90 | + # scan recursively all files & directories in the root directory |
| 91 | + |
| 92 | + for path_file, dirs, files in os.walk(directory): |
| 93 | + |
| 94 | + threadsDir = [] |
| 95 | + threadsFile = [] |
| 96 | + |
| 97 | + for dir_name in dirs: |
| 98 | + folder_path = os.path.join(path_file, dir_name) |
| 99 | + |
| 100 | + # get depth of the current directory by the count of the os separator in a path |
| 101 | + # and compare it with the count of the root directory |
| 102 | + if self.is_superior_max_depth(folder_path) is False: |
| 103 | + self.paths_explored.append(folder_path) |
| 104 | + |
| 105 | + # a folder can't be updated, the only data we get is his creation time |
| 106 | + # a folder get created during running time if not present in our list |
| 107 | + |
| 108 | + if folder_path not in self.synchronize_dict.keys(): |
| 109 | + # directory created |
| 110 | + # add it to dictionary |
| 111 | + self.synchronize_dict[folder_path] = Directory(folder_path) |
| 112 | + |
| 113 | + # create it on FTP server |
| 114 | + split_path = folder_path.split(self.root_directory) |
| 115 | + srv_full_path = '{}{}'.format(self.ftp.directory, split_path[1]) |
| 116 | + directory_split = srv_full_path.rsplit(os.path.sep, 1)[0] |
| 117 | + if not self.ftp.if_exist(srv_full_path, self.ftp.get_folder_content(directory_split)): |
| 118 | + threadsDir.append(th.Thread(target=self.create_dir, args=(srv_full_path,))) |
| 119 | + threadsDir.append(th.Thread(target=self.create_dir, args=(srv_full_path,))) |
| 120 | + threadsDir[-1].start() |
| 121 | + |
| 122 | + # add this directory to the FTP server |
| 123 | + # self.ftp.create_folder(srv_full_path) |
| 124 | + if not threadsDir: |
| 125 | + for threads in threadsDir: |
| 126 | + threads.join() |
| 127 | + |
| 128 | + for file_name in files: |
| 129 | + file_path = os.path.join(path_file, file_name) |
| 130 | + |
| 131 | + # get depth of the current file by the count of the os separator in a path |
| 132 | + # and compare it with the count of the root directory |
| 133 | + if self.is_superior_max_depth(file_path) is False and \ |
| 134 | + (self.contain_excluded_extensions(file_path) is False): |
| 135 | + |
| 136 | + self.paths_explored.append(file_path) |
| 137 | + # try if already in the dictionary |
| 138 | + if file_path in self.synchronize_dict.keys(): |
| 139 | + |
| 140 | + # if yes and he get updated, we update this file on the FTP server |
| 141 | + if self.synchronize_dict[file_path].update_instance() == 1: |
| 142 | + # file get updates |
| 143 | + split_path = file_path.split(self.root_directory) |
| 144 | + srv_full_path = '{}{}'.format(self.ftp.directory, split_path[1]) |
| 145 | + self.ftp.remove_file(srv_full_path) |
| 146 | + # update this file on the FTP server |
| 147 | + self.ftp.file_transfer(path_file, srv_full_path, file_name) |
| 148 | + |
| 149 | + else: |
| 150 | + |
| 151 | + # file get created |
| 152 | + self.synchronize_dict[file_path] = File(file_path) |
| 153 | + split_path = file_path.split(self.root_directory) |
| 154 | + srv_full_path = '{}{}'.format(self.ftp.directory, split_path[1]) |
| 155 | + # add this file on the FTP server |
| 156 | + self.ftp.file_transfer(path_file, srv_full_path, file_name) |
| 157 | + |
| 158 | + def any_removals(self): |
| 159 | + # if the length of the files & folders to synchronize == number of path explored |
| 160 | + # no file / folder got removed |
| 161 | + if len(self.synchronize_dict.keys()) == len(self.paths_explored): |
| 162 | + return |
| 163 | + |
| 164 | + # get the list of the files & folders removed |
| 165 | + path_removed_list = [key for key in self.synchronize_dict.keys() if key not in self.paths_explored] |
| 166 | + |
| 167 | + for removed_path in path_removed_list: |
| 168 | + # check if the current path is not in the list of path already deleted |
| 169 | + # indeed we can't modify path_removed_list now because we're iterating over it |
| 170 | + if removed_path not in self.to_remove_from_dict: |
| 171 | + # get the instance of the files / folders deleted |
| 172 | + # then use the appropriate methods to remove it from the FTP server |
| 173 | + if isinstance(self.synchronize_dict[removed_path], File): |
| 174 | + split_path = removed_path.split(self.root_directory) |
| 175 | + srv_full_path = '{}{}'.format(self.ftp.directory, split_path[1]) |
| 176 | + self.ftp.remove_file(srv_full_path) |
| 177 | + self.to_remove_from_dict.append(removed_path) |
| 178 | + |
| 179 | + elif isinstance(self.synchronize_dict[removed_path], Directory): |
| 180 | + split_path = removed_path.split(self.root_directory) |
| 181 | + srv_full_path = '{}{}'.format(self.ftp.directory, split_path[1]) |
| 182 | + self.to_remove_from_dict.append(removed_path) |
| 183 | + # if it's a directory, we need to delete all the files and directories he contains |
| 184 | + self.remove_all_in_directory(removed_path, srv_full_path, path_removed_list) |
| 185 | + |
| 186 | + # all the files / folders deleted in the local directory need to be deleted |
| 187 | + # from the dictionary use to synchronize |
| 188 | + for to_remove in self.to_remove_from_dict: |
| 189 | + if to_remove in self.synchronize_dict.keys(): |
| 190 | + del self.synchronize_dict[to_remove] |
| 191 | + |
| 192 | + def remove_all_in_directory(self, removed_directory, srv_full_path, path_removed_list): |
| 193 | + directory_containers = {} |
| 194 | + for path in path_removed_list: |
| 195 | + |
| 196 | + # path string contains removed_directory and this path did not get already deleted |
| 197 | + if removed_directory != path and removed_directory in path \ |
| 198 | + and path not in self.to_remove_from_dict: |
| 199 | + |
| 200 | + # if no path associated to the current depth we init it |
| 201 | + if len(path.split(os.path.sep)) not in directory_containers.keys(): |
| 202 | + directory_containers[len(path.split(os.path.sep))] = [path] |
| 203 | + else: |
| 204 | + # if some paths are already associated to the current depth |
| 205 | + # we only append the current path |
| 206 | + directory_containers[len(path.split(os.path.sep))].append(path) |
| 207 | + |
| 208 | + # sort the path depending on the file depth |
| 209 | + sorted_containers = sorted(directory_containers.values()) |
| 210 | + |
| 211 | + # we iterate starting from the innermost file |
| 212 | + for i in range(len(sorted_containers)-1, -1, -1): |
| 213 | + for to_delete in sorted_containers[i]: |
| 214 | + to_delete_ftp = "{0}{1}{2}".format(self.ftp.directory, os.path.sep, to_delete.split(self.root_directory)[1]) |
| 215 | + if isinstance(self.synchronize_dict[to_delete], File): |
| 216 | + self.ftp.remove_file(to_delete_ftp) |
| 217 | + self.to_remove_from_dict.append(to_delete) |
| 218 | + else: |
| 219 | + # if it's again a directory, we delete all his containers also |
| 220 | + self.remove_all_in_directory(to_delete, to_delete_ftp, path_removed_list) |
| 221 | + # once all the containers of the directory got removed |
| 222 | + # we can delete the directory also |
| 223 | + self.ftp.remove_folder(srv_full_path) |
| 224 | + self.to_remove_from_dict.append(removed_directory) |
| 225 | + |
| 226 | + # subtract current number of os separator to the number of os separator for the root directory |
| 227 | + # if it's superior to the max depth, we do nothing |
| 228 | + def is_superior_max_depth(self, path): |
| 229 | + if (len(path.split(os.path.sep)) - self.os_separator_count) <= self.depth: |
| 230 | + return False |
| 231 | + else: |
| 232 | + return True |
| 233 | + |
| 234 | + # check if the file contains a prohibited extensions |
| 235 | + def contain_excluded_extensions(self, file): |
| 236 | + extension = file.split(".")[1] |
| 237 | + if ".{0}".format(extension) in self.excluded_extensions: |
| 238 | + return True |
| 239 | + else: |
| 240 | + return False |
0 commit comments