From b9454f2865b97957344b082f3b78d181788f01f7 Mon Sep 17 00:00:00 2001 From: Philip Trauner Date: Sun, 11 Sep 2016 20:35:27 +0200 Subject: [PATCH] Broke during migration to WebSockets Will be replaced by behem0th --- Shared/Sync.py | 505 ------------------------------------------------- 1 file changed, 505 deletions(-) delete mode 100644 Shared/Sync.py diff --git a/Shared/Sync.py b/Shared/Sync.py deleted file mode 100644 index 58a6f0d..0000000 --- a/Shared/Sync.py +++ /dev/null @@ -1,505 +0,0 @@ -import os -import hashlib -import time -import base64 -import random -import pickle -import shutil -import _thread - -import Routing -import Logging - -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler - - -class File: - def __init__(self, relpath, hash, mtime): - self.relpath = relpath - self.hash = hash - self.mtime = mtime - - - def __eq__(self, other): - if type(other) is File: - return self.__dict__ == other.__dict__ - return False - - - def __repr__(self): - return "%s: %s, %d" % (self.relpath, self.hash, self.mtime) - - -class DeletedStorage: - def __init__(self, storage_path="deleted_files.pickle"): - try: - self.files = pickle.load(open(storage_path, "rb")) - except (FileNotFoundError, EOFError): - self.files = [] - self.storage_path = storage_path - - def add(self, file): - if type(file) is File: - if file not in self.files: - self.files.append(file) - else: - raise TypeError("only objects of type File can be added") - - def remove(self, file): - if type(file) is File: - if file in self.files: - del self.files[self.files.index(file)] - else: - raise TypeError("only objects of type File can be removed") - - def close(self): - mode = "wb" - if not self.storage_path in os.listdir("."): - mode = "ab" - pickle.dump(self.files, open(self.storage_path, mode)) - - - -def relative_recursive_ls(path, relative_to, exclude=[]): - if path[-1] != "/": - path += "/" - files = [] - for item in os.listdir(path): - if item not in exclude: - if os.path.isdir(path + item): - files += relative_recursive_ls(path + item, relative_to, exclude) - else: - files.append(os.path.relpath(path + item, relative_to)) - return files - - -def relative_recursive_item_ls(path, relative_to, exclude=[]): - if path[-1] != "/": - path += "/" - items = [] - for item in os.listdir(path): - if item not in exclude: - items.append(os.path.relpath(path + item, relative_to)) - if os.path.isdir(path + item): - items += relative_recursive_ls(path + item, relative_to, exclude) - return items - - -def relative_recursive_folder_ls(path, relative_to, exclude=[]): - if path[-1] != "/": - path += "/" - folders = [] - for item in os.listdir(path): - if item not in exclude: - if os.path.isdir(path + item): - folders.append(os.path.relpath(path + item, relative_to)) - folders += relative_recursive_folder_ls(path + item, relative_to, exclude) - return folders - - -def md5(path): - return hashlib.md5(open(path, "rb").read()).hexdigest() - -def get_mtime(path): - return os.path.getmtime(path) - - -def get_name_from_path(path): - name = os.path.basename(path) - if name: - return name - else: - return None - - -def get_file_content(path): - return open(path, "rb+").read() - -def base64_str_decode(path): - return base64.b64encode(get_file_content(path)).decode() - - -class ReloadHandler(FileSystemEventHandler): - def __init__(self, sync, path): - self.sync = sync - self.path = path - self.supressed_events = {} - self.deleted_ignore = [] - self.moved_ignore = [] - - - def on_modified(self, event): - if not event.is_directory and os.path.isfile(event.src_path): - if event.src_path in self.supressed_events: - if self.supressed_events[event.src_path] != md5(event.src_path): - del self.supressed_events[event.src_path] - self.sync.modified(event) - else: - self.supressed_events[event.src_path] = md5(event.src_path) - self.sync.modified(event) - - - def on_created(self, event): - self.on_modified(event) - - - def on_deleted(self, event): - # Check if file was actually deleted and doesn't exist again - if not os.path.exists(event.src_path): - if event.is_directory: - self.sync.deleted_dir(event) - else: - self.sync.deleted(event) - - - def on_moved(self, event): - if event.is_directory: - if not event.dest_path in self.moved_ignore: - for item in relative_recursive_ls(event.dest_path, self.path): - print(self.path + "/" + item) - if not item in self.moved_ignore: - self.moved_ignore.append(self.path + "/" + item) - self.sync.moved_dir(event) - else: - del self.moved_ignore[self.moved_ignore.index(event.dest_path)] - else: - if not event.dest_path in self.moved_ignore: - self.sync.moved(event) - else: - del self.moved_ignore[self.moved_ignore.index(event.dest_path)] - - -class SyncClient(Routing.ClientRoute): - def __init__(self, sock, folder, route, exclude=[".DS_Store", ".fl0w"], debug=False): - self.sock = sock - self.folder = folder if folder[-1] == "/" else folder + "/" - self.started = False - self.route = route - self.exclude = exclude - self.debug = debug - self.files = relative_recursive_ls(folder, folder, exclude=self.exclude) - self.suppressed_fs_events = [] - self.observer = Observer() - self.observer.schedule(ReloadHandler(self, self.folder), path=self.folder, recursive=True) - self.observer.start() - - - def start(self): - if self.debug: - Logging.info("Sync client started!") - out = {"list" : {}} - for file in self.files: - out["list"].update({file : {"mtime" : get_mtime(self.folder + file), - "md5" : md5(self.folder + file)}}) - self.sock.send(out, self.route) - - - def suppress_fs_event(self, file): - if not file in self.suppressed_fs_events: - self.suppressed_fs_events.append(file) - - - def unsuppress_fs_event(self, file): - if file in self.suppressed_fs_events: - del self.suppressed_fs_events[self.suppressed_fs_events.index(file)] - - - def run(self, data, handler): - if type(data) is dict: - if "add" in data: - for file in data["add"]: - self.suppress_fs_event(file) - folder_path = self.folder + os.path.dirname(file) - if not os.path.exists(folder_path): - os.makedirs(folder_path) - mode = "wb+" if os.path.exists(self.folder + file) else "ab+" - open(self.folder + file, mode).write(base64.b64decode(data["add"][file]["content"])) - os.utime(self.folder + file, (data["add"][file]["mtime"], data["add"][file]["mtime"])) - """ - for file in data["add"]: - if not file in self.suppressed_fs_events: - self.suppress_fs_event(file) - folder_path = self.folder + os.path.dirname(file) - if not os.path.exists(folder_path): - os.makedirs(folder_path) - if os.path.exists: - open(self.folder + file, "wb+").write(base64.b64decode(data["add"][file]["content"])) - else: - open(self.folder + file, "ab+").write(base64.b64decode(data["add"][file]["content"])) - os.utime(self.folder + file, (data["add"][file]["mtime"], data["add"][file]["mtime"])) - else: - self.unsuppress_fs_event(file) - """ - elif "del" in data: - for file in data["del"]: - self.suppress_fs_event(file) - try: - os.remove(self.folder + file) - except FileNotFoundError: - self.unsuppress_fs_event(file) - Logging.warning("Possible server misbehaviour") - elif "deldir" in data: - for dir in data["deldir"]: - if os.path.exists(self.folder + dir): - for folder in relative_recursive_folder_ls(self.folder + dir, self.folder): - self.suppress_fs_event(folder) - for file in relative_recursive_ls(self.folder + dir, self.folder): - self.suppress_fs_event(file) - self.suppress_fs_event(dir) - shutil.rmtree(self.folder + dir) - else: - Logging.warning("Possible server misbehaviour") - elif "mvd" in data: - for file in data["mvd"]: - if os.path.isfile(self.folder + file): - new_name = self.folder + data["mvd"][file] - if os.path.exists(os.path.dirname(new_name)): - self.suppress_fs_event(file) - shutil.move(self.folder + file, new_name) - elif "req" in data: - for file in data["req"]: - if file in self.files: - self.sock.send({"add" : {file : {"content" : base64_str_decode(self.folder + file), - "mtime" : os.path.getmtime(self.folder + file)}}}, self.route) - - - def modified(self, event, created=False): - if self.debug and not created: - Logging.info("Modified file: '%s'" % event.src_path) - relpath = os.path.relpath(event.src_path, self.folder) - if relpath not in self.files: - self.files.append(relpath) - if not relpath in self.suppressed_fs_events: - self.sock.send({"add" : {relpath : {"content" : base64_str_decode(event.src_path), - "mtime" : os.path.getmtime(event.src_path)}}}, self.route) - else: - self.unsuppress_fs_event(relpath) - - - def created(self, event): - if self.debug: - Logging.info("Created file: '%s'" % event.src_path) - self.modified(event, created=True) - - - def deleted(self, event): - if self.debug: - Logging.info("Deleted file: '%s'" % event.src_path) - relpath = os.path.relpath(event.src_path, self.folder) - if relpath in self.files: - del self.files[self.files.index(relpath)] - if not relpath in self.suppressed_fs_events: - self.sock.send({"del" : [relpath]}, self.route) - else: - self.unsuppress_fs_event(relpath) - - - def deleted_dir(self, event): - if self.debug: - Logging.info("Deleted folder: '%s'" % event.src_path) - relpath = os.path.relpath(event.src_path, self.folder) - if not relpath in self.suppressed_fs_events: - self.sock.send({"deldir" : [relpath]}, self.route) - else: - self.unsuppress_fs_event(relpath) - - def moved(self, event): - if self.debug: - Logging.info("Moved file: '%s'" % event.src_path) - src_relpath = os.path.relpath(event.src_path, self.folder) - dest_relpath = os.path.relpath(event.dest_path, self.folder) - if src_relpath in self.files: - del self.files[self.files.index(src_relpath)] - self.files.append(dest_relpath) - if not src_relpath in self.suppressed_fs_events: - self.sock.send({"mvd" : {src_relpath : dest_relpath}}, self.route) - else: - self.unsuppress_fs_event(src_relpath) - - - def moved_dir(self, event): - print(event) - print(dir(event)) - - -class SyncServer(Routing.ServerRoute): - REQUIRED = (Routing.BROADCAST, Routing.ROUTE) - - def __init__(self, folder, channel, exclude=[".DS_Store", ".fl0w"], debug=False, deleted_db_path="deleted.db", modified_hook=None, deleted_hook=None): - self.folder = folder if folder[-1] == "/" else folder + "/" - self.channel = channel - self.exclude = exclude - self.debug = debug - self.deleted_storage = DeletedStorage(deleted_db_path) - self.modified_hook = modified_hook - self.deleted_hook = deleted_hook - self.broadcast_file_excludes = {} - self.broadcast_dir_excludes = {} - self.ignore_events = [] - self.route = None # Set by REQUIRED - self.broadcast = None # Set by REQUIRED - self.started = False - - - def start(self, handler): - if not self.started: - if self.debug: - Logging.info("Sync server on route '%s' started!" % self.route) - self.files = relative_recursive_ls(self.folder, self.folder, exclude=self.exclude) - observer = Observer() - observer.schedule(ReloadHandler(self, self.folder), path=self.folder, recursive=True) - observer.start() - self.started = True - - - def run(self, data, handler): - if type(data) is dict: - if "list" in data: - client_files = [] - for file in data["list"]: - client_file = File(file, data["list"][file]["md5"], data["list"][file]["mtime"]) - client_files.append(client_file) - # If file doesn't exist on server check if it's in deleted storage - if not file in self.files: - # Delete storage works with objects - file = File(file, data["list"][file]["md5"], data["list"][file]["mtime"]) - if file in self.deleted_storage.files: - handler.sock.send({"del" : [file.relpath]}, self.route) - else: - # Request the file if it's not in delete storage - handler.sock.send({"req" : [file.relpath]}, self.route) - # Add all files that are not on the client but preserve newer ones by requesting them - for file in self.files: - file = File(file, md5(self.folder + file), get_mtime(self.folder + file)) - # Make sure that the newer version will be requested - for client_file in client_files: - if file.relpath == client_file.relpath: - if client_file.mtime > file.mtime: - client_files.append(file) # Hack that makes the iteration below work - handler.sock.send({"req" : [file.relpath]}, self.route) - # Not the same file because mtime is different - if file not in client_files: - handler.sock.send({"add" : {file.relpath : {"content" : base64_str_decode(self.folder + file.relpath), - "mtime" : os.path.getmtime(self.folder + file.relpath)}}}, self.route) - elif "add" in data: - for file in data["add"]: - folder_path = self.folder + os.path.dirname(file) - if not os.path.exists(folder_path): - os.makedirs(folder_path) - mode = "wb+" if os.path.exists(self.folder + file) else "ab+" - self.broadcast_file_excludes[file] = handler - open(self.folder + file, mode).write(base64.b64decode(data["add"][file]["content"])) - os.utime(self.folder + file, (data["add"][file]["mtime"], data["add"][file]["mtime"])) - elif "deldir" in data: - for dir in data["deldir"]: - if os.path.exists(self.folder + dir): - for folder in relative_recursive_folder_ls(self.folder + dir, self.folder): - self.ignore_events.append(folder) - for file in relative_recursive_ls(self.folder + dir, self.folder): - self.ignore_events.append(file) - if file in self.files: - del self.files[self.files.index(file)] - self.broadcast_dir_excludes[dir] = handler - shutil.rmtree(self.folder + dir) - else: - Logging.warning("Possible client misbehaviour (%s:%d)" % (handler.info[0], handler.info[1])) - elif "del" in data: - for file in data["del"]: - self.broadcast_file_excludes[file] = handler - self.deleted_storage.add(File(file, md5(self.folder + file), get_mtime(self.folder + file))) - try: - os.remove(self.folder + file) - except FileNotFoundError: - del self.broadcast_file_excludes[file] - Logging.warning("Possible client misbehaviour (%s:%d)" % (handler.info[0], handler.info[1])) - elif "mvd" in data: - for file in data["mvd"]: - if os.path.isfile(self.folder + file): - new_name = self.folder + data["mvd"][file] - if os.path.exists(os.path.dirname(new_name)): - self.broadcast_file_excludes[file] = handler - shutil.move(self.folder + file, new_name) - - - def modified(self, event, created=False): - if self.debug and not created: - Logging.info("Modified file: '%s'" % event.src_path) - relpath = os.path.relpath(event.src_path, self.folder) - if relpath not in self.files: - self.files.append(relpath) - exclude = [] - if relpath in self.broadcast_file_excludes: - exclude.append(self.broadcast_file_excludes[relpath]) - self.broadcast.broadcast({"add" : {relpath : { - "content" : base64_str_decode(event.src_path), - "mtime" : os.path.getmtime(event.src_path)}}}, self.route, self.channel, exclude=exclude) - if self.modified_hook != None: - self.modified_hook(self.folder, relpath, exclude[0] if len(exclude) == 1 else None) - if exclude != []: - del self.broadcast_file_excludes[relpath] - - - def created(self, event): - if self.debug: - Logging.info("Created file: '%s'" % event.src_path) - self.modified(event, created=True) - - - - def deleted(self, event): - if self.debug: - Logging.info("Deleted file: '%s'" % event.src_path) - relpath = os.path.relpath(event.src_path, self.folder) - if not relpath in self.ignore_events: - if relpath in self.files: - del self.files[self.files.index(relpath)] - exclude = [] - if relpath in self.broadcast_file_excludes: - exclude.append(self.broadcast_file_excludes[relpath]) - self.broadcast.broadcast({"del" : [relpath]}, self.route, self.channel, exclude=exclude) - if self.deleted_hook != None: - self.deleted_hook(self.folder, relpath, exclude[0] if len(exclude) == 1 else None) - if exclude != []: - del self.broadcast_file_excludes[relpath] - else: - del self.ignore_events[self.ignore_events.index(relpath)] - - - def deleted_dir(self, event): - if self.debug: - Logging.info("Deleted folder: '%s'" % event.src_path) - relpath = os.path.relpath(event.src_path, self.folder) - if not relpath in self.ignore_events: - exclude = [] - if relpath in self.broadcast_dir_excludes: - exclude.append(self.broadcast_dir_excludes[relpath]) - self.broadcast.broadcast({"deldir" : [relpath]}, self.route, self.channel, exclude=exclude) - if exclude != []: - del self.broadcast_dir_excludes[relpath] - else: - del self.ignore_events[self.ignore_events.index(relpath)] - - - def moved(self, event): - if self.debug: - Logging.info("Moved file: '%s'" % event.src_path) - src_relpath = os.path.relpath(event.src_path, self.folder) - dest_relpath = os.path.relpath(event.dest_path, self.folder) - if src_relpath in self.files: - del self.files[self.files.index(src_relpath)] - self.files.append(dest_relpath) - exclude = [] - if src_relpath in self.broadcast_file_excludes: - exclude.append(self.broadcast_file_excludes[src_relpath]) - self.broadcast.broadcast({"mvd" : {src_relpath : dest_relpath}}, self.route, self.channel, exclude=exclude) - if exclude != []: - del self.broadcast_file_excludes[src_relpath] - - - def moved_dir(self, event): - print(event) - - - def stop(self): - self.deleted_storage.close() \ No newline at end of file