Broke during migration to WebSockets
Will be replaced by behem0th
This commit is contained in:
Philip Trauner
parent
8638b98179
commit
b9454f2865
1 changed files with 0 additions and 505 deletions
505
Shared/Sync.py
505
Shared/Sync.py
|
|
@ -1,505 +0,0 @@
|
|||
import os
|
||||
import hashlib
|
||||
import time
|
||||
import base64
|
||||
import random
|
||||
import pickle
|
||||
import shutil
|
||||
import _thread
|
||||
|
||||
import Routing
|
||||
import Logging
|
||||
|
||||
from watchdog.observers import Observer
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
|
||||
|
||||
class File:
|
||||
def __init__(self, relpath, hash, mtime):
|
||||
self.relpath = relpath
|
||||
self.hash = hash
|
||||
self.mtime = mtime
|
||||
|
||||
|
||||
def __eq__(self, other):
|
||||
if type(other) is File:
|
||||
return self.__dict__ == other.__dict__
|
||||
return False
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
return "%s: %s, %d" % (self.relpath, self.hash, self.mtime)
|
||||
|
||||
|
||||
class DeletedStorage:
|
||||
def __init__(self, storage_path="deleted_files.pickle"):
|
||||
try:
|
||||
self.files = pickle.load(open(storage_path, "rb"))
|
||||
except (FileNotFoundError, EOFError):
|
||||
self.files = []
|
||||
self.storage_path = storage_path
|
||||
|
||||
def add(self, file):
|
||||
if type(file) is File:
|
||||
if file not in self.files:
|
||||
self.files.append(file)
|
||||
else:
|
||||
raise TypeError("only objects of type File can be added")
|
||||
|
||||
def remove(self, file):
|
||||
if type(file) is File:
|
||||
if file in self.files:
|
||||
del self.files[self.files.index(file)]
|
||||
else:
|
||||
raise TypeError("only objects of type File can be removed")
|
||||
|
||||
def close(self):
|
||||
mode = "wb"
|
||||
if not self.storage_path in os.listdir("."):
|
||||
mode = "ab"
|
||||
pickle.dump(self.files, open(self.storage_path, mode))
|
||||
|
||||
|
||||
|
||||
def relative_recursive_ls(path, relative_to, exclude=[]):
|
||||
if path[-1] != "/":
|
||||
path += "/"
|
||||
files = []
|
||||
for item in os.listdir(path):
|
||||
if item not in exclude:
|
||||
if os.path.isdir(path + item):
|
||||
files += relative_recursive_ls(path + item, relative_to, exclude)
|
||||
else:
|
||||
files.append(os.path.relpath(path + item, relative_to))
|
||||
return files
|
||||
|
||||
|
||||
def relative_recursive_item_ls(path, relative_to, exclude=[]):
|
||||
if path[-1] != "/":
|
||||
path += "/"
|
||||
items = []
|
||||
for item in os.listdir(path):
|
||||
if item not in exclude:
|
||||
items.append(os.path.relpath(path + item, relative_to))
|
||||
if os.path.isdir(path + item):
|
||||
items += relative_recursive_ls(path + item, relative_to, exclude)
|
||||
return items
|
||||
|
||||
|
||||
def relative_recursive_folder_ls(path, relative_to, exclude=[]):
|
||||
if path[-1] != "/":
|
||||
path += "/"
|
||||
folders = []
|
||||
for item in os.listdir(path):
|
||||
if item not in exclude:
|
||||
if os.path.isdir(path + item):
|
||||
folders.append(os.path.relpath(path + item, relative_to))
|
||||
folders += relative_recursive_folder_ls(path + item, relative_to, exclude)
|
||||
return folders
|
||||
|
||||
|
||||
def md5(path):
|
||||
return hashlib.md5(open(path, "rb").read()).hexdigest()
|
||||
|
||||
def get_mtime(path):
|
||||
return os.path.getmtime(path)
|
||||
|
||||
|
||||
def get_name_from_path(path):
|
||||
name = os.path.basename(path)
|
||||
if name:
|
||||
return name
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def get_file_content(path):
|
||||
return open(path, "rb+").read()
|
||||
|
||||
def base64_str_decode(path):
|
||||
return base64.b64encode(get_file_content(path)).decode()
|
||||
|
||||
|
||||
class ReloadHandler(FileSystemEventHandler):
|
||||
def __init__(self, sync, path):
|
||||
self.sync = sync
|
||||
self.path = path
|
||||
self.supressed_events = {}
|
||||
self.deleted_ignore = []
|
||||
self.moved_ignore = []
|
||||
|
||||
|
||||
def on_modified(self, event):
|
||||
if not event.is_directory and os.path.isfile(event.src_path):
|
||||
if event.src_path in self.supressed_events:
|
||||
if self.supressed_events[event.src_path] != md5(event.src_path):
|
||||
del self.supressed_events[event.src_path]
|
||||
self.sync.modified(event)
|
||||
else:
|
||||
self.supressed_events[event.src_path] = md5(event.src_path)
|
||||
self.sync.modified(event)
|
||||
|
||||
|
||||
def on_created(self, event):
|
||||
self.on_modified(event)
|
||||
|
||||
|
||||
def on_deleted(self, event):
|
||||
# Check if file was actually deleted and doesn't exist again
|
||||
if not os.path.exists(event.src_path):
|
||||
if event.is_directory:
|
||||
self.sync.deleted_dir(event)
|
||||
else:
|
||||
self.sync.deleted(event)
|
||||
|
||||
|
||||
def on_moved(self, event):
|
||||
if event.is_directory:
|
||||
if not event.dest_path in self.moved_ignore:
|
||||
for item in relative_recursive_ls(event.dest_path, self.path):
|
||||
print(self.path + "/" + item)
|
||||
if not item in self.moved_ignore:
|
||||
self.moved_ignore.append(self.path + "/" + item)
|
||||
self.sync.moved_dir(event)
|
||||
else:
|
||||
del self.moved_ignore[self.moved_ignore.index(event.dest_path)]
|
||||
else:
|
||||
if not event.dest_path in self.moved_ignore:
|
||||
self.sync.moved(event)
|
||||
else:
|
||||
del self.moved_ignore[self.moved_ignore.index(event.dest_path)]
|
||||
|
||||
|
||||
class SyncClient(Routing.ClientRoute):
|
||||
def __init__(self, sock, folder, route, exclude=[".DS_Store", ".fl0w"], debug=False):
|
||||
self.sock = sock
|
||||
self.folder = folder if folder[-1] == "/" else folder + "/"
|
||||
self.started = False
|
||||
self.route = route
|
||||
self.exclude = exclude
|
||||
self.debug = debug
|
||||
self.files = relative_recursive_ls(folder, folder, exclude=self.exclude)
|
||||
self.suppressed_fs_events = []
|
||||
self.observer = Observer()
|
||||
self.observer.schedule(ReloadHandler(self, self.folder), path=self.folder, recursive=True)
|
||||
self.observer.start()
|
||||
|
||||
|
||||
def start(self):
|
||||
if self.debug:
|
||||
Logging.info("Sync client started!")
|
||||
out = {"list" : {}}
|
||||
for file in self.files:
|
||||
out["list"].update({file : {"mtime" : get_mtime(self.folder + file),
|
||||
"md5" : md5(self.folder + file)}})
|
||||
self.sock.send(out, self.route)
|
||||
|
||||
|
||||
def suppress_fs_event(self, file):
|
||||
if not file in self.suppressed_fs_events:
|
||||
self.suppressed_fs_events.append(file)
|
||||
|
||||
|
||||
def unsuppress_fs_event(self, file):
|
||||
if file in self.suppressed_fs_events:
|
||||
del self.suppressed_fs_events[self.suppressed_fs_events.index(file)]
|
||||
|
||||
|
||||
def run(self, data, handler):
|
||||
if type(data) is dict:
|
||||
if "add" in data:
|
||||
for file in data["add"]:
|
||||
self.suppress_fs_event(file)
|
||||
folder_path = self.folder + os.path.dirname(file)
|
||||
if not os.path.exists(folder_path):
|
||||
os.makedirs(folder_path)
|
||||
mode = "wb+" if os.path.exists(self.folder + file) else "ab+"
|
||||
open(self.folder + file, mode).write(base64.b64decode(data["add"][file]["content"]))
|
||||
os.utime(self.folder + file, (data["add"][file]["mtime"], data["add"][file]["mtime"]))
|
||||
"""
|
||||
for file in data["add"]:
|
||||
if not file in self.suppressed_fs_events:
|
||||
self.suppress_fs_event(file)
|
||||
folder_path = self.folder + os.path.dirname(file)
|
||||
if not os.path.exists(folder_path):
|
||||
os.makedirs(folder_path)
|
||||
if os.path.exists:
|
||||
open(self.folder + file, "wb+").write(base64.b64decode(data["add"][file]["content"]))
|
||||
else:
|
||||
open(self.folder + file, "ab+").write(base64.b64decode(data["add"][file]["content"]))
|
||||
os.utime(self.folder + file, (data["add"][file]["mtime"], data["add"][file]["mtime"]))
|
||||
else:
|
||||
self.unsuppress_fs_event(file)
|
||||
"""
|
||||
elif "del" in data:
|
||||
for file in data["del"]:
|
||||
self.suppress_fs_event(file)
|
||||
try:
|
||||
os.remove(self.folder + file)
|
||||
except FileNotFoundError:
|
||||
self.unsuppress_fs_event(file)
|
||||
Logging.warning("Possible server misbehaviour")
|
||||
elif "deldir" in data:
|
||||
for dir in data["deldir"]:
|
||||
if os.path.exists(self.folder + dir):
|
||||
for folder in relative_recursive_folder_ls(self.folder + dir, self.folder):
|
||||
self.suppress_fs_event(folder)
|
||||
for file in relative_recursive_ls(self.folder + dir, self.folder):
|
||||
self.suppress_fs_event(file)
|
||||
self.suppress_fs_event(dir)
|
||||
shutil.rmtree(self.folder + dir)
|
||||
else:
|
||||
Logging.warning("Possible server misbehaviour")
|
||||
elif "mvd" in data:
|
||||
for file in data["mvd"]:
|
||||
if os.path.isfile(self.folder + file):
|
||||
new_name = self.folder + data["mvd"][file]
|
||||
if os.path.exists(os.path.dirname(new_name)):
|
||||
self.suppress_fs_event(file)
|
||||
shutil.move(self.folder + file, new_name)
|
||||
elif "req" in data:
|
||||
for file in data["req"]:
|
||||
if file in self.files:
|
||||
self.sock.send({"add" : {file : {"content" : base64_str_decode(self.folder + file),
|
||||
"mtime" : os.path.getmtime(self.folder + file)}}}, self.route)
|
||||
|
||||
|
||||
def modified(self, event, created=False):
|
||||
if self.debug and not created:
|
||||
Logging.info("Modified file: '%s'" % event.src_path)
|
||||
relpath = os.path.relpath(event.src_path, self.folder)
|
||||
if relpath not in self.files:
|
||||
self.files.append(relpath)
|
||||
if not relpath in self.suppressed_fs_events:
|
||||
self.sock.send({"add" : {relpath : {"content" : base64_str_decode(event.src_path),
|
||||
"mtime" : os.path.getmtime(event.src_path)}}}, self.route)
|
||||
else:
|
||||
self.unsuppress_fs_event(relpath)
|
||||
|
||||
|
||||
def created(self, event):
|
||||
if self.debug:
|
||||
Logging.info("Created file: '%s'" % event.src_path)
|
||||
self.modified(event, created=True)
|
||||
|
||||
|
||||
def deleted(self, event):
|
||||
if self.debug:
|
||||
Logging.info("Deleted file: '%s'" % event.src_path)
|
||||
relpath = os.path.relpath(event.src_path, self.folder)
|
||||
if relpath in self.files:
|
||||
del self.files[self.files.index(relpath)]
|
||||
if not relpath in self.suppressed_fs_events:
|
||||
self.sock.send({"del" : [relpath]}, self.route)
|
||||
else:
|
||||
self.unsuppress_fs_event(relpath)
|
||||
|
||||
|
||||
def deleted_dir(self, event):
|
||||
if self.debug:
|
||||
Logging.info("Deleted folder: '%s'" % event.src_path)
|
||||
relpath = os.path.relpath(event.src_path, self.folder)
|
||||
if not relpath in self.suppressed_fs_events:
|
||||
self.sock.send({"deldir" : [relpath]}, self.route)
|
||||
else:
|
||||
self.unsuppress_fs_event(relpath)
|
||||
|
||||
def moved(self, event):
|
||||
if self.debug:
|
||||
Logging.info("Moved file: '%s'" % event.src_path)
|
||||
src_relpath = os.path.relpath(event.src_path, self.folder)
|
||||
dest_relpath = os.path.relpath(event.dest_path, self.folder)
|
||||
if src_relpath in self.files:
|
||||
del self.files[self.files.index(src_relpath)]
|
||||
self.files.append(dest_relpath)
|
||||
if not src_relpath in self.suppressed_fs_events:
|
||||
self.sock.send({"mvd" : {src_relpath : dest_relpath}}, self.route)
|
||||
else:
|
||||
self.unsuppress_fs_event(src_relpath)
|
||||
|
||||
|
||||
def moved_dir(self, event):
|
||||
print(event)
|
||||
print(dir(event))
|
||||
|
||||
|
||||
class SyncServer(Routing.ServerRoute):
|
||||
REQUIRED = (Routing.BROADCAST, Routing.ROUTE)
|
||||
|
||||
def __init__(self, folder, channel, exclude=[".DS_Store", ".fl0w"], debug=False, deleted_db_path="deleted.db", modified_hook=None, deleted_hook=None):
|
||||
self.folder = folder if folder[-1] == "/" else folder + "/"
|
||||
self.channel = channel
|
||||
self.exclude = exclude
|
||||
self.debug = debug
|
||||
self.deleted_storage = DeletedStorage(deleted_db_path)
|
||||
self.modified_hook = modified_hook
|
||||
self.deleted_hook = deleted_hook
|
||||
self.broadcast_file_excludes = {}
|
||||
self.broadcast_dir_excludes = {}
|
||||
self.ignore_events = []
|
||||
self.route = None # Set by REQUIRED
|
||||
self.broadcast = None # Set by REQUIRED
|
||||
self.started = False
|
||||
|
||||
|
||||
def start(self, handler):
|
||||
if not self.started:
|
||||
if self.debug:
|
||||
Logging.info("Sync server on route '%s' started!" % self.route)
|
||||
self.files = relative_recursive_ls(self.folder, self.folder, exclude=self.exclude)
|
||||
observer = Observer()
|
||||
observer.schedule(ReloadHandler(self, self.folder), path=self.folder, recursive=True)
|
||||
observer.start()
|
||||
self.started = True
|
||||
|
||||
|
||||
def run(self, data, handler):
|
||||
if type(data) is dict:
|
||||
if "list" in data:
|
||||
client_files = []
|
||||
for file in data["list"]:
|
||||
client_file = File(file, data["list"][file]["md5"], data["list"][file]["mtime"])
|
||||
client_files.append(client_file)
|
||||
# If file doesn't exist on server check if it's in deleted storage
|
||||
if not file in self.files:
|
||||
# Delete storage works with objects
|
||||
file = File(file, data["list"][file]["md5"], data["list"][file]["mtime"])
|
||||
if file in self.deleted_storage.files:
|
||||
handler.sock.send({"del" : [file.relpath]}, self.route)
|
||||
else:
|
||||
# Request the file if it's not in delete storage
|
||||
handler.sock.send({"req" : [file.relpath]}, self.route)
|
||||
# Add all files that are not on the client but preserve newer ones by requesting them
|
||||
for file in self.files:
|
||||
file = File(file, md5(self.folder + file), get_mtime(self.folder + file))
|
||||
# Make sure that the newer version will be requested
|
||||
for client_file in client_files:
|
||||
if file.relpath == client_file.relpath:
|
||||
if client_file.mtime > file.mtime:
|
||||
client_files.append(file) # Hack that makes the iteration below work
|
||||
handler.sock.send({"req" : [file.relpath]}, self.route)
|
||||
# Not the same file because mtime is different
|
||||
if file not in client_files:
|
||||
handler.sock.send({"add" : {file.relpath : {"content" : base64_str_decode(self.folder + file.relpath),
|
||||
"mtime" : os.path.getmtime(self.folder + file.relpath)}}}, self.route)
|
||||
elif "add" in data:
|
||||
for file in data["add"]:
|
||||
folder_path = self.folder + os.path.dirname(file)
|
||||
if not os.path.exists(folder_path):
|
||||
os.makedirs(folder_path)
|
||||
mode = "wb+" if os.path.exists(self.folder + file) else "ab+"
|
||||
self.broadcast_file_excludes[file] = handler
|
||||
open(self.folder + file, mode).write(base64.b64decode(data["add"][file]["content"]))
|
||||
os.utime(self.folder + file, (data["add"][file]["mtime"], data["add"][file]["mtime"]))
|
||||
elif "deldir" in data:
|
||||
for dir in data["deldir"]:
|
||||
if os.path.exists(self.folder + dir):
|
||||
for folder in relative_recursive_folder_ls(self.folder + dir, self.folder):
|
||||
self.ignore_events.append(folder)
|
||||
for file in relative_recursive_ls(self.folder + dir, self.folder):
|
||||
self.ignore_events.append(file)
|
||||
if file in self.files:
|
||||
del self.files[self.files.index(file)]
|
||||
self.broadcast_dir_excludes[dir] = handler
|
||||
shutil.rmtree(self.folder + dir)
|
||||
else:
|
||||
Logging.warning("Possible client misbehaviour (%s:%d)" % (handler.info[0], handler.info[1]))
|
||||
elif "del" in data:
|
||||
for file in data["del"]:
|
||||
self.broadcast_file_excludes[file] = handler
|
||||
self.deleted_storage.add(File(file, md5(self.folder + file), get_mtime(self.folder + file)))
|
||||
try:
|
||||
os.remove(self.folder + file)
|
||||
except FileNotFoundError:
|
||||
del self.broadcast_file_excludes[file]
|
||||
Logging.warning("Possible client misbehaviour (%s:%d)" % (handler.info[0], handler.info[1]))
|
||||
elif "mvd" in data:
|
||||
for file in data["mvd"]:
|
||||
if os.path.isfile(self.folder + file):
|
||||
new_name = self.folder + data["mvd"][file]
|
||||
if os.path.exists(os.path.dirname(new_name)):
|
||||
self.broadcast_file_excludes[file] = handler
|
||||
shutil.move(self.folder + file, new_name)
|
||||
|
||||
|
||||
def modified(self, event, created=False):
|
||||
if self.debug and not created:
|
||||
Logging.info("Modified file: '%s'" % event.src_path)
|
||||
relpath = os.path.relpath(event.src_path, self.folder)
|
||||
if relpath not in self.files:
|
||||
self.files.append(relpath)
|
||||
exclude = []
|
||||
if relpath in self.broadcast_file_excludes:
|
||||
exclude.append(self.broadcast_file_excludes[relpath])
|
||||
self.broadcast.broadcast({"add" : {relpath : {
|
||||
"content" : base64_str_decode(event.src_path),
|
||||
"mtime" : os.path.getmtime(event.src_path)}}}, self.route, self.channel, exclude=exclude)
|
||||
if self.modified_hook != None:
|
||||
self.modified_hook(self.folder, relpath, exclude[0] if len(exclude) == 1 else None)
|
||||
if exclude != []:
|
||||
del self.broadcast_file_excludes[relpath]
|
||||
|
||||
|
||||
def created(self, event):
|
||||
if self.debug:
|
||||
Logging.info("Created file: '%s'" % event.src_path)
|
||||
self.modified(event, created=True)
|
||||
|
||||
|
||||
|
||||
def deleted(self, event):
|
||||
if self.debug:
|
||||
Logging.info("Deleted file: '%s'" % event.src_path)
|
||||
relpath = os.path.relpath(event.src_path, self.folder)
|
||||
if not relpath in self.ignore_events:
|
||||
if relpath in self.files:
|
||||
del self.files[self.files.index(relpath)]
|
||||
exclude = []
|
||||
if relpath in self.broadcast_file_excludes:
|
||||
exclude.append(self.broadcast_file_excludes[relpath])
|
||||
self.broadcast.broadcast({"del" : [relpath]}, self.route, self.channel, exclude=exclude)
|
||||
if self.deleted_hook != None:
|
||||
self.deleted_hook(self.folder, relpath, exclude[0] if len(exclude) == 1 else None)
|
||||
if exclude != []:
|
||||
del self.broadcast_file_excludes[relpath]
|
||||
else:
|
||||
del self.ignore_events[self.ignore_events.index(relpath)]
|
||||
|
||||
|
||||
def deleted_dir(self, event):
|
||||
if self.debug:
|
||||
Logging.info("Deleted folder: '%s'" % event.src_path)
|
||||
relpath = os.path.relpath(event.src_path, self.folder)
|
||||
if not relpath in self.ignore_events:
|
||||
exclude = []
|
||||
if relpath in self.broadcast_dir_excludes:
|
||||
exclude.append(self.broadcast_dir_excludes[relpath])
|
||||
self.broadcast.broadcast({"deldir" : [relpath]}, self.route, self.channel, exclude=exclude)
|
||||
if exclude != []:
|
||||
del self.broadcast_dir_excludes[relpath]
|
||||
else:
|
||||
del self.ignore_events[self.ignore_events.index(relpath)]
|
||||
|
||||
|
||||
def moved(self, event):
|
||||
if self.debug:
|
||||
Logging.info("Moved file: '%s'" % event.src_path)
|
||||
src_relpath = os.path.relpath(event.src_path, self.folder)
|
||||
dest_relpath = os.path.relpath(event.dest_path, self.folder)
|
||||
if src_relpath in self.files:
|
||||
del self.files[self.files.index(src_relpath)]
|
||||
self.files.append(dest_relpath)
|
||||
exclude = []
|
||||
if src_relpath in self.broadcast_file_excludes:
|
||||
exclude.append(self.broadcast_file_excludes[src_relpath])
|
||||
self.broadcast.broadcast({"mvd" : {src_relpath : dest_relpath}}, self.route, self.channel, exclude=exclude)
|
||||
if exclude != []:
|
||||
del self.broadcast_file_excludes[src_relpath]
|
||||
|
||||
|
||||
def moved_dir(self, event):
|
||||
print(event)
|
||||
|
||||
|
||||
def stop(self):
|
||||
self.deleted_storage.close()
|
||||
Reference in a new issue