From 5c7b8c760cc66f1e5263b5e9c63e4d253a3c2305 Mon Sep 17 00:00:00 2001 From: Kyle Barry Date: Tue, 1 Mar 2022 09:37:03 +0200 Subject: [PATCH 1/6] update dropbox and add setup.py --- requirements.txt | 2 +- setup.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 setup.py diff --git a/requirements.txt b/requirements.txt index 65247b8..e23f23c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -dropbox==11.26.0 \ No newline at end of file +dropbox==11.27.0 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b618506 --- /dev/null +++ b/setup.py @@ -0,0 +1,10 @@ +from setuptools import setup + +setup( + name='cloudsync', + version='0.1', + description='A useful module', + author='aicpp(https://github.com/aicpp), Kyle Barry', + packages=['cloudsync'], + install_requires=['dropbox'], +) From b6d180942ee3e5a013802fdb82f12ecf74adf88f Mon Sep 17 00:00:00 2001 From: Kyle Barry Date: Tue, 1 Mar 2022 10:20:14 +0200 Subject: [PATCH 2/6] create local state dir in setup script --- cloudsync.py => cloudsync/cloudsync.py | 2 +- dropboxsync.py => cloudsync/dropboxsync.py | 2 +- filters.py => cloudsync/filters.py | 0 logger.py => cloudsync/logger.py | 0 setup.py | 16 ++++++++++++++++ 5 files changed, 18 insertions(+), 2 deletions(-) rename cloudsync.py => cloudsync/cloudsync.py (99%) rename dropboxsync.py => cloudsync/dropboxsync.py (99%) rename filters.py => cloudsync/filters.py (100%) rename logger.py => cloudsync/logger.py (100%) diff --git a/cloudsync.py b/cloudsync/cloudsync.py similarity index 99% rename from cloudsync.py rename to cloudsync/cloudsync.py index 2a74c43..21143a7 100755 --- a/cloudsync.py +++ b/cloudsync/cloudsync.py @@ -1,7 +1,7 @@ #!/usr/bin/python import os -import sys, locale +import sys import dropboxsync import logging diff --git a/dropboxsync.py b/cloudsync/dropboxsync.py similarity index 99% rename from dropboxsync.py rename to cloudsync/dropboxsync.py index d1ff086..c588c62 100644 --- a/dropboxsync.py +++ b/cloudsync/dropboxsync.py @@ -5,7 +5,7 @@ import datetime import dropbox -from dropbox.files import FileMetadata, FolderMetadata +from dropbox.files import FileMetadata import os import time import filters diff --git a/filters.py b/cloudsync/filters.py similarity index 100% rename from filters.py rename to cloudsync/filters.py diff --git a/logger.py b/cloudsync/logger.py similarity index 100% rename from logger.py rename to cloudsync/logger.py diff --git a/setup.py b/setup.py index b618506..9c80f21 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,19 @@ from setuptools import setup +from setuptools.command.install import install +from pathlib import Path + + +class CreateLocalStateDir(install): + """ + Creates a local state directory to store + hashes, source and target files, keys, etc. + """ + + def run(self): + home = Path.home() / ".cloudsync" + home.mkdir(parents=True, exist_ok=True) + install.run(self) + setup( name='cloudsync', @@ -7,4 +22,5 @@ author='aicpp(https://github.com/aicpp), Kyle Barry', packages=['cloudsync'], install_requires=['dropbox'], + cmdclass={'install': CreateLocalStateDir}, ) From 1ed3a6ddbe69b8413c3caed2a6a1b3d121fa77c3 Mon Sep 17 00:00:00 2001 From: Kyle Barry Date: Tue, 1 Mar 2022 16:18:25 +0200 Subject: [PATCH 3/6] add SyncFile and FileHandler objects to more cleanly work with files --- cloudsync/__init__.py | 0 cloudsync/cloudsync.py | 8 +- cloudsync/dropboxsync.py | 200 +++++++----------- cloudsync/filters.py | 102 --------- cloudsync/sync_file/__init__.py | 1 + cloudsync/sync_file/file_handler/__init__.py | 2 + .../sync_file/file_handler/dropbox_handler.py | 42 ++++ .../sync_file/file_handler/file_handler.py | 47 ++++ .../sync_file/file_handler/local_handler.py | 34 +++ cloudsync/sync_file/filters/__init__.py | 1 + cloudsync/sync_file/filters/filter_params.py | 47 ++++ cloudsync/sync_file/sync_file.py | 60 ++++++ requirements.txt | 3 +- 13 files changed, 313 insertions(+), 234 deletions(-) create mode 100644 cloudsync/__init__.py delete mode 100644 cloudsync/filters.py create mode 100644 cloudsync/sync_file/__init__.py create mode 100644 cloudsync/sync_file/file_handler/__init__.py create mode 100644 cloudsync/sync_file/file_handler/dropbox_handler.py create mode 100644 cloudsync/sync_file/file_handler/file_handler.py create mode 100644 cloudsync/sync_file/file_handler/local_handler.py create mode 100644 cloudsync/sync_file/filters/__init__.py create mode 100644 cloudsync/sync_file/filters/filter_params.py create mode 100644 cloudsync/sync_file/sync_file.py diff --git a/cloudsync/__init__.py b/cloudsync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudsync/cloudsync.py b/cloudsync/cloudsync.py index 21143a7..5bb2cec 100755 --- a/cloudsync/cloudsync.py +++ b/cloudsync/cloudsync.py @@ -7,7 +7,8 @@ import logging import logger as lgr import argparse -import filters as ftr + +from sync_file.filters import FilterParameters def isCronMode(): @@ -63,9 +64,8 @@ def main(): dbSync.setLogger(logger) dbSync.prepare() - filters = [] - filters.append(ftr.FileFilterDays(matchDays=dbSync.args['match_days'])) - filters.append(ftr.FileFilterMask()) + filters = FilterParameters() + filters.days = dbSync.args['match_days'] dbSync.filterSourceFiles(filters) dbSync.synchronize() diff --git a/cloudsync/dropboxsync.py b/cloudsync/dropboxsync.py index c588c62..134d384 100644 --- a/cloudsync/dropboxsync.py +++ b/cloudsync/dropboxsync.py @@ -3,33 +3,40 @@ import logging import contextlib -import datetime import dropbox from dropbox.files import FileMetadata +from dropbox.exceptions import ApiError import os +from pathlib import Path import time -import filters +from typing import List, Optional import unicodedata +from sync_file import SyncFile +from sync_file.file_handler import DropboxFileHandler, LocalFileHandler + class DropboxSync(object): """ Class to help synchronize files to/from dropbox use Dropbox API v2 (https://github.com/dropbox/dropbox-sdk-python) """ + + locList: List[SyncFile] = [] + dbList: List[SyncFile] = [] + filterItems: List[SyncFile] = [] + sourceFilesMatched: List[SyncFile] = [] + db_handler: Optional[DropboxFileHandler] = None + def __init__(self, args): self.args=args self.dbx = None - self.localDir = self.normalizeDir(args['localdir']) - self.dropboxDir = self.normalizeDir(args['dropboxdir']) + self.localDir = Path(args['localdir']) + self.dropboxDir = Path(args['dropboxdir']) self.directionToDb = args['direction'] == 'todropbox' self.timeoutSec = 2 * 60 self.logger = logging.getLogger(__name__) self.logger.addHandler(logging.NullHandler()) - self.dbList = [] - self.locList = [] - self.filterItems = [] - self.sourceFilesMatched = [] def setLogger(self, logger): self.logger = logger @@ -49,6 +56,7 @@ def prepare(self): def prepareDropboxAuth(self): self.logger.debug('Connecting to dropbox using token...') self.dbx = dropbox.Dropbox(self.args['token']) + self.db_handler = DropboxFileHandler(self.dbx) self.logger.debug('Dropbox connected') def checkLocalDir(self): @@ -71,76 +79,34 @@ def checkDropboxDir(self): """ self.logger.debug('Checking if Dropbox folder exists...') try: - self.dbx.files_list_folder(self.dropboxDir) + self.dbx.files_list_folder(str(self.dropboxDir)) self.logger.debug('Dropbox folder exists') except: - self.logger.error(f"Folder {self.dropboxDir} does not exist on Dropbox") + self.logger.error(f"Folder {str(self.dropboxDir)} does not exist on Dropbox") exit(-1) - def listLocalFiles(self): self.logger.debug('Getting list of local files...') - locList = [unicodedata.normalize('NFC', f) for f in os.listdir(self.localDir) if os.path.isfile(os.path.join(self.localDir,f))] - self.locList = [self.filterItemByLocal(f) for f in locList] - self.logger.debug('Local files:%s' % len(self.locList)) + self.locList = [ + SyncFile(self.localDir / Path(unicodedata.normalize('NFC', f))) + for f in os.listdir(self.localDir) + if os.path.isfile(self.localDir / f) + ] + self.logger.debug(f'Local files: {len(self.locList)}') return True - def mtime(self, filePath): - mtime = os.path.getmtime(filePath) - return datetime.datetime(*time.gmtime(mtime)[:6]) - # t = os.path.getmtime(filePath) - # return datetime.datetime.fromtimestamp(t) - - def filterItemByLocal(self, fileName): - filePath = os.path.join(self.localDir, fileName) - return filters.FileFilterItem( - name=fileName, - mtime=self.mtime(filePath), - size=os.path.getsize(filePath)) - - def filterItemByDropbox(self, fileMd): - the_dict = { - 'name': fileMd.name - } - - #Check if file has these attributes before filtering by them - if hasattr(fileMd, 'cliend_modified'): - the_dict['client_modified'] = fileMd.client_modified - if hasattr(fileMd, 'size'): - the_dict['size'] = fileMd.size - return filters.FileFilterItem( - **the_dict - ) - - def normalizeDir(self, directory): - result = directory.replace(os.path.sep, '/') - result = os.path.expanduser(result) - while '//' in result: - result = result.replace('//', '/') - result = result.rstrip('/') - result = unicodedata.normalize('NFC', result) - return result - # filtration def listFilterItems(self): - resItems = [] if self.directionToDb: self.filterItems = self.locList else: self.filterItems = self.dbList def filterSourceFiles(self, filters): - resFiles = self.filterItems - sourceCount = len(resFiles) - self.logger.debug('Source files:%s' % (len(resFiles))) - for fltr in filters: - prevCount = len(resFiles) - resFiles = fltr.filterFiles(resFiles) - resCount = len(resFiles) - if resCount != prevCount: - self.logger.debug('Filter \'%s\': %s -> %s' % (fltr.__class__.__name__, prevCount, resCount)) - self.sourceFilesMatched = resFiles - self.logger.info('--- Filter source files: %d -> %d' % (sourceCount, len(resFiles))) + source_count = len(self.filterItems) + self.logger.debug(f'Source files: {source_count}') + self.sourceFilesMatched = [f for f in self.filterItems if not f.filter(filters)] + self.logger.info(f'--- Filter source files: {source_count} -> {len(self.sourceFilesMatched)}') # synchronize def synchronize(self): @@ -158,22 +124,25 @@ def synchronize(self): def deleteLocalFiles(self): # remove local - sourceNames = [fileItem.fileName for fileItem in self.sourceFilesMatched] - delList = [fileItem for fileItem in self.locList if fileItem.fileName not in sourceNames] + sourceNames = [fileItem.name for fileItem in self.sourceFilesMatched] + delList = [fileItem for fileItem in self.locList if fileItem.name not in sourceNames] if not delList: return - self.logger.debug('Local files to delete:%s' % len(delList)) + self.logger.debug(f'Local files to delete: {len(delList)}') for fileItem in delList: - os.remove(os.path.join(self.localDir, fileItem.fileName)) - self.logger.info('--- Delete %d/%d local files' % (len(delList), len(self.locList))) + fileItem.delete() + self.logger.info(f'--- Deleted {len(delList)}/{len(self.locList)} local files') def syncToLocal(self): countSuccess = 0 countSkip = 0 countFails = 0 for fileItem in self.sourceFilesMatched: + + self.db_handler.file = self.dropboxDir / fileItem.name + if fileItem in self.locList: - self.logger.debug('Skip existed:%s' % fileItem.fileName) + self.logger.debug(f'Skip existed: {fileItem.name}') countSkip += 1 continue if self.downloadFile(fileItem): @@ -187,8 +156,8 @@ def syncToLocal(self): def deleteDropboxFiles(self): """ Delete not matched files from Dropbox directory """ - sourceNames = [fileItem.fileName for fileItem in self.sourceFilesMatched] - delList = [fileItem for fileItem in self.dbList if fileItem.fileName not in sourceNames] + sourceNames = [fileItem.name for fileItem in self.sourceFilesMatched] + delList = [fileItem for fileItem in self.dbList if fileItem.name not in sourceNames] if not delList: return self.logger.debug('Dropbox files to delete:%s' % len(delList)) @@ -202,7 +171,7 @@ def syncToDropbox(self): countFails = 0 for fileItem in self.sourceFilesMatched: if fileItem in self.dbList: - self.logger.debug('Skip existed:%s' % fileItem.fileName) + self.logger.debug('Skip existed:%s' % fileItem) countSkip += 1 continue if self.uploadFile(fileItem): @@ -219,91 +188,68 @@ def listDropboxFiles(self): """List a folder. Return an array of filter items """ - path = self.dropboxDir self.logger.debug('Downloading dropbox list files...') try: with self.stopwatch(__name__): - res = self.dbx.files_list_folder(path) - except dropbox.exceptions.ApiError as err: + res = self.dbx.files_list_folder(str(self.dropboxDir)) + except ApiError as err: self.dbList = [] - raise Exception('Folder listing failed for %s -- assumed empty:%s' % (path, err)) + raise Exception('Folder listing failed for %s -- assumed empty:%s' % (str(self.dropboxDir), err)) else: self.logger.debug('Dropbox files:%s' % len(res.entries)) - self.dbList = [self.filterItemByDropbox(fileMd) for fileMd in res.entries] + self.dbList = [SyncFile(self.dropboxDir / dbfile.name, file_handler=self.db_handler) for dbfile in res.entries] - def downloadFile(self, fileItem): + def downloadFile(self, file_item: SyncFile): """Download a file. Return True when success, or False if error occurs. """ - fileName = fileItem.fileName - dbItem = next((f for f in self.dbList if f.fileName == fileName), None) - dbPath = os.path.join(self.dropboxDir, fileName) - locPath = os.path.join(self.localDir, fileName) - self.logger.debug('Downloading %s (%d bytes) ...' % (fileName, dbItem.fileSize)) + + db_path = self.dropboxDir / file_item.name + local_path = self.localDir / file_item.name + file_size = SyncFile(db_path, file_handler=self.db_handler).size + + self.logger.debug(f'Downloading {file_item.name} ({file_size} bytes) ...') with self.stopwatch('downloading'): try: - md = self.dbx.files_download_to_file(locPath, dbPath) - except dropbox.exceptions.ApiError as err: - raise Exception('%s - API error:%s' % (fileName, err)) - self.logger.debug('Success download - %s (%d bytes)' % (fileName, dbItem.fileSize)) + self.dbx.files_download_to_file(str(local_path), str(db_path)) + except ApiError as err: + raise Exception(f'{file_item.name} - API error: {err}') + self.logger.debug(f'Success download - {file_item.name} ({file_size} bytes)') return True - def uploadFile(self, fileItem): + def uploadFile(self, file_item: SyncFile): """Upload a file. Return the request response, or None in case of error. """ - fileName = fileItem.fileName - dbPath = os.path.join(self.dropboxDir, fileName) - locPath = os.path.join(self.localDir, fileName) + db_path = self.dropboxDir / file_item.name + local_path = self.localDir / file_item.name + file_size = file_item.size mode = dropbox.files.WriteMode.overwrite - # mtime0 = os.path.getmtime(locPath) - # mtime = datetime.datetime(*time.gmtime(mtime0)[:6]) - mtime = self.mtime(locPath) - with open(locPath, 'rb') as f: + + with open(local_path, 'rb') as f: data = f.read() - self.logger.debug('Uploading %s (%d bytes) ...' % (fileName, len(data))) - # self.logger.debug('mtime %d %d ...' % (mtime, fileItem.fileModifyTime)) + self.logger.debug(f'Uploading {file_item.name} ({file_size} bytes) ...') with self.stopwatch('uploading'): try: - res = self.dbx.files_upload( - data, dbPath, mode, - client_modified=mtime, + self.dbx.files_upload( + data, db_path, mode, + client_modified=file_item.mod_time, autorename=False, mute=True) except dropbox.exceptions.ApiError as err: - raise Exception('%s - API error:%s' % (fileName, err)) - # self.logger.debug('Success upload - res:%s' % (res)) - self.logger.debug('Success upload - %s (%s bytes)' % (fileName, len(data))) + raise Exception(f'{file_item.name}- API error: {err}') + self.logger.debug(f'Success upload - {file_item.name} ({file_size} bytes)') return True - def deleteFile(self, fileItem): - self.logger.debug('Deleting - \'%s\'' % (fileItem.fileName)) + def deleteFile(self, file_item: SyncFile): + self.logger.debug(f'Deleting - \'{file_item.name}\'') with self.stopwatch('deleting'): try: - md = self.dbx.files_delete(os.path.join(self.dropboxDir, fileItem.fileName)) - except dropbox.exceptions.ApiError as err: - raise Exception('%s - API error:%s' % (fileItem.fileName, err)) - self.logger.debug('Success delete - %s' % fileItem.fileName) - - # process helpers - def fixLocalTimestamps(self): - for fileName in self.locList: - self._debugFixLocalTimestamp(fileName) - self.logger.debug('Timestamps fixed in local files:%s' % len(self.locList)) - - def _debugFixLocalTimestamp(self, fileName): - from datetime import datetime - basename = os.path.splitext(fileName)[0] - newTime0 = None - if "." in basename: - newTime0 = datetime.strptime(basename, "%Y-%m-%d %H.%M.%S") - else: - newTime0 = datetime.strptime(basename, "%Y-%m-%d %H-%M-%S") - - newTime = time.mktime(newTime0.timetuple()) - # self.logger.debug('%s - %s -> %s' % (fileName, newTime0, newTime)) - os.utime(os.path.join(self.localDir, fileName), (newTime, newTime)) + file_item.delete() + except ApiError as err: + raise Exception(f'{file_item.name} - API error: {err}') + self.logger.debug(f'Success delete - {file_item.name}') @contextlib.contextmanager def stopwatch(self, message): diff --git a/cloudsync/filters.py b/cloudsync/filters.py deleted file mode 100644 index b9f23ce..0000000 --- a/cloudsync/filters.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - -import datetime -import os - -class FileFilterItem(object): - """ - Contains file properties for filter - """ - def __init__(self, name=None, mtime=None, size=None): - self.fileName = name - self.fileModifyTime = mtime - self.fileSize = size - - def __eq__(self, other): - if type(other) is type(self): - return self.__dict__ == other.__dict__ - return False - - # def __eq__(self, other): - # if type(other) is not type(self): - # return False - # if self.fileName != other.fileName: - # return False - # if self.fileModifyTime != other.fileModibyTime: - # return False - # if self.fileSize != other.fileSize: - # return False - # return True - - def __ne__(self, other): - return not self.__eq__(other) - -class FileFilterBase(object): - """ - Base class to filter files from source list to target - Should get [FileFilterItem] objects - """ - def __init__(self): - pass - - def checkItemType(self, fileItem): - """ Check to match type [FileFilterItem] """ - if not isinstance(fileItem, FileFilterItem): - raise Exception('invalid type') - - def isMatch(self, fileItem): - """ Return true, if file is match filter """ - return True - - def filterFiles(self, files): - """ Return matched files """ - return [f for f in files if self.isMatch(f)] - -class FileFilterDays(FileFilterBase): - """ - Match only files which modification time is newer than matchDays days - """ - - def __init__(self, matchDays=None): - super(FileFilterDays, self).__init__() - self.matchDays = 0 - if matchDays: - self.matchDays = int(matchDays) - - def isMatch(self, fileItem): - if not self.matchDays: - return True - self.checkItemType(fileItem) - if fileItem.fileModifyTime: - now = datetime.datetime.now() - diff = now - fileItem.fileModifyTime - diffDays = diff.days - return diffDays < self.matchDays - return True - -class FileFilterMask(FileFilterBase): - """ - Exclude temporary files by mask (use fnmatch.fnmatch) - """ - - def __init__(self): - super(FileFilterMask, self).__init__() - self.excludeMasks = self.defaultMasks() - - def defaultMasks(self): - result = [] - result.append('.*') # any hidden files - result.append('~*') # temp files - result.append('thumbs.db') # windows thumbs - return result - - def addMask(self, fileMask): - self.excludeMasks.append(fileMask) - - def isMatch(self, fileItem): - import fnmatch - self.checkItemType(fileItem) - for fileMask in self.excludeMasks: - if fnmatch.fnmatch(fileItem.fileName, fileMask): - return False - return True diff --git a/cloudsync/sync_file/__init__.py b/cloudsync/sync_file/__init__.py new file mode 100644 index 0000000..36495c7 --- /dev/null +++ b/cloudsync/sync_file/__init__.py @@ -0,0 +1 @@ +from .sync_file import SyncFile \ No newline at end of file diff --git a/cloudsync/sync_file/file_handler/__init__.py b/cloudsync/sync_file/file_handler/__init__.py new file mode 100644 index 0000000..a12bbeb --- /dev/null +++ b/cloudsync/sync_file/file_handler/__init__.py @@ -0,0 +1,2 @@ +from .local_handler import LocalFileHandler +from .dropbox_handler import DropboxFileHandler diff --git a/cloudsync/sync_file/file_handler/dropbox_handler.py b/cloudsync/sync_file/file_handler/dropbox_handler.py new file mode 100644 index 0000000..2fcb332 --- /dev/null +++ b/cloudsync/sync_file/file_handler/dropbox_handler.py @@ -0,0 +1,42 @@ +import time + +from dropbox import Dropbox +from dropbox.files import FileMetadata + +from .file_handler import FileHandler + + +class DropboxFileHandler(FileHandler): + + def __init__(self, db_obj: Dropbox): + self.dbx = db_obj + + @property + def mod_time(self): + metadata = self.dbx.files_get_metadata(str(self.file)) + assert isinstance(metadata, FileMetadata) + + client_modify = time.mktime(metadata.client_modified.timetuple()) + server_modify = time.mktime(metadata.server_modified.timetuple()) + + if client_modify > server_modify: + return client_modify + return server_modify + + def create(self): + pass + + def delete(self): + self.dbx.files_delete_v2(str(self.file)) + + def hash(self): + metadata = self.dbx.files_get_metadata(str(self.file)) + assert isinstance(metadata, FileMetadata) + + return metadata.content_hash + + def size(self): + metadata = self.dbx.files_get_metadata(str(self.file)) + assert isinstance(metadata, FileMetadata) + + return metadata.size diff --git a/cloudsync/sync_file/file_handler/file_handler.py b/cloudsync/sync_file/file_handler/file_handler.py new file mode 100644 index 0000000..f18cdda --- /dev/null +++ b/cloudsync/sync_file/file_handler/file_handler.py @@ -0,0 +1,47 @@ +from abc import ABC, abstractmethod +from pathlib import Path + +from ..filters import FilterParameters + + +class FileHandler(ABC): + """ + Generic class that facilitates reading file metadata, + creating, deleting and modifying files. + """ + + _file: Path = "" + + @property + def file(self): + return self._file + + @file.setter + def file(self, new_file: Path): + self._file = new_file + + @property + @abstractmethod + def mod_time(self): + pass + + @property + @abstractmethod + def hash(self): + pass + + @property + @abstractmethod + def size(self): + pass + + @abstractmethod + def create(self): + pass + + @abstractmethod + def delete(self): + pass + + def filter(self, params: FilterParameters) -> bool: + return params.filter_days(self.mod_time) or params.filter_name(str(self.file)) diff --git a/cloudsync/sync_file/file_handler/local_handler.py b/cloudsync/sync_file/file_handler/local_handler.py new file mode 100644 index 0000000..07a7d01 --- /dev/null +++ b/cloudsync/sync_file/file_handler/local_handler.py @@ -0,0 +1,34 @@ +import hashlib +import os + +from .file_handler import FileHandler + +DROPBOX_HASH_CHUNK_SIZE = 4*1024*1024 + + +class LocalFileHandler(FileHandler): + + @property + def mod_time(self): + return int(os.path.getmtime(self.file)) + + def create(self): + self.file.open("a+") + + def delete(self): + self.file.unlink() + + def hash(self): + with open(self.file, 'rb') as f: + block_hashes = b'' + while True: + chunk = f.read(DROPBOX_HASH_CHUNK_SIZE) + if not chunk: + break + block_hashes += hashlib.sha256(chunk).digest() + return hashlib.sha256(block_hashes).hexdigest() + + def size(self): + return self.file.stat().st_size + + diff --git a/cloudsync/sync_file/filters/__init__.py b/cloudsync/sync_file/filters/__init__.py new file mode 100644 index 0000000..077e867 --- /dev/null +++ b/cloudsync/sync_file/filters/__init__.py @@ -0,0 +1 @@ +from .filter_params import FilterParameters diff --git a/cloudsync/sync_file/filters/filter_params.py b/cloudsync/sync_file/filters/filter_params.py new file mode 100644 index 0000000..fbe49a4 --- /dev/null +++ b/cloudsync/sync_file/filters/filter_params.py @@ -0,0 +1,47 @@ +from datetime import datetime, timedelta +import re +import time + + +class FilterParameters: + + def __init__(self): + self._days = None + self._size = None + self._name_regexes = [ + re.compile('^\\..+|/\\..+'), # match hidden files + re.compile('^~.*|/~.*'), # match temporary files + ] + + @property + def days(self): + return self._days + + @days.setter + def days(self, val: int): + self._days = val + + @property + def size(self): + return self._size + + @size.setter + def size(self, val: int): + self._size = val + + def filter_name(self, file_name: str) -> bool: + for reg in self._name_regexes: + if reg.match(file_name): + return True + return False + + def filter_days(self, file_mod: int) -> bool: + if self._days is None: + return False + + threshold_day = time.mktime((datetime.today() - timedelta(days=self.days)).timetuple()) + + return threshold_day < file_mod + + def filter_size(self, file_size: int) -> bool: + return self.size > file_size diff --git a/cloudsync/sync_file/sync_file.py b/cloudsync/sync_file/sync_file.py new file mode 100644 index 0000000..3e611d8 --- /dev/null +++ b/cloudsync/sync_file/sync_file.py @@ -0,0 +1,60 @@ +from pathlib import Path + +from .file_handler import LocalFileHandler +from .filters import FilterParameters + + +class SyncFile: + """ + File object for easy comparison between local and Dropbox files. + Allows operations (such as equality checking) to be done independently + of the file source. + """ + + def __init__(self, raw_file: Path, file_handler=None): + self.file_handler = file_handler if file_handler is not None else LocalFileHandler() + self.file_handler.file = raw_file + self._name = raw_file + + @property + def mod_time(self): + return self.file_handler.mod_time + + @property + def name(self): + return self._name.name + + @property + def hash(self): + return self.file_handler.hash() + + @property + def size(self): + return self.file_handler.size() + + def filter(self, params: FilterParameters) -> bool: + return self.file_handler.filter(params) + + def delete(self): + self.file_handler.delete() + + def __repr__(self): + return str(self) + + def __str__(self): + return str(self.name) + + def __eq__(self, other: 'SyncFile'): + if other.name != self.name: + return False + + if other.size != self.size: + return False + + if self.mod_time < other.mod_time: + return self.hash == other.hash + + return False + + def __ne__(self, other): + return not self == other diff --git a/requirements.txt b/requirements.txt index e23f23c..9435a89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -dropbox==11.27.0 \ No newline at end of file +dropbox==11.27.0 +PyQt5==5.15.6 \ No newline at end of file From f713fa17274f0a64fb38e072063bad7511baf65d Mon Sep 17 00:00:00 2001 From: Kyle Barry Date: Tue, 1 Mar 2022 17:32:55 +0200 Subject: [PATCH 4/6] add recursive syncing --- cloudsync/cloudsync.py | 31 +++++++++++++++++++++++++++++-- cloudsync/dropboxsync.py | 21 +++++++++++---------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/cloudsync/cloudsync.py b/cloudsync/cloudsync.py index 5bb2cec..f3e097c 100755 --- a/cloudsync/cloudsync.py +++ b/cloudsync/cloudsync.py @@ -1,6 +1,7 @@ #!/usr/bin/python import os +from pathlib import Path import sys import dropboxsync @@ -59,8 +60,32 @@ def main(): sys.exit(2) try: - # - dbSync = dropboxsync.DropboxSync(vars(args)) + + the_args = vars(args) + db_dir = Path(the_args['dropboxdir']) + local_dir = Path(the_args['localdir']) + + + + for root, dirs, files in os.walk(the_args['localdir']): + local_path = Path(root) + db_path = db_dir / local_path.relative_to(local_dir) + for folder in dirs: + the_args['localdir'] = str(local_path / folder) + the_args['dropboxdir'] = str(db_path / folder) + dbSync = dropboxsync.DropboxSync(**the_args) + dbSync.setLogger(logger) + dbSync.prepare() + + filters = FilterParameters() + filters.days = dbSync.args['match_days'] + dbSync.filterSourceFiles(filters) + + dbSync.synchronize() + the_args['localdir'] = str(local_dir) + the_args['dropboxdir'] = str(db_dir) + + dbSync = dropboxsync.DropboxSync(**the_args) dbSync.setLogger(logger) dbSync.prepare() @@ -69,6 +94,8 @@ def main(): dbSync.filterSourceFiles(filters) dbSync.synchronize() + + except: logger.exception('') diff --git a/cloudsync/dropboxsync.py b/cloudsync/dropboxsync.py index 134d384..9f051f7 100644 --- a/cloudsync/dropboxsync.py +++ b/cloudsync/dropboxsync.py @@ -3,6 +3,7 @@ import logging import contextlib +import datetime import dropbox from dropbox.files import FileMetadata from dropbox.exceptions import ApiError @@ -26,13 +27,13 @@ class DropboxSync(object): filterItems: List[SyncFile] = [] sourceFilesMatched: List[SyncFile] = [] db_handler: Optional[DropboxFileHandler] = None + dbx: Optional[dropbox.Dropbox] = None - def __init__(self, args): - self.args=args - self.dbx = None - self.localDir = Path(args['localdir']) - self.dropboxDir = Path(args['dropboxdir']) - self.directionToDb = args['direction'] == 'todropbox' + def __init__(self, **kwargs): + self.args = kwargs + self.localDir = Path(kwargs['localdir']) + self.dropboxDir = Path(kwargs['dropboxdir']) + self.directionToDb = kwargs['direction'] == 'todropbox' self.timeoutSec = 2 * 60 self.logger = logging.getLogger(__name__) @@ -82,8 +83,8 @@ def checkDropboxDir(self): self.dbx.files_list_folder(str(self.dropboxDir)) self.logger.debug('Dropbox folder exists') except: - self.logger.error(f"Folder {str(self.dropboxDir)} does not exist on Dropbox") - exit(-1) + self.logger.debug(f"Folder {str(self.dropboxDir)} does not exist on Dropbox, creating...") + self.dbx.files_create_folder_v2(str(self.dropboxDir)) def listLocalFiles(self): self.logger.debug('Getting list of local files...') @@ -233,8 +234,8 @@ def uploadFile(self, file_item: SyncFile): with self.stopwatch('uploading'): try: self.dbx.files_upload( - data, db_path, mode, - client_modified=file_item.mod_time, + data, str(db_path), mode, + client_modified=datetime.datetime.utcfromtimestamp(file_item.mod_time), autorename=False, mute=True) except dropbox.exceptions.ApiError as err: From d59f6cbd433d799eac05a7d0baa06ace2857cfed Mon Sep 17 00:00:00 2001 From: Kyle Barry Date: Wed, 2 Mar 2022 09:52:57 +0200 Subject: [PATCH 5/6] check path type and not delete folders --- cloudsync/cloudsync.py | 26 +---------------- cloudsync/dropboxsync.py | 18 +++++++----- cloudsync/sync_file/file_handler/__init__.py | 1 + .../sync_file/file_handler/dropbox_handler.py | 29 +++++++++++++++---- .../sync_file/file_handler/file_handler.py | 10 +++++++ .../sync_file/file_handler/local_handler.py | 9 +++++- cloudsync/sync_file/sync_file.py | 4 +++ 7 files changed, 58 insertions(+), 39 deletions(-) diff --git a/cloudsync/cloudsync.py b/cloudsync/cloudsync.py index f3e097c..f875efc 100755 --- a/cloudsync/cloudsync.py +++ b/cloudsync/cloudsync.py @@ -61,31 +61,7 @@ def main(): try: - the_args = vars(args) - db_dir = Path(the_args['dropboxdir']) - local_dir = Path(the_args['localdir']) - - - - for root, dirs, files in os.walk(the_args['localdir']): - local_path = Path(root) - db_path = db_dir / local_path.relative_to(local_dir) - for folder in dirs: - the_args['localdir'] = str(local_path / folder) - the_args['dropboxdir'] = str(db_path / folder) - dbSync = dropboxsync.DropboxSync(**the_args) - dbSync.setLogger(logger) - dbSync.prepare() - - filters = FilterParameters() - filters.days = dbSync.args['match_days'] - dbSync.filterSourceFiles(filters) - - dbSync.synchronize() - the_args['localdir'] = str(local_dir) - the_args['dropboxdir'] = str(db_dir) - - dbSync = dropboxsync.DropboxSync(**the_args) + dbSync = dropboxsync.DropboxSync(**vars(args)) dbSync.setLogger(logger) dbSync.prepare() diff --git a/cloudsync/dropboxsync.py b/cloudsync/dropboxsync.py index 9f051f7..40cc42f 100644 --- a/cloudsync/dropboxsync.py +++ b/cloudsync/dropboxsync.py @@ -14,7 +14,7 @@ import unicodedata from sync_file import SyncFile -from sync_file.file_handler import DropboxFileHandler, LocalFileHandler +from sync_file.file_handler import DropboxFileHandler, FileType class DropboxSync(object): """ @@ -26,7 +26,6 @@ class DropboxSync(object): dbList: List[SyncFile] = [] filterItems: List[SyncFile] = [] sourceFilesMatched: List[SyncFile] = [] - db_handler: Optional[DropboxFileHandler] = None dbx: Optional[dropbox.Dropbox] = None def __init__(self, **kwargs): @@ -57,7 +56,6 @@ def prepare(self): def prepareDropboxAuth(self): self.logger.debug('Connecting to dropbox using token...') self.dbx = dropbox.Dropbox(self.args['token']) - self.db_handler = DropboxFileHandler(self.dbx) self.logger.debug('Dropbox connected') def checkLocalDir(self): @@ -126,7 +124,9 @@ def synchronize(self): def deleteLocalFiles(self): # remove local sourceNames = [fileItem.name for fileItem in self.sourceFilesMatched] - delList = [fileItem for fileItem in self.locList if fileItem.name not in sourceNames] + delList = [fileItem for fileItem in self.locList + if fileItem.name not in sourceNames + and fileItem.type == FileType.FILE] if not delList: return self.logger.debug(f'Local files to delete: {len(delList)}') @@ -140,7 +140,7 @@ def syncToLocal(self): countFails = 0 for fileItem in self.sourceFilesMatched: - self.db_handler.file = self.dropboxDir / fileItem.name + # self.db_handler.file = self.dropboxDir / fileItem.name if fileItem in self.locList: self.logger.debug(f'Skip existed: {fileItem.name}') @@ -158,7 +158,10 @@ def syncToLocal(self): def deleteDropboxFiles(self): """ Delete not matched files from Dropbox directory """ sourceNames = [fileItem.name for fileItem in self.sourceFilesMatched] - delList = [fileItem for fileItem in self.dbList if fileItem.name not in sourceNames] + delList = [fileItem for fileItem in self.dbList + if fileItem.name not in sourceNames + and fileItem.type == FileType.FILE + ] if not delList: return self.logger.debug('Dropbox files to delete:%s' % len(delList)) @@ -199,7 +202,8 @@ def listDropboxFiles(self): raise Exception('Folder listing failed for %s -- assumed empty:%s' % (str(self.dropboxDir), err)) else: self.logger.debug('Dropbox files:%s' % len(res.entries)) - self.dbList = [SyncFile(self.dropboxDir / dbfile.name, file_handler=self.db_handler) for dbfile in res.entries] + self.dbList = [SyncFile(self.dropboxDir / dbfile.name, file_handler=DropboxFileHandler(self.dbx)) for dbfile in res.entries] + print(self.dbList) def downloadFile(self, file_item: SyncFile): """Download a file. diff --git a/cloudsync/sync_file/file_handler/__init__.py b/cloudsync/sync_file/file_handler/__init__.py index a12bbeb..932d5be 100644 --- a/cloudsync/sync_file/file_handler/__init__.py +++ b/cloudsync/sync_file/file_handler/__init__.py @@ -1,2 +1,3 @@ from .local_handler import LocalFileHandler from .dropbox_handler import DropboxFileHandler +from .file_handler import FileType diff --git a/cloudsync/sync_file/file_handler/dropbox_handler.py b/cloudsync/sync_file/file_handler/dropbox_handler.py index 2fcb332..eeb1f1b 100644 --- a/cloudsync/sync_file/file_handler/dropbox_handler.py +++ b/cloudsync/sync_file/file_handler/dropbox_handler.py @@ -1,16 +1,24 @@ import time +from typing import Union from dropbox import Dropbox -from dropbox.files import FileMetadata +from dropbox.files import Metadata, FileMetadata, FolderMetadata -from .file_handler import FileHandler +from .file_handler import FileHandler, FileType class DropboxFileHandler(FileHandler): + __metadata: Union[Metadata, FileMetadata, FolderMetadata, None] = None + def __init__(self, db_obj: Dropbox): self.dbx = db_obj + def __get_metadata(self): + if self.__metadata is None: + self.__metadata = self.dbx.files_get_metadata(str(self.file)) + return self.__metadata + @property def mod_time(self): metadata = self.dbx.files_get_metadata(str(self.file)) @@ -35,8 +43,17 @@ def hash(self): return metadata.content_hash - def size(self): - metadata = self.dbx.files_get_metadata(str(self.file)) - assert isinstance(metadata, FileMetadata) + def size(self) -> int: + return self.__get_metadata().size + + def type(self) -> FileType: + metadata = self.__get_metadata() + + if isinstance(metadata, FileMetadata): + return FileType.FILE + + elif isinstance(metadata, FolderMetadata): + return FileType.FOLDER - return metadata.size + else: + raise Exception(f"unknown type: {type(metadata)}") diff --git a/cloudsync/sync_file/file_handler/file_handler.py b/cloudsync/sync_file/file_handler/file_handler.py index f18cdda..e413f3f 100644 --- a/cloudsync/sync_file/file_handler/file_handler.py +++ b/cloudsync/sync_file/file_handler/file_handler.py @@ -1,9 +1,15 @@ from abc import ABC, abstractmethod +from enum import Enum, auto from pathlib import Path from ..filters import FilterParameters +class FileType(Enum): + FOLDER = auto() + FILE = auto() + + class FileHandler(ABC): """ Generic class that facilitates reading file metadata, @@ -43,5 +49,9 @@ def create(self): def delete(self): pass + @abstractmethod + def type(self): + pass + def filter(self, params: FilterParameters) -> bool: return params.filter_days(self.mod_time) or params.filter_name(str(self.file)) diff --git a/cloudsync/sync_file/file_handler/local_handler.py b/cloudsync/sync_file/file_handler/local_handler.py index 07a7d01..1feb491 100644 --- a/cloudsync/sync_file/file_handler/local_handler.py +++ b/cloudsync/sync_file/file_handler/local_handler.py @@ -1,7 +1,7 @@ import hashlib import os -from .file_handler import FileHandler +from .file_handler import FileHandler, FileType DROPBOX_HASH_CHUNK_SIZE = 4*1024*1024 @@ -31,4 +31,11 @@ def hash(self): def size(self): return self.file.stat().st_size + def type(self): + if self.file.is_file(): + return FileType.FILE + else: + return FileType.FOLDER + + diff --git a/cloudsync/sync_file/sync_file.py b/cloudsync/sync_file/sync_file.py index 3e611d8..92723b3 100644 --- a/cloudsync/sync_file/sync_file.py +++ b/cloudsync/sync_file/sync_file.py @@ -32,6 +32,10 @@ def hash(self): def size(self): return self.file_handler.size() + @property + def type(self): + return self.file_handler.type() + def filter(self, params: FilterParameters) -> bool: return self.file_handler.filter(params) From a3108ce7bec969774762589926c229346cd577b2 Mon Sep 17 00:00:00 2001 From: Kyle Barry Date: Wed, 2 Mar 2022 11:58:52 +0200 Subject: [PATCH 6/6] add recursive folder syncing --- cloudsync/cloudsync.py | 2 +- cloudsync/dropboxsync.py | 121 ++++++++++++------ .../sync_file/file_handler/dropbox_handler.py | 4 +- cloudsync/sync_file/sync_file.py | 4 +- 4 files changed, 88 insertions(+), 43 deletions(-) diff --git a/cloudsync/cloudsync.py b/cloudsync/cloudsync.py index f875efc..15dd913 100755 --- a/cloudsync/cloudsync.py +++ b/cloudsync/cloudsync.py @@ -67,7 +67,7 @@ def main(): filters = FilterParameters() filters.days = dbSync.args['match_days'] - dbSync.filterSourceFiles(filters) + dbSync.apply_filter(filters) dbSync.synchronize() diff --git a/cloudsync/dropboxsync.py b/cloudsync/dropboxsync.py index 40cc42f..e8de14b 100644 --- a/cloudsync/dropboxsync.py +++ b/cloudsync/dropboxsync.py @@ -3,6 +3,7 @@ import logging import contextlib +from copy import copy import datetime import dropbox from dropbox.files import FileMetadata @@ -14,6 +15,7 @@ import unicodedata from sync_file import SyncFile +from sync_file.filters import FilterParameters from sync_file.file_handler import DropboxFileHandler, FileType class DropboxSync(object): @@ -22,11 +24,14 @@ class DropboxSync(object): use Dropbox API v2 (https://github.com/dropbox/dropbox-sdk-python) """ - locList: List[SyncFile] = [] - dbList: List[SyncFile] = [] + local_files: List[SyncFile] = [] + db_files: List[SyncFile] = [] + local_folders: List[SyncFile] = [] + db_folders: List[SyncFile] = [] filterItems: List[SyncFile] = [] sourceFilesMatched: List[SyncFile] = [] dbx: Optional[dropbox.Dropbox] = None + filters: Optional[FilterParameters] = None def __init__(self, **kwargs): self.args = kwargs @@ -46,12 +51,7 @@ def prepare(self): self.logger.info('--- Mode: %s' % self.args['direction']) self.prepareDropboxAuth() self.checkDropboxAuth() - self.checkDropboxDir() - self.checkLocalDir() - self.listDropboxFiles() - self.listLocalFiles() - self.listFilterItems() def prepareDropboxAuth(self): self.logger.debug('Connecting to dropbox using token...') @@ -60,7 +60,10 @@ def prepareDropboxAuth(self): def checkLocalDir(self): if not os.path.exists(self.localDir): - raise Exception('Local path is not exists:%s' % self.localDir) + if self.directionToDb: + raise Exception('Local path is not exists:%s' % self.localDir) + else: + os.mkdir(self.localDir) if not os.path.isdir(self.localDir): raise Exception('Local path is not directory:%s' % self.localDir) @@ -86,32 +89,75 @@ def checkDropboxDir(self): def listLocalFiles(self): self.logger.debug('Getting list of local files...') - self.locList = [ - SyncFile(self.localDir / Path(unicodedata.normalize('NFC', f))) - for f in os.listdir(self.localDir) - if os.path.isfile(self.localDir / f) - ] - self.logger.debug(f'Local files: {len(self.locList)}') + + self.local_files = [] + self.local_folders = [] + + for f in os.listdir(self.localDir): + entry = SyncFile(self.localDir / Path(unicodedata.normalize('NFC', f))) + if entry.type == FileType.FILE: + self.local_files.append(entry) + else: + self.local_folders.append(entry) + + self.logger.debug(f'Local files: {len(self.local_files)}') return True # filtration def listFilterItems(self): if self.directionToDb: - self.filterItems = self.locList + self.filterItems = self.local_files else: - self.filterItems = self.dbList + self.filterItems = self.db_files - def filterSourceFiles(self, filters): + def __filter_files(self, filters): source_count = len(self.filterItems) self.logger.debug(f'Source files: {source_count}') self.sourceFilesMatched = [f for f in self.filterItems if not f.filter(filters)] self.logger.info(f'--- Filter source files: {source_count} -> {len(self.sourceFilesMatched)}') + def apply_filter(self, filters: FilterParameters): + self.filters = filters + # synchronize def synchronize(self): - # for debug - #self.fixLocalTimestamps() + old_db_root = self.dropboxDir + old_local_root = self.localDir + + if self.directionToDb: + stack = [self.localDir] + relative_to = old_local_root + else: + stack = [self.dropboxDir] + relative_to = old_db_root + + while len(stack) > 0: + + entry = stack.pop() + entry = Path(entry.relative_to(relative_to)) + + self.dropboxDir = self.dropboxDir / entry + self.localDir = self.localDir / entry + + self.checkLocalDir() + self.checkDropboxDir() + self.listDropboxFiles() + self.listLocalFiles() + self.listFilterItems() + self.__filter_files(self.filters) + + if self.directionToDb: + stack.extend([Path(self.localDir/f.name) for f in self.local_folders]) + else: + stack.extend([Path(self.dropboxDir/f.name) for f in self.db_folders]) + + self.__do_sync() + + self.dropboxDir = old_db_root + self.localDir = old_local_root + + def __do_sync(self): if self.directionToDb: self.deleteDropboxFiles() self.syncToDropbox() @@ -119,20 +165,16 @@ def synchronize(self): self.deleteLocalFiles() self.syncToLocal() - return True - def deleteLocalFiles(self): # remove local sourceNames = [fileItem.name for fileItem in self.sourceFilesMatched] - delList = [fileItem for fileItem in self.locList - if fileItem.name not in sourceNames - and fileItem.type == FileType.FILE] + delList = [fileItem for fileItem in self.local_files if fileItem.name not in sourceNames] if not delList: return self.logger.debug(f'Local files to delete: {len(delList)}') for fileItem in delList: fileItem.delete() - self.logger.info(f'--- Deleted {len(delList)}/{len(self.locList)} local files') + self.logger.info(f'--- Deleted {len(delList)}/{len(self.local_files)} local files') def syncToLocal(self): countSuccess = 0 @@ -142,7 +184,7 @@ def syncToLocal(self): # self.db_handler.file = self.dropboxDir / fileItem.name - if fileItem in self.locList: + if fileItem in self.local_files: self.logger.debug(f'Skip existed: {fileItem.name}') countSkip += 1 continue @@ -158,24 +200,21 @@ def syncToLocal(self): def deleteDropboxFiles(self): """ Delete not matched files from Dropbox directory """ sourceNames = [fileItem.name for fileItem in self.sourceFilesMatched] - delList = [fileItem for fileItem in self.dbList - if fileItem.name not in sourceNames - and fileItem.type == FileType.FILE - ] + delList = [fileItem for fileItem in self.db_files if fileItem.name not in sourceNames] if not delList: return self.logger.debug('Dropbox files to delete:%s' % len(delList)) for fileItem in delList: self.deleteFile(fileItem) - self.logger.info('--- Success delete %d/%d dropbox files' % (len(delList), len(self.dbList))) + self.logger.info('--- Success delete %d/%d dropbox files' % (len(delList), len(self.db_files))) def syncToDropbox(self): countSuccess = 0 countSkip = 0 countFails = 0 for fileItem in self.sourceFilesMatched: - if fileItem in self.dbList: - self.logger.debug('Skip existed:%s' % fileItem) + if fileItem in self.db_files: + self.logger.debug(f'Skip existed: {str(fileItem.name)}') countSkip += 1 continue if self.uploadFile(fileItem): @@ -193,17 +232,23 @@ def listDropboxFiles(self): Return an array of filter items """ self.logger.debug('Downloading dropbox list files...') + self.db_files = [] + self.db_folders = [] try: with self.stopwatch(__name__): res = self.dbx.files_list_folder(str(self.dropboxDir)) except ApiError as err: - self.dbList = [] + self.db_files = [] raise Exception('Folder listing failed for %s -- assumed empty:%s' % (str(self.dropboxDir), err)) else: - self.logger.debug('Dropbox files:%s' % len(res.entries)) - self.dbList = [SyncFile(self.dropboxDir / dbfile.name, file_handler=DropboxFileHandler(self.dbx)) for dbfile in res.entries] - print(self.dbList) + for f in res.entries: + entry = SyncFile(self.dropboxDir / f.name, file_handler=DropboxFileHandler(self.dbx)) + if entry.type == FileType.FILE: + self.db_files.append(entry) + else: + self.db_folders.append(entry) + self.logger.debug(f'Dropbox files: {len(self.db_files)}') def downloadFile(self, file_item: SyncFile): """Download a file. @@ -212,7 +257,7 @@ def downloadFile(self, file_item: SyncFile): db_path = self.dropboxDir / file_item.name local_path = self.localDir / file_item.name - file_size = SyncFile(db_path, file_handler=self.db_handler).size + file_size = SyncFile(db_path, file_handler=DropboxFileHandler(self.dbx)).size self.logger.debug(f'Downloading {file_item.name} ({file_size} bytes) ...') with self.stopwatch('downloading'): diff --git a/cloudsync/sync_file/file_handler/dropbox_handler.py b/cloudsync/sync_file/file_handler/dropbox_handler.py index eeb1f1b..0a702a6 100644 --- a/cloudsync/sync_file/file_handler/dropbox_handler.py +++ b/cloudsync/sync_file/file_handler/dropbox_handler.py @@ -28,8 +28,8 @@ def mod_time(self): server_modify = time.mktime(metadata.server_modified.timetuple()) if client_modify > server_modify: - return client_modify - return server_modify + return int(client_modify) + return int(server_modify) def create(self): pass diff --git a/cloudsync/sync_file/sync_file.py b/cloudsync/sync_file/sync_file.py index 92723b3..5e00dcc 100644 --- a/cloudsync/sync_file/sync_file.py +++ b/cloudsync/sync_file/sync_file.py @@ -46,7 +46,7 @@ def __repr__(self): return str(self) def __str__(self): - return str(self.name) + return f"{str(self.name)}\n{str(self.mod_time)}\n{str(self.hash)}" def __eq__(self, other: 'SyncFile'): if other.name != self.name: @@ -55,7 +55,7 @@ def __eq__(self, other: 'SyncFile'): if other.size != self.size: return False - if self.mod_time < other.mod_time: + if self.mod_time != other.mod_time: return self.hash == other.hash return False