diff --git a/cloudsync/__init__.py b/cloudsync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudsync.py b/cloudsync/cloudsync.py similarity index 86% rename from cloudsync.py rename to cloudsync/cloudsync.py index 2a74c43..15dd913 100755 --- a/cloudsync.py +++ b/cloudsync/cloudsync.py @@ -1,13 +1,15 @@ #!/usr/bin/python import os -import sys, locale +from pathlib import Path +import sys import dropboxsync import logging import logger as lgr import argparse -import filters as ftr + +from sync_file.filters import FilterParameters def isCronMode(): @@ -58,17 +60,18 @@ def main(): sys.exit(2) try: - # - dbSync = dropboxsync.DropboxSync(vars(args)) + + dbSync = dropboxsync.DropboxSync(**vars(args)) dbSync.setLogger(logger) dbSync.prepare() - filters = [] - filters.append(ftr.FileFilterDays(matchDays=dbSync.args['match_days'])) - filters.append(ftr.FileFilterMask()) - dbSync.filterSourceFiles(filters) + filters = FilterParameters() + filters.days = dbSync.args['match_days'] + dbSync.apply_filter(filters) dbSync.synchronize() + + except: logger.exception('') diff --git a/cloudsync/dropboxsync.py b/cloudsync/dropboxsync.py new file mode 100644 index 0000000..e8de14b --- /dev/null +++ b/cloudsync/dropboxsync.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python + +import logging +import contextlib + +from copy import copy +import datetime +import dropbox +from dropbox.files import FileMetadata +from dropbox.exceptions import ApiError +import os +from pathlib import Path +import time +from typing import List, Optional +import unicodedata + +from sync_file import SyncFile +from sync_file.filters import FilterParameters +from sync_file.file_handler import DropboxFileHandler, FileType + +class DropboxSync(object): + """ + Class to help synchronize files to/from dropbox + use Dropbox API v2 (https://github.com/dropbox/dropbox-sdk-python) + """ + + local_files: List[SyncFile] = [] + db_files: List[SyncFile] = [] + local_folders: List[SyncFile] = [] + db_folders: List[SyncFile] = [] + filterItems: List[SyncFile] = [] + sourceFilesMatched: List[SyncFile] = [] + dbx: Optional[dropbox.Dropbox] = None + filters: Optional[FilterParameters] = None + + def __init__(self, **kwargs): + self.args = kwargs + self.localDir = Path(kwargs['localdir']) + self.dropboxDir = Path(kwargs['dropboxdir']) + self.directionToDb = kwargs['direction'] == 'todropbox' + + self.timeoutSec = 2 * 60 + self.logger = logging.getLogger(__name__) + self.logger.addHandler(logging.NullHandler()) + + def setLogger(self, logger): + self.logger = logger + + # prepare + def prepare(self): + self.logger.info('--- Mode: %s' % self.args['direction']) + self.prepareDropboxAuth() + self.checkDropboxAuth() + + + def prepareDropboxAuth(self): + self.logger.debug('Connecting to dropbox using token...') + self.dbx = dropbox.Dropbox(self.args['token']) + self.logger.debug('Dropbox connected') + + def checkLocalDir(self): + if not os.path.exists(self.localDir): + if self.directionToDb: + raise Exception('Local path is not exists:%s' % self.localDir) + else: + os.mkdir(self.localDir) + if not os.path.isdir(self.localDir): + raise Exception('Local path is not directory:%s' % self.localDir) + + def checkDropboxAuth(self): + """ + Checks Dropbox uploader is initialized to Dropbox account + """ + self.logger.debug('Getting info about dropbox account...') + acc = self.dbx.users_get_current_account() + self.logger.debug('Dropbox account: [%s_%s] mail:%s' % (acc.country, acc.locale, acc.email)) + + def checkDropboxDir(self): + """ + Checks that Dropbox folder exists. + """ + self.logger.debug('Checking if Dropbox folder exists...') + try: + self.dbx.files_list_folder(str(self.dropboxDir)) + self.logger.debug('Dropbox folder exists') + except: + self.logger.debug(f"Folder {str(self.dropboxDir)} does not exist on Dropbox, creating...") + self.dbx.files_create_folder_v2(str(self.dropboxDir)) + + def listLocalFiles(self): + self.logger.debug('Getting list of local files...') + + self.local_files = [] + self.local_folders = [] + + for f in os.listdir(self.localDir): + entry = SyncFile(self.localDir / Path(unicodedata.normalize('NFC', f))) + if entry.type == FileType.FILE: + self.local_files.append(entry) + else: + self.local_folders.append(entry) + + self.logger.debug(f'Local files: {len(self.local_files)}') + return True + + # filtration + def listFilterItems(self): + if self.directionToDb: + self.filterItems = self.local_files + else: + self.filterItems = self.db_files + + def __filter_files(self, filters): + source_count = len(self.filterItems) + self.logger.debug(f'Source files: {source_count}') + self.sourceFilesMatched = [f for f in self.filterItems if not f.filter(filters)] + self.logger.info(f'--- Filter source files: {source_count} -> {len(self.sourceFilesMatched)}') + + def apply_filter(self, filters: FilterParameters): + self.filters = filters + + # synchronize + def synchronize(self): + + old_db_root = self.dropboxDir + old_local_root = self.localDir + + if self.directionToDb: + stack = [self.localDir] + relative_to = old_local_root + else: + stack = [self.dropboxDir] + relative_to = old_db_root + + while len(stack) > 0: + + entry = stack.pop() + entry = Path(entry.relative_to(relative_to)) + + self.dropboxDir = self.dropboxDir / entry + self.localDir = self.localDir / entry + + self.checkLocalDir() + self.checkDropboxDir() + self.listDropboxFiles() + self.listLocalFiles() + self.listFilterItems() + self.__filter_files(self.filters) + + if self.directionToDb: + stack.extend([Path(self.localDir/f.name) for f in self.local_folders]) + else: + stack.extend([Path(self.dropboxDir/f.name) for f in self.db_folders]) + + self.__do_sync() + + self.dropboxDir = old_db_root + self.localDir = old_local_root + + def __do_sync(self): + if self.directionToDb: + self.deleteDropboxFiles() + self.syncToDropbox() + else: + self.deleteLocalFiles() + self.syncToLocal() + + def deleteLocalFiles(self): + # remove local + sourceNames = [fileItem.name for fileItem in self.sourceFilesMatched] + delList = [fileItem for fileItem in self.local_files if fileItem.name not in sourceNames] + if not delList: + return + self.logger.debug(f'Local files to delete: {len(delList)}') + for fileItem in delList: + fileItem.delete() + self.logger.info(f'--- Deleted {len(delList)}/{len(self.local_files)} local files') + + def syncToLocal(self): + countSuccess = 0 + countSkip = 0 + countFails = 0 + for fileItem in self.sourceFilesMatched: + + # self.db_handler.file = self.dropboxDir / fileItem.name + + if fileItem in self.local_files: + self.logger.debug(f'Skip existed: {fileItem.name}') + countSkip += 1 + continue + if self.downloadFile(fileItem): + countSuccess += 1 + else: + countFails += 1 + # print stat + strSkip = ' Skip:%d' % countSkip if countSkip else '' + strFails = ' Fails:%d' % countFails if countFails else '' + self.logger.info('--- Download %d/%d%s%s' % (countSuccess, len(self.sourceFilesMatched), strSkip, strFails)) + + def deleteDropboxFiles(self): + """ Delete not matched files from Dropbox directory """ + sourceNames = [fileItem.name for fileItem in self.sourceFilesMatched] + delList = [fileItem for fileItem in self.db_files if fileItem.name not in sourceNames] + if not delList: + return + self.logger.debug('Dropbox files to delete:%s' % len(delList)) + for fileItem in delList: + self.deleteFile(fileItem) + self.logger.info('--- Success delete %d/%d dropbox files' % (len(delList), len(self.db_files))) + + def syncToDropbox(self): + countSuccess = 0 + countSkip = 0 + countFails = 0 + for fileItem in self.sourceFilesMatched: + if fileItem in self.db_files: + self.logger.debug(f'Skip existed: {str(fileItem.name)}') + countSkip += 1 + continue + if self.uploadFile(fileItem): + countSuccess += 1 + else: + countFails += 1 + # print stat + strSkip = ' Skip:%d' % countSkip if countSkip else '' + strFails = ' Fails:%d' % countFails if countFails else '' + self.logger.info('--- Success upload %d/%d%s%s' % (countSuccess, len(self.sourceFilesMatched), strSkip, strFails)) + + # dropbox helpers + def listDropboxFiles(self): + """List a folder. + Return an array of filter items + """ + self.logger.debug('Downloading dropbox list files...') + self.db_files = [] + self.db_folders = [] + + try: + with self.stopwatch(__name__): + res = self.dbx.files_list_folder(str(self.dropboxDir)) + except ApiError as err: + self.db_files = [] + raise Exception('Folder listing failed for %s -- assumed empty:%s' % (str(self.dropboxDir), err)) + else: + for f in res.entries: + entry = SyncFile(self.dropboxDir / f.name, file_handler=DropboxFileHandler(self.dbx)) + if entry.type == FileType.FILE: + self.db_files.append(entry) + else: + self.db_folders.append(entry) + self.logger.debug(f'Dropbox files: {len(self.db_files)}') + + def downloadFile(self, file_item: SyncFile): + """Download a file. + Return True when success, or False if error occurs. + """ + + db_path = self.dropboxDir / file_item.name + local_path = self.localDir / file_item.name + file_size = SyncFile(db_path, file_handler=DropboxFileHandler(self.dbx)).size + + self.logger.debug(f'Downloading {file_item.name} ({file_size} bytes) ...') + with self.stopwatch('downloading'): + try: + self.dbx.files_download_to_file(str(local_path), str(db_path)) + except ApiError as err: + raise Exception(f'{file_item.name} - API error: {err}') + self.logger.debug(f'Success download - {file_item.name} ({file_size} bytes)') + return True + + def uploadFile(self, file_item: SyncFile): + """Upload a file. + Return the request response, or None in case of error. + """ + db_path = self.dropboxDir / file_item.name + local_path = self.localDir / file_item.name + file_size = file_item.size + mode = dropbox.files.WriteMode.overwrite + + with open(local_path, 'rb') as f: + data = f.read() + self.logger.debug(f'Uploading {file_item.name} ({file_size} bytes) ...') + with self.stopwatch('uploading'): + try: + self.dbx.files_upload( + data, str(db_path), mode, + client_modified=datetime.datetime.utcfromtimestamp(file_item.mod_time), + autorename=False, + mute=True) + except dropbox.exceptions.ApiError as err: + raise Exception(f'{file_item.name}- API error: {err}') + self.logger.debug(f'Success upload - {file_item.name} ({file_size} bytes)') + return True + + def deleteFile(self, file_item: SyncFile): + self.logger.debug(f'Deleting - \'{file_item.name}\'') + with self.stopwatch('deleting'): + try: + file_item.delete() + except ApiError as err: + raise Exception(f'{file_item.name} - API error: {err}') + self.logger.debug(f'Success delete - {file_item.name}') + + @contextlib.contextmanager + def stopwatch(self, message): + """Context manager to print how long a block of code took.""" + t0 = time.time() + try: + yield + finally: + t1 = time.time() + self.logger.debug('Total elapsed time for %s: %.3f' % (message, t1 - t0)) + + diff --git a/logger.py b/cloudsync/logger.py similarity index 100% rename from logger.py rename to cloudsync/logger.py diff --git a/cloudsync/sync_file/__init__.py b/cloudsync/sync_file/__init__.py new file mode 100644 index 0000000..36495c7 --- /dev/null +++ b/cloudsync/sync_file/__init__.py @@ -0,0 +1 @@ +from .sync_file import SyncFile \ No newline at end of file diff --git a/cloudsync/sync_file/file_handler/__init__.py b/cloudsync/sync_file/file_handler/__init__.py new file mode 100644 index 0000000..932d5be --- /dev/null +++ b/cloudsync/sync_file/file_handler/__init__.py @@ -0,0 +1,3 @@ +from .local_handler import LocalFileHandler +from .dropbox_handler import DropboxFileHandler +from .file_handler import FileType diff --git a/cloudsync/sync_file/file_handler/dropbox_handler.py b/cloudsync/sync_file/file_handler/dropbox_handler.py new file mode 100644 index 0000000..0a702a6 --- /dev/null +++ b/cloudsync/sync_file/file_handler/dropbox_handler.py @@ -0,0 +1,59 @@ +import time +from typing import Union + +from dropbox import Dropbox +from dropbox.files import Metadata, FileMetadata, FolderMetadata + +from .file_handler import FileHandler, FileType + + +class DropboxFileHandler(FileHandler): + + __metadata: Union[Metadata, FileMetadata, FolderMetadata, None] = None + + def __init__(self, db_obj: Dropbox): + self.dbx = db_obj + + def __get_metadata(self): + if self.__metadata is None: + self.__metadata = self.dbx.files_get_metadata(str(self.file)) + return self.__metadata + + @property + def mod_time(self): + metadata = self.dbx.files_get_metadata(str(self.file)) + assert isinstance(metadata, FileMetadata) + + client_modify = time.mktime(metadata.client_modified.timetuple()) + server_modify = time.mktime(metadata.server_modified.timetuple()) + + if client_modify > server_modify: + return int(client_modify) + return int(server_modify) + + def create(self): + pass + + def delete(self): + self.dbx.files_delete_v2(str(self.file)) + + def hash(self): + metadata = self.dbx.files_get_metadata(str(self.file)) + assert isinstance(metadata, FileMetadata) + + return metadata.content_hash + + def size(self) -> int: + return self.__get_metadata().size + + def type(self) -> FileType: + metadata = self.__get_metadata() + + if isinstance(metadata, FileMetadata): + return FileType.FILE + + elif isinstance(metadata, FolderMetadata): + return FileType.FOLDER + + else: + raise Exception(f"unknown type: {type(metadata)}") diff --git a/cloudsync/sync_file/file_handler/file_handler.py b/cloudsync/sync_file/file_handler/file_handler.py new file mode 100644 index 0000000..e413f3f --- /dev/null +++ b/cloudsync/sync_file/file_handler/file_handler.py @@ -0,0 +1,57 @@ +from abc import ABC, abstractmethod +from enum import Enum, auto +from pathlib import Path + +from ..filters import FilterParameters + + +class FileType(Enum): + FOLDER = auto() + FILE = auto() + + +class FileHandler(ABC): + """ + Generic class that facilitates reading file metadata, + creating, deleting and modifying files. + """ + + _file: Path = "" + + @property + def file(self): + return self._file + + @file.setter + def file(self, new_file: Path): + self._file = new_file + + @property + @abstractmethod + def mod_time(self): + pass + + @property + @abstractmethod + def hash(self): + pass + + @property + @abstractmethod + def size(self): + pass + + @abstractmethod + def create(self): + pass + + @abstractmethod + def delete(self): + pass + + @abstractmethod + def type(self): + pass + + def filter(self, params: FilterParameters) -> bool: + return params.filter_days(self.mod_time) or params.filter_name(str(self.file)) diff --git a/cloudsync/sync_file/file_handler/local_handler.py b/cloudsync/sync_file/file_handler/local_handler.py new file mode 100644 index 0000000..1feb491 --- /dev/null +++ b/cloudsync/sync_file/file_handler/local_handler.py @@ -0,0 +1,41 @@ +import hashlib +import os + +from .file_handler import FileHandler, FileType + +DROPBOX_HASH_CHUNK_SIZE = 4*1024*1024 + + +class LocalFileHandler(FileHandler): + + @property + def mod_time(self): + return int(os.path.getmtime(self.file)) + + def create(self): + self.file.open("a+") + + def delete(self): + self.file.unlink() + + def hash(self): + with open(self.file, 'rb') as f: + block_hashes = b'' + while True: + chunk = f.read(DROPBOX_HASH_CHUNK_SIZE) + if not chunk: + break + block_hashes += hashlib.sha256(chunk).digest() + return hashlib.sha256(block_hashes).hexdigest() + + def size(self): + return self.file.stat().st_size + + def type(self): + if self.file.is_file(): + return FileType.FILE + else: + return FileType.FOLDER + + + diff --git a/cloudsync/sync_file/filters/__init__.py b/cloudsync/sync_file/filters/__init__.py new file mode 100644 index 0000000..077e867 --- /dev/null +++ b/cloudsync/sync_file/filters/__init__.py @@ -0,0 +1 @@ +from .filter_params import FilterParameters diff --git a/cloudsync/sync_file/filters/filter_params.py b/cloudsync/sync_file/filters/filter_params.py new file mode 100644 index 0000000..fbe49a4 --- /dev/null +++ b/cloudsync/sync_file/filters/filter_params.py @@ -0,0 +1,47 @@ +from datetime import datetime, timedelta +import re +import time + + +class FilterParameters: + + def __init__(self): + self._days = None + self._size = None + self._name_regexes = [ + re.compile('^\\..+|/\\..+'), # match hidden files + re.compile('^~.*|/~.*'), # match temporary files + ] + + @property + def days(self): + return self._days + + @days.setter + def days(self, val: int): + self._days = val + + @property + def size(self): + return self._size + + @size.setter + def size(self, val: int): + self._size = val + + def filter_name(self, file_name: str) -> bool: + for reg in self._name_regexes: + if reg.match(file_name): + return True + return False + + def filter_days(self, file_mod: int) -> bool: + if self._days is None: + return False + + threshold_day = time.mktime((datetime.today() - timedelta(days=self.days)).timetuple()) + + return threshold_day < file_mod + + def filter_size(self, file_size: int) -> bool: + return self.size > file_size diff --git a/cloudsync/sync_file/sync_file.py b/cloudsync/sync_file/sync_file.py new file mode 100644 index 0000000..5e00dcc --- /dev/null +++ b/cloudsync/sync_file/sync_file.py @@ -0,0 +1,64 @@ +from pathlib import Path + +from .file_handler import LocalFileHandler +from .filters import FilterParameters + + +class SyncFile: + """ + File object for easy comparison between local and Dropbox files. + Allows operations (such as equality checking) to be done independently + of the file source. + """ + + def __init__(self, raw_file: Path, file_handler=None): + self.file_handler = file_handler if file_handler is not None else LocalFileHandler() + self.file_handler.file = raw_file + self._name = raw_file + + @property + def mod_time(self): + return self.file_handler.mod_time + + @property + def name(self): + return self._name.name + + @property + def hash(self): + return self.file_handler.hash() + + @property + def size(self): + return self.file_handler.size() + + @property + def type(self): + return self.file_handler.type() + + def filter(self, params: FilterParameters) -> bool: + return self.file_handler.filter(params) + + def delete(self): + self.file_handler.delete() + + def __repr__(self): + return str(self) + + def __str__(self): + return f"{str(self.name)}\n{str(self.mod_time)}\n{str(self.hash)}" + + def __eq__(self, other: 'SyncFile'): + if other.name != self.name: + return False + + if other.size != self.size: + return False + + if self.mod_time != other.mod_time: + return self.hash == other.hash + + return False + + def __ne__(self, other): + return not self == other diff --git a/dropboxsync.py b/dropboxsync.py deleted file mode 100644 index d1ff086..0000000 --- a/dropboxsync.py +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env python - -import logging -import contextlib - -import datetime -import dropbox -from dropbox.files import FileMetadata, FolderMetadata -import os -import time -import filters -import unicodedata - -class DropboxSync(object): - """ - Class to help synchronize files to/from dropbox - use Dropbox API v2 (https://github.com/dropbox/dropbox-sdk-python) - """ - def __init__(self, args): - self.args=args - self.dbx = None - self.localDir = self.normalizeDir(args['localdir']) - self.dropboxDir = self.normalizeDir(args['dropboxdir']) - self.directionToDb = args['direction'] == 'todropbox' - - self.timeoutSec = 2 * 60 - self.logger = logging.getLogger(__name__) - self.logger.addHandler(logging.NullHandler()) - self.dbList = [] - self.locList = [] - self.filterItems = [] - self.sourceFilesMatched = [] - - def setLogger(self, logger): - self.logger = logger - - # prepare - def prepare(self): - self.logger.info('--- Mode: %s' % self.args['direction']) - self.prepareDropboxAuth() - self.checkDropboxAuth() - self.checkDropboxDir() - self.checkLocalDir() - - self.listDropboxFiles() - self.listLocalFiles() - self.listFilterItems() - - def prepareDropboxAuth(self): - self.logger.debug('Connecting to dropbox using token...') - self.dbx = dropbox.Dropbox(self.args['token']) - self.logger.debug('Dropbox connected') - - def checkLocalDir(self): - if not os.path.exists(self.localDir): - raise Exception('Local path is not exists:%s' % self.localDir) - if not os.path.isdir(self.localDir): - raise Exception('Local path is not directory:%s' % self.localDir) - - def checkDropboxAuth(self): - """ - Checks Dropbox uploader is initialized to Dropbox account - """ - self.logger.debug('Getting info about dropbox account...') - acc = self.dbx.users_get_current_account() - self.logger.debug('Dropbox account: [%s_%s] mail:%s' % (acc.country, acc.locale, acc.email)) - - def checkDropboxDir(self): - """ - Checks that Dropbox folder exists. - """ - self.logger.debug('Checking if Dropbox folder exists...') - try: - self.dbx.files_list_folder(self.dropboxDir) - self.logger.debug('Dropbox folder exists') - except: - self.logger.error(f"Folder {self.dropboxDir} does not exist on Dropbox") - exit(-1) - - - def listLocalFiles(self): - self.logger.debug('Getting list of local files...') - locList = [unicodedata.normalize('NFC', f) for f in os.listdir(self.localDir) if os.path.isfile(os.path.join(self.localDir,f))] - self.locList = [self.filterItemByLocal(f) for f in locList] - self.logger.debug('Local files:%s' % len(self.locList)) - return True - - def mtime(self, filePath): - mtime = os.path.getmtime(filePath) - return datetime.datetime(*time.gmtime(mtime)[:6]) - # t = os.path.getmtime(filePath) - # return datetime.datetime.fromtimestamp(t) - - def filterItemByLocal(self, fileName): - filePath = os.path.join(self.localDir, fileName) - return filters.FileFilterItem( - name=fileName, - mtime=self.mtime(filePath), - size=os.path.getsize(filePath)) - - def filterItemByDropbox(self, fileMd): - the_dict = { - 'name': fileMd.name - } - - #Check if file has these attributes before filtering by them - if hasattr(fileMd, 'cliend_modified'): - the_dict['client_modified'] = fileMd.client_modified - if hasattr(fileMd, 'size'): - the_dict['size'] = fileMd.size - return filters.FileFilterItem( - **the_dict - ) - - def normalizeDir(self, directory): - result = directory.replace(os.path.sep, '/') - result = os.path.expanduser(result) - while '//' in result: - result = result.replace('//', '/') - result = result.rstrip('/') - result = unicodedata.normalize('NFC', result) - return result - - # filtration - def listFilterItems(self): - resItems = [] - if self.directionToDb: - self.filterItems = self.locList - else: - self.filterItems = self.dbList - - def filterSourceFiles(self, filters): - resFiles = self.filterItems - sourceCount = len(resFiles) - self.logger.debug('Source files:%s' % (len(resFiles))) - for fltr in filters: - prevCount = len(resFiles) - resFiles = fltr.filterFiles(resFiles) - resCount = len(resFiles) - if resCount != prevCount: - self.logger.debug('Filter \'%s\': %s -> %s' % (fltr.__class__.__name__, prevCount, resCount)) - self.sourceFilesMatched = resFiles - self.logger.info('--- Filter source files: %d -> %d' % (sourceCount, len(resFiles))) - - # synchronize - def synchronize(self): - # for debug - #self.fixLocalTimestamps() - - if self.directionToDb: - self.deleteDropboxFiles() - self.syncToDropbox() - else: - self.deleteLocalFiles() - self.syncToLocal() - - return True - - def deleteLocalFiles(self): - # remove local - sourceNames = [fileItem.fileName for fileItem in self.sourceFilesMatched] - delList = [fileItem for fileItem in self.locList if fileItem.fileName not in sourceNames] - if not delList: - return - self.logger.debug('Local files to delete:%s' % len(delList)) - for fileItem in delList: - os.remove(os.path.join(self.localDir, fileItem.fileName)) - self.logger.info('--- Delete %d/%d local files' % (len(delList), len(self.locList))) - - def syncToLocal(self): - countSuccess = 0 - countSkip = 0 - countFails = 0 - for fileItem in self.sourceFilesMatched: - if fileItem in self.locList: - self.logger.debug('Skip existed:%s' % fileItem.fileName) - countSkip += 1 - continue - if self.downloadFile(fileItem): - countSuccess += 1 - else: - countFails += 1 - # print stat - strSkip = ' Skip:%d' % countSkip if countSkip else '' - strFails = ' Fails:%d' % countFails if countFails else '' - self.logger.info('--- Download %d/%d%s%s' % (countSuccess, len(self.sourceFilesMatched), strSkip, strFails)) - - def deleteDropboxFiles(self): - """ Delete not matched files from Dropbox directory """ - sourceNames = [fileItem.fileName for fileItem in self.sourceFilesMatched] - delList = [fileItem for fileItem in self.dbList if fileItem.fileName not in sourceNames] - if not delList: - return - self.logger.debug('Dropbox files to delete:%s' % len(delList)) - for fileItem in delList: - self.deleteFile(fileItem) - self.logger.info('--- Success delete %d/%d dropbox files' % (len(delList), len(self.dbList))) - - def syncToDropbox(self): - countSuccess = 0 - countSkip = 0 - countFails = 0 - for fileItem in self.sourceFilesMatched: - if fileItem in self.dbList: - self.logger.debug('Skip existed:%s' % fileItem.fileName) - countSkip += 1 - continue - if self.uploadFile(fileItem): - countSuccess += 1 - else: - countFails += 1 - # print stat - strSkip = ' Skip:%d' % countSkip if countSkip else '' - strFails = ' Fails:%d' % countFails if countFails else '' - self.logger.info('--- Success upload %d/%d%s%s' % (countSuccess, len(self.sourceFilesMatched), strSkip, strFails)) - - # dropbox helpers - def listDropboxFiles(self): - """List a folder. - Return an array of filter items - """ - path = self.dropboxDir - self.logger.debug('Downloading dropbox list files...') - - try: - with self.stopwatch(__name__): - res = self.dbx.files_list_folder(path) - except dropbox.exceptions.ApiError as err: - self.dbList = [] - raise Exception('Folder listing failed for %s -- assumed empty:%s' % (path, err)) - else: - self.logger.debug('Dropbox files:%s' % len(res.entries)) - self.dbList = [self.filterItemByDropbox(fileMd) for fileMd in res.entries] - - def downloadFile(self, fileItem): - """Download a file. - Return True when success, or False if error occurs. - """ - fileName = fileItem.fileName - dbItem = next((f for f in self.dbList if f.fileName == fileName), None) - dbPath = os.path.join(self.dropboxDir, fileName) - locPath = os.path.join(self.localDir, fileName) - self.logger.debug('Downloading %s (%d bytes) ...' % (fileName, dbItem.fileSize)) - with self.stopwatch('downloading'): - try: - md = self.dbx.files_download_to_file(locPath, dbPath) - except dropbox.exceptions.ApiError as err: - raise Exception('%s - API error:%s' % (fileName, err)) - self.logger.debug('Success download - %s (%d bytes)' % (fileName, dbItem.fileSize)) - return True - - def uploadFile(self, fileItem): - """Upload a file. - Return the request response, or None in case of error. - """ - fileName = fileItem.fileName - dbPath = os.path.join(self.dropboxDir, fileName) - locPath = os.path.join(self.localDir, fileName) - mode = dropbox.files.WriteMode.overwrite - # mtime0 = os.path.getmtime(locPath) - # mtime = datetime.datetime(*time.gmtime(mtime0)[:6]) - mtime = self.mtime(locPath) - with open(locPath, 'rb') as f: - data = f.read() - self.logger.debug('Uploading %s (%d bytes) ...' % (fileName, len(data))) - # self.logger.debug('mtime %d %d ...' % (mtime, fileItem.fileModifyTime)) - with self.stopwatch('uploading'): - try: - res = self.dbx.files_upload( - data, dbPath, mode, - client_modified=mtime, - autorename=False, - mute=True) - except dropbox.exceptions.ApiError as err: - raise Exception('%s - API error:%s' % (fileName, err)) - # self.logger.debug('Success upload - res:%s' % (res)) - self.logger.debug('Success upload - %s (%s bytes)' % (fileName, len(data))) - return True - - def deleteFile(self, fileItem): - self.logger.debug('Deleting - \'%s\'' % (fileItem.fileName)) - with self.stopwatch('deleting'): - try: - md = self.dbx.files_delete(os.path.join(self.dropboxDir, fileItem.fileName)) - except dropbox.exceptions.ApiError as err: - raise Exception('%s - API error:%s' % (fileItem.fileName, err)) - self.logger.debug('Success delete - %s' % fileItem.fileName) - - # process helpers - def fixLocalTimestamps(self): - for fileName in self.locList: - self._debugFixLocalTimestamp(fileName) - self.logger.debug('Timestamps fixed in local files:%s' % len(self.locList)) - - def _debugFixLocalTimestamp(self, fileName): - from datetime import datetime - basename = os.path.splitext(fileName)[0] - newTime0 = None - if "." in basename: - newTime0 = datetime.strptime(basename, "%Y-%m-%d %H.%M.%S") - else: - newTime0 = datetime.strptime(basename, "%Y-%m-%d %H-%M-%S") - - newTime = time.mktime(newTime0.timetuple()) - # self.logger.debug('%s - %s -> %s' % (fileName, newTime0, newTime)) - os.utime(os.path.join(self.localDir, fileName), (newTime, newTime)) - - @contextlib.contextmanager - def stopwatch(self, message): - """Context manager to print how long a block of code took.""" - t0 = time.time() - try: - yield - finally: - t1 = time.time() - self.logger.debug('Total elapsed time for %s: %.3f' % (message, t1 - t0)) - - diff --git a/filters.py b/filters.py deleted file mode 100644 index b9f23ce..0000000 --- a/filters.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - -import datetime -import os - -class FileFilterItem(object): - """ - Contains file properties for filter - """ - def __init__(self, name=None, mtime=None, size=None): - self.fileName = name - self.fileModifyTime = mtime - self.fileSize = size - - def __eq__(self, other): - if type(other) is type(self): - return self.__dict__ == other.__dict__ - return False - - # def __eq__(self, other): - # if type(other) is not type(self): - # return False - # if self.fileName != other.fileName: - # return False - # if self.fileModifyTime != other.fileModibyTime: - # return False - # if self.fileSize != other.fileSize: - # return False - # return True - - def __ne__(self, other): - return not self.__eq__(other) - -class FileFilterBase(object): - """ - Base class to filter files from source list to target - Should get [FileFilterItem] objects - """ - def __init__(self): - pass - - def checkItemType(self, fileItem): - """ Check to match type [FileFilterItem] """ - if not isinstance(fileItem, FileFilterItem): - raise Exception('invalid type') - - def isMatch(self, fileItem): - """ Return true, if file is match filter """ - return True - - def filterFiles(self, files): - """ Return matched files """ - return [f for f in files if self.isMatch(f)] - -class FileFilterDays(FileFilterBase): - """ - Match only files which modification time is newer than matchDays days - """ - - def __init__(self, matchDays=None): - super(FileFilterDays, self).__init__() - self.matchDays = 0 - if matchDays: - self.matchDays = int(matchDays) - - def isMatch(self, fileItem): - if not self.matchDays: - return True - self.checkItemType(fileItem) - if fileItem.fileModifyTime: - now = datetime.datetime.now() - diff = now - fileItem.fileModifyTime - diffDays = diff.days - return diffDays < self.matchDays - return True - -class FileFilterMask(FileFilterBase): - """ - Exclude temporary files by mask (use fnmatch.fnmatch) - """ - - def __init__(self): - super(FileFilterMask, self).__init__() - self.excludeMasks = self.defaultMasks() - - def defaultMasks(self): - result = [] - result.append('.*') # any hidden files - result.append('~*') # temp files - result.append('thumbs.db') # windows thumbs - return result - - def addMask(self, fileMask): - self.excludeMasks.append(fileMask) - - def isMatch(self, fileItem): - import fnmatch - self.checkItemType(fileItem) - for fileMask in self.excludeMasks: - if fnmatch.fnmatch(fileItem.fileName, fileMask): - return False - return True diff --git a/requirements.txt b/requirements.txt index 65247b8..9435a89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -dropbox==11.26.0 \ No newline at end of file +dropbox==11.27.0 +PyQt5==5.15.6 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9c80f21 --- /dev/null +++ b/setup.py @@ -0,0 +1,26 @@ +from setuptools import setup +from setuptools.command.install import install +from pathlib import Path + + +class CreateLocalStateDir(install): + """ + Creates a local state directory to store + hashes, source and target files, keys, etc. + """ + + def run(self): + home = Path.home() / ".cloudsync" + home.mkdir(parents=True, exist_ok=True) + install.run(self) + + +setup( + name='cloudsync', + version='0.1', + description='A useful module', + author='aicpp(https://github.com/aicpp), Kyle Barry', + packages=['cloudsync'], + install_requires=['dropbox'], + cmdclass={'install': CreateLocalStateDir}, +)