From 842951bbd5829e7a8433da34b66af9ba7cc11fdb Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Mon, 30 Sep 2019 20:27:02 +0100 Subject: [PATCH 01/11] fix touch timestamp; duplicate of #39 --- adb-sync | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adb-sync b/adb-sync index 3b4eaf6..8379dbf 100755 --- a/adb-sync +++ b/adb-sync @@ -303,14 +303,14 @@ class AdbFileSystem(GlobLike, OSLike): # TODO(rpolzer): Find out why this does not work (returns status 255). """Set the time of a file to a specified unix time.""" atime, mtime = times - timestr = time.strftime('%Y%m%d.%H%M%S', + timestr = time.strftime('%Y%m%d%H%M.%S', time.localtime(mtime)).encode('ascii') if subprocess.call( self.adb + [b'shell', b'touch -mt %s %s' % (timestr, self.QuoteArgument(path))]) != 0: raise OSError('touch failed') - timestr = time.strftime('%Y%m%d.%H%M%S', + timestr = time.strftime('%Y%m%d%H%M.%S', time.localtime(atime)).encode('ascii') if subprocess.call( self.adb + From 82edea46e9ad559afbbf23961f6c0ed912cd34ec Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Fri, 4 Oct 2019 00:35:18 +0100 Subject: [PATCH 02/11] fix changes being ignored because file size stayed the same --- adb-sync | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/adb-sync b/adb-sync index 8379dbf..3d98a64 100755 --- a/adb-sync +++ b/adb-sync @@ -558,10 +558,6 @@ class FileSyncer(object): elif stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode): # Dir vs file? Nothing to do here yet. pass - else: - # File vs file? Compare sizes. - if localstat.st_size == remotestat.st_size: - continue l2r = self.local_to_remote r2l = self.remote_to_local if l2r and r2l: @@ -573,6 +569,9 @@ class FileSyncer(object): r2l = False elif localminute < remoteminute: l2r = False + else: + # same time stamp + continue if l2r and r2l: logging.warning('Unresolvable: %r', name) continue From f3fb013d66192e5b644208d13c8f22b9e64a689c Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Sat, 5 Oct 2019 11:35:12 +0100 Subject: [PATCH 03/11] further fix on time push - android's mtime needs utc --- adb-sync | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adb-sync b/adb-sync index 3d98a64..47801f8 100755 --- a/adb-sync +++ b/adb-sync @@ -304,14 +304,14 @@ class AdbFileSystem(GlobLike, OSLike): """Set the time of a file to a specified unix time.""" atime, mtime = times timestr = time.strftime('%Y%m%d%H%M.%S', - time.localtime(mtime)).encode('ascii') + time.gmtime(mtime)).encode('ascii') if subprocess.call( self.adb + [b'shell', b'touch -mt %s %s' % (timestr, self.QuoteArgument(path))]) != 0: raise OSError('touch failed') timestr = time.strftime('%Y%m%d%H%M.%S', - time.localtime(atime)).encode('ascii') + time.gmtime(atime)).encode('ascii') if subprocess.call( self.adb + [b'shell', From 5230c7775a8462be85ae14bf10de37987570bb2b Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Sun, 6 Oct 2019 00:28:20 +0200 Subject: [PATCH 04/11] use UTC for everything! --- adb-sync | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/adb-sync b/adb-sync index 47801f8..067f74e 100755 --- a/adb-sync +++ b/adb-sync @@ -24,6 +24,7 @@ import re import stat import subprocess import time +import calendar from types import TracebackType from typing import Callable, cast, Dict, List, IO, Iterable, Optional, Tuple, Type @@ -177,7 +178,7 @@ class AdbFileSystem(GlobLike, OSLike): st_mode |= stat.S_IFSOCK st_size = None if groups['st_size'] is None else int(groups['st_size']) st_mtime = int( - time.mktime( + calendar.timegm( time.strptime( match.group('st_mtime').decode('ascii'), '%Y-%m-%d %H:%M'))) @@ -231,7 +232,7 @@ class AdbFileSystem(GlobLike, OSLike): """List the contents of a directory, caching them for later lstat calls.""" with Stdout(self.adb + [b'shell', - b'ls -al %s' % (self.QuoteArgument(path + b'/'),)]) as stdout: + b'TZ=utc ls -al %s' % (self.QuoteArgument(path + b'/'),)]) as stdout: for line in stdout: if line.startswith(b'total '): continue @@ -252,7 +253,7 @@ class AdbFileSystem(GlobLike, OSLike): return self.stat_cache[path] with Stdout( self.adb + - [b'shell', b'ls -ald %s' % (self.QuoteArgument(path),)]) as stdout: + [b'shell', b'TZ=utc ls -ald %s' % (self.QuoteArgument(path),)]) as stdout: for line in stdout: if line.startswith(b'total '): continue @@ -269,7 +270,7 @@ class AdbFileSystem(GlobLike, OSLike): return self.stat_cache[path] with Stdout( self.adb + - [b'shell', b'ls -aldL %s' % (self.QuoteArgument(path),)]) as stdout: + [b'shell', b'TZ=utc ls -aldL %s' % (self.QuoteArgument(path),)]) as stdout: for line in stdout: if line.startswith(b'total '): continue @@ -308,14 +309,14 @@ class AdbFileSystem(GlobLike, OSLike): if subprocess.call( self.adb + [b'shell', - b'touch -mt %s %s' % (timestr, self.QuoteArgument(path))]) != 0: + b'TZ=utc touch -mt %s %s' % (timestr, self.QuoteArgument(path))]) != 0: raise OSError('touch failed') timestr = time.strftime('%Y%m%d%H%M.%S', time.gmtime(atime)).encode('ascii') if subprocess.call( self.adb + [b'shell', - b'touch -at %s %s' % (timestr, self.QuoteArgument(path))]) != 0: + b'TZ=utc touch -at %s %s' % (timestr, self.QuoteArgument(path))]) != 0: raise OSError('touch failed') def glob(self, path: bytes) -> Iterable[bytes]: # glob's name, so pylint: disable=g-bad-name @@ -639,9 +640,9 @@ class FileSyncer(object): self.num_bytes += s.st_size if not self.dry_run: if self.preserve_times: - logging.info('%s-Times: accessed %s, modified %s', self.push[i], - time.asctime(time.localtime(s.st_atime)), - time.asctime(time.localtime(s.st_mtime))) + logging.info('%s-Times: accessed %s UTC, modified %s UTC', self.push[i], + time.asctime(time.gmtime(s.st_atime)), + time.asctime(time.gmtime(s.st_mtime))) self.dst_fs[i].utime(dst_name, (s.st_atime, s.st_mtime)) def TimeReport(self) -> None: From ba563f177197a0ec0f46636a965ee6e7f9e262cd Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Sun, 6 Oct 2019 00:29:11 +0200 Subject: [PATCH 05/11] display unicode in logging properly --- adb-sync | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/adb-sync b/adb-sync index 067f74e..2326425 100755 --- a/adb-sync +++ b/adb-sync @@ -376,7 +376,7 @@ def BuildFileList(fs: OSLike, path: bytes, follow_links: bool, elif stat.S_ISLNK(statresult.st_mode) and not follow_links: yield prefix, statresult else: - logging.info('Unsupported file: %r.', path) + logging.info('Unsupported file: %r.', path.decode('utf-8')) def DiffLists(a: Iterable[Tuple[bytes, os.stat_result]], @@ -461,7 +461,7 @@ class DeleteInterruptedFile(object): exc_tb: Optional[TracebackType]) -> bool: if exc_type is not None: logging.info('Interrupted-%s-Delete: %r', - 'Pull' if self.fs == os else 'Push', self.name) + 'Pull' if self.fs == os else 'Push', self.name.decode('utf-8')) if not self.dry_run: self.fs.unlink(self.name) return False @@ -538,7 +538,7 @@ class FileSyncer(object): else: for name, s in reversed(self.dst_only[i]): dst_name = self.dst[i] + name - logging.info('%s-Delete: %r', self.push[i], dst_name) + logging.info('%s-Delete: %r', self.push[i], dst_name.decode('utf-8')) if stat.S_ISDIR(s.st_mode): if not self.dry_run: self.dst_fs[i].rmdir(dst_name) @@ -574,7 +574,7 @@ class FileSyncer(object): # same time stamp continue if l2r and r2l: - logging.warning('Unresolvable: %r', name) + logging.warning('Unresolvable: %r', name.decode('utf-8')) continue if l2r: i = 0 # Local to remote operation. @@ -585,7 +585,7 @@ class FileSyncer(object): src_stat = remotestat dst_stat = localstat dst_name = self.dst[i] + name - logging.info('%s-Delete-Conflicting: %r', self.push[i], dst_name) + logging.info('%s-Delete-Conflicting: %r', self.push[i], dst_name.decode('utf-8')) if stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode): if not self.allow_replace: logging.info('Would have to replace to do this. ' @@ -628,7 +628,7 @@ class FileSyncer(object): for name, s in self.src_only[i]: src_name = self.src[i] + name dst_name = self.dst[i] + name - logging.info('%s: %r', self.push[i], dst_name) + logging.info('%s: %r', self.push[i], dst_name.decode('utf-8')) if stat.S_ISDIR(s.st_mode): if not self.dry_run: self.dst_fs[i].makedirs(dst_name) @@ -863,7 +863,7 @@ def main() -> None: return for i in range(len(localpaths)): - logging.info('Sync: local %r, remote %r', localpaths[i], remotepaths[i]) + logging.info('Sync: local %r, remote %r', localpaths[i].decode('utf-8'), remotepaths[i].decode('utf-8')) syncer = FileSyncer(adb, localpaths[i], remotepaths[i], local_to_remote, remote_to_local, preserve_times, delete_missing, allow_overwrite, allow_replace, copy_links, dry_run) From f7f9533e3fb42539984ef3a52a7d24978aeb57a4 Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Tue, 15 Oct 2019 01:43:26 +0100 Subject: [PATCH 06/11] handle situation where file system does not allow timestamp pushes --- adb-sync | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/adb-sync b/adb-sync index 2326425..9779ece 100755 --- a/adb-sync +++ b/adb-sync @@ -567,8 +567,14 @@ class FileSyncer(object): localminute = int(localstat.st_mtime / 60) remoteminute = int(remotestat.st_mtime / 60) if localminute > remoteminute: + if localstat.st_size == remotestat.st_size and localstat.st_mtime == localstat.st_atime: + # files have same size and the more recent file has not been modified since creation + continue r2l = False elif localminute < remoteminute: + if localstat.st_size == remotestat.st_size and remotestat.st_mtime == remotestat.st_atime: + # files have same size and the more recent file has not been modified since creation + continue l2r = False else: # same time stamp @@ -643,7 +649,10 @@ class FileSyncer(object): logging.info('%s-Times: accessed %s UTC, modified %s UTC', self.push[i], time.asctime(time.gmtime(s.st_atime)), time.asctime(time.gmtime(s.st_mtime))) - self.dst_fs[i].utime(dst_name, (s.st_atime, s.st_mtime)) + try: + self.dst_fs[i].utime(dst_name, (s.st_atime, s.st_mtime)) + except Exception: + logging.warning('Unable to set times') def TimeReport(self) -> None: """Report time and amount of data transferred.""" From efd69dcd4cdb3aff0e0d953d7e3ba161c2eee0e5 Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Tue, 15 Oct 2019 11:22:15 +0100 Subject: [PATCH 07/11] fix timestamp related logic for one direction sync --- adb-sync | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/adb-sync b/adb-sync index 9779ece..0a4aa17 100755 --- a/adb-sync +++ b/adb-sync @@ -559,26 +559,26 @@ class FileSyncer(object): elif stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode): # Dir vs file? Nothing to do here yet. pass + # Truncate times to full minutes, as Android's "ls" only outputs minute + # accuracy. + localminute = int(localstat.st_mtime / 60) + remoteminute = int(remotestat.st_mtime / 60) + if localstat.st_size == remotestat.st_size: + if localminute == remoteminute: + continue + if localminute > remoteminute and localstat.st_mtime == localstat.st_atime: + # files have same size and the more recent file has not been modified since creation + continue + if localminute < remoteminute and remotestat.st_mtime == remotestat.st_atime: + # files have same size and the more recent file has not been modified since creation + continue l2r = self.local_to_remote r2l = self.remote_to_local if l2r and r2l: - # Truncate times to full minutes, as Android's "ls" only outputs minute - # accuracy. - localminute = int(localstat.st_mtime / 60) - remoteminute = int(remotestat.st_mtime / 60) if localminute > remoteminute: - if localstat.st_size == remotestat.st_size and localstat.st_mtime == localstat.st_atime: - # files have same size and the more recent file has not been modified since creation - continue r2l = False elif localminute < remoteminute: - if localstat.st_size == remotestat.st_size and remotestat.st_mtime == remotestat.st_atime: - # files have same size and the more recent file has not been modified since creation - continue l2r = False - else: - # same time stamp - continue if l2r and r2l: logging.warning('Unresolvable: %r', name.decode('utf-8')) continue From 67e587b8bf6a6dfe99c7fee5c110e41ab0f802b2 Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Tue, 15 Oct 2019 19:29:39 +0100 Subject: [PATCH 08/11] remove the optimization for equal sized file when times differ, as they just don't work --- adb-sync | 6 ------ 1 file changed, 6 deletions(-) diff --git a/adb-sync b/adb-sync index 0a4aa17..409405e 100755 --- a/adb-sync +++ b/adb-sync @@ -566,12 +566,6 @@ class FileSyncer(object): if localstat.st_size == remotestat.st_size: if localminute == remoteminute: continue - if localminute > remoteminute and localstat.st_mtime == localstat.st_atime: - # files have same size and the more recent file has not been modified since creation - continue - if localminute < remoteminute and remotestat.st_mtime == remotestat.st_atime: - # files have same size and the more recent file has not been modified since creation - continue l2r = self.local_to_remote r2l = self.remote_to_local if l2r and r2l: From 9483953692486bbbb5dd4d7c2ba57550147d2fdd Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Tue, 15 Oct 2019 19:38:28 +0100 Subject: [PATCH 09/11] remove try / catch block over timestamp setting --- adb-sync | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/adb-sync b/adb-sync index 409405e..1ef3740 100755 --- a/adb-sync +++ b/adb-sync @@ -643,10 +643,7 @@ class FileSyncer(object): logging.info('%s-Times: accessed %s UTC, modified %s UTC', self.push[i], time.asctime(time.gmtime(s.st_atime)), time.asctime(time.gmtime(s.st_mtime))) - try: - self.dst_fs[i].utime(dst_name, (s.st_atime, s.st_mtime)) - except Exception: - logging.warning('Unable to set times') + self.dst_fs[i].utime(dst_name, (s.st_atime, s.st_mtime)) def TimeReport(self) -> None: """Report time and amount of data transferred.""" From 8b590dd047ff4c59e2d393f2124f8d01f3af552f Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Mon, 25 Nov 2019 00:44:22 +0000 Subject: [PATCH 10/11] when -L (copy-link) not specified, ignore links completely --- adb-sync | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adb-sync b/adb-sync index 1ef3740..55e70ce 100755 --- a/adb-sync +++ b/adb-sync @@ -374,7 +374,7 @@ def BuildFileList(fs: OSLike, path: bytes, follow_links: bool, elif stat.S_ISREG(statresult.st_mode): yield prefix, statresult elif stat.S_ISLNK(statresult.st_mode) and not follow_links: - yield prefix, statresult + pass # ignore links completely else: logging.info('Unsupported file: %r.', path.decode('utf-8')) From 9d7227f5032fb18dcc3d6c8977f90d009fd71acb Mon Sep 17 00:00:00 2001 From: Lingnan Dai Date: Mon, 25 Nov 2019 19:05:02 +0000 Subject: [PATCH 11/11] when -L (copy-link) is specified, allow writing to true targets pointed to by the link (local only) --- adb-sync | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/adb-sync b/adb-sync index 55e70ce..fcd75fa 100755 --- a/adb-sync +++ b/adb-sync @@ -585,6 +585,9 @@ class FileSyncer(object): src_stat = remotestat dst_stat = localstat dst_name = self.dst[i] + name + if r2l and self.copy_links: + # use true destination if we allow copy links (only for local) + dst_name = os.path.realpath(dst_name) logging.info('%s-Delete-Conflicting: %r', self.push[i], dst_name.decode('utf-8')) if stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode): if not self.allow_replace: @@ -628,6 +631,9 @@ class FileSyncer(object): for name, s in self.src_only[i]: src_name = self.src[i] + name dst_name = self.dst[i] + name + if i == 1 and self.copy_links: + # use true destination if we allow copy links (only for local) + dst_name = os.path.realpath(dst_name) logging.info('%s: %r', self.push[i], dst_name.decode('utf-8')) if stat.S_ISDIR(s.st_mode): if not self.dry_run: