diff --git a/src/clusterfuzz/_internal/google_cloud_utils/storage.py b/src/clusterfuzz/_internal/google_cloud_utils/storage.py index b8ae76fc93..5915f655a0 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/storage.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/storage.py @@ -240,7 +240,6 @@ def list_blobs(self, remote_path, recursive=True, names_only=False): client = _storage_client() bucket = client.bucket(bucket_name) - properties = {} if recursive: delimiter = None @@ -252,23 +251,37 @@ def list_blobs(self, remote_path, recursive=True, names_only=False): else: fields = None - iterator = bucket.list_blobs( - prefix=path, delimiter=delimiter, fields=fields) - for blob in iterator: - properties['bucket'] = bucket_name - properties['name'] = blob.name - properties['updated'] = blob.updated - properties['size'] = blob.size - - yield properties - - if not recursive: - # When doing delimiter listings, the "directories" will be in `prefixes`. - for prefix in iterator.prefixes: - properties['bucket'] = bucket_name - properties['name'] = prefix + iterations = 0 + while True: + iterations += 1 + iterator = bucket.list_blobs( + prefix=path, delimiter=delimiter, fields=fields) + for blob in iterator: + properties = { + 'bucket': bucket_name, + 'name': blob.name, + 'updated': blob.updated, + 'size': blob.size, + } + yield properties + if not recursive: + # When doing delimiter listings, the "directories" will be in + # `prefixes`. + for prefix in iterator.prefixes: + properties = { + 'bucket': bucket_name, + 'name': prefix, + } + yield properties + + next_page_token = iterator.next_page_token + if next_page_token is None: + break + if iterations and iterations % 50 == 0: + logs.error('Might be infinite looping.') + def copy_file_from(self, remote_path, local_path): """Copy file from a remote path to a local path.""" client = _storage_client()