From a1e2cae620bf195c70ea97ae09f3ea0db8488861 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 6 Dec 2020 15:29:09 +0100 Subject: [PATCH 1/4] consider exclude in shed update by considering exlude when - build and downloading tar balls - diff shed update should now consider exclude in .shed.yml --- planemo/shed/__init__.py | 15 +++++++++++---- planemo/shed/interface.py | 4 +++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/planemo/shed/__init__.py b/planemo/shed/__init__.py index 8db7ab35c..0a71563da 100644 --- a/planemo/shed/__init__.py +++ b/planemo/shed/__init__.py @@ -289,9 +289,10 @@ def report_non_existent_repository(realized_repository): def upload_repository(ctx, realized_repository, **kwds): """Upload a tool directory as a tarball to a tool shed.""" path = realized_repository.path + exclude = realized_repository.config.get("exclude", []) tar_path = kwds.get("tar") if not tar_path: - tar_path = build_tarball(path, **kwds) + tar_path = build_tarball(path, exclude, **kwds) if kwds.get("tar_only", False): name = realized_repository.pattern_to_file_name("shed_upload.tar.gz") shutil.copy(tar_path, name) @@ -355,6 +356,7 @@ def diff_repo(ctx, realized_repository, **kwds): def _diff_in(ctx, working, realized_repository, **kwds): path = realized_repository.path + exclude = realized_repository.config.get("exclude", []) shed_target_source = kwds.get("shed_target_source") label_a = "_%s_" % (shed_target_source if shed_target_source else "workingdir") @@ -399,7 +401,7 @@ def _diff_in(ctx, working, realized_repository, **kwds): **new_kwds ) else: - tar_path = build_tarball(path) + tar_path = build_tarball(path, exclude) os.mkdir(mine) shell(['tar', '-xzf', tar_path, '-C', mine]) shutil.rmtree(tar_path, ignore_errors=True) @@ -415,6 +417,8 @@ def _diff_in(ctx, working, realized_repository, **kwds): xml_diff = diff_and_remove(working, label_a, label_b, sys.stdout) cmd = ['diff', '-r', label_a, label_b] + for e in exclude: + cmd.extend(['--exclude', e]) if output: with open(output, 'ab') as fh: raw_diff = shell(cmd, cwd=working, stdout=fh) @@ -703,6 +707,7 @@ def create_repository_for(ctx, tsi, name, repo_config): def download_tarball(ctx, shed_context, realized_repository, **kwds): + exclude = realized_repository.config.get("exclude", []) repo_id = realized_repository.find_repository_id(ctx, shed_context) if repo_id is None: message = "Unable to find repository id, cannot download." @@ -714,7 +719,7 @@ def download_tarball(ctx, shed_context, realized_repository, **kwds): else: destination = destination_pattern to_directory = not destination.endswith("gz") - download_tar(shed_context.tsi, repo_id, destination, to_directory=to_directory) + download_tar(shed_context.tsi, repo_id, destination, to_directory=to_directory, exclude=exclude) if to_directory: clean = kwds.get("clean", False) if clean: @@ -723,7 +728,7 @@ def download_tarball(ctx, shed_context, realized_repository, **kwds): os.remove(archival_file) -def build_tarball(realized_path, **kwds): +def build_tarball(realized_path, exclude, **kwds): """Build a tool-shed tar ball for the specified path, caller is responsible for deleting this file. """ @@ -731,6 +736,8 @@ def build_tarball(realized_path, **kwds): # Simplest solution to sorting the files is to use a list, files = [] for dirpath, dirnames, filenames in os.walk(realized_path): + dirnames[:] = [d for d in dirnames if d not in exclude] + filenames[:] = [f for f in filenames if f not in exclude] for f in filenames: files.append(os.path.join(dirpath, f)) files.sort() diff --git a/planemo/shed/interface.py b/planemo/shed/interface.py index 9e25b058f..691ae1184 100644 --- a/planemo/shed/interface.py +++ b/planemo/shed/interface.py @@ -99,13 +99,15 @@ def find_category_ids(tsi, categories): return category_ids -def download_tar(tsi, repo_id, destination, to_directory): +def download_tar(tsi, repo_id, destination, to_directory, exclude): base_url = tsi.base_url if not base_url.endswith("/"): base_url += "/" download_url = REPOSITORY_DOWNLOAD_TEMPLATE % (base_url, repo_id) if to_directory: tar_args = ['-xzf', '-', '--strip-components=1'] + for e in exclude: + tar_args.extend(["--exclude", e]) untar_to(download_url, tar_args=tar_args, dest_dir=destination) else: untar_to(download_url, path=destination) From 3150db860fdc5111eb2465d88b9914d722efd99e Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 6 Dec 2020 16:52:37 +0100 Subject: [PATCH 2/4] use _shed_config_excludes --- planemo/shed/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planemo/shed/__init__.py b/planemo/shed/__init__.py index 0a71563da..2ea819492 100644 --- a/planemo/shed/__init__.py +++ b/planemo/shed/__init__.py @@ -289,7 +289,7 @@ def report_non_existent_repository(realized_repository): def upload_repository(ctx, realized_repository, **kwds): """Upload a tool directory as a tarball to a tool shed.""" path = realized_repository.path - exclude = realized_repository.config.get("exclude", []) + exclude = _shed_config_excludes(realized_repository.config) tar_path = kwds.get("tar") if not tar_path: tar_path = build_tarball(path, exclude, **kwds) @@ -707,7 +707,7 @@ def create_repository_for(ctx, tsi, name, repo_config): def download_tarball(ctx, shed_context, realized_repository, **kwds): - exclude = realized_repository.config.get("exclude", []) + exclude = _shed_config_excludes(realized_repository.config) repo_id = realized_repository.find_repository_id(ctx, shed_context) if repo_id is None: message = "Unable to find repository id, cannot download." From 731f5535c966374625a7782e762670d17a5b0961 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 6 Dec 2020 17:06:07 +0100 Subject: [PATCH 3/4] use exclude in _glob --- planemo/shed/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/planemo/shed/__init__.py b/planemo/shed/__init__.py index 2ea819492..b3c4758ae 100644 --- a/planemo/shed/__init__.py +++ b/planemo/shed/__init__.py @@ -1065,6 +1065,7 @@ def _repo_names(self): def _realized_files(self, name): config = self._realize_config(name) + exclude = _shed_config_excludes(config) realized_files = [] missing = [] for include_info in config["include"]: @@ -1078,7 +1079,7 @@ def _realized_files(self, name): for source in source_list: include = include_info.copy() include["source"] = source - included = RealizedFile.realized_files_for(self.path, include) + included = RealizedFile.realized_files_for(self.path, include, exclude) if not included: missing.append(include) else: @@ -1160,7 +1161,7 @@ def realize_to(self, directory): os.symlink(source_path, target_path) @staticmethod - def realized_files_for(path, include_info): + def realized_files_for(path, include_info, exclude): if not isinstance(include_info, dict): include_info = {"source": include_info} source = include_info.get("source") @@ -1175,7 +1176,7 @@ def realized_files_for(path, include_info): if "*" in source or "?" in source or os.path.isdir(abs_source): raise ValueError("destination must be a directory (with trailing slash) if source is a folder or uses wildcards") realized_files = [] - for globbed_file in _glob(path, source): + for globbed_file in _glob(path, source, exclude): src = os.path.relpath(globbed_file, path) if not destination.endswith("/"): # Given a filename, just use it! @@ -1342,11 +1343,14 @@ def install_args(self, ctx, shed_context): ) -def _glob(path, pattern): +def _glob(path, pattern, exclude=None): pattern = os.path.join(path, pattern) if os.path.isdir(pattern): pattern = "%s/**" % pattern - return glob.glob(pattern) + if exclude is None: + return glob.glob(pattern) + else: + return [_ for _ in glob.glob(pattern) if _ not in exclude] def _shed_config_excludes(config): From 54a3faa387d0040ba9b9aa4eed4d793b2f6356e8 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 13 Dec 2020 18:13:43 +0100 Subject: [PATCH 4/4] also consider exclude in shed_build --- planemo/commands/cmd_shed_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/planemo/commands/cmd_shed_build.py b/planemo/commands/cmd_shed_build.py index 3348c66da..f6a0e2a17 100644 --- a/planemo/commands/cmd_shed_build.py +++ b/planemo/commands/cmd_shed_build.py @@ -20,7 +20,8 @@ def cli(ctx, path, **kwds): (which you could upload to the Tool Shed manually). """ def build(realized_repository): - tarpath = shed.build_tarball(realized_repository.path) + exclude = shed._shed_config_excludes(realized_repository.config) + tarpath = shed.build_tarball(realized_repository.path, exclude) outpath = realized_repository.real_path + ".tar.gz" shutil.move(tarpath, outpath) print("Created: %s" % (outpath))