From c78bd5eafa3d9450305cc8b32ba9cf8c64c42ba7 Mon Sep 17 00:00:00 2001
From: M Bernt <m.bernt@ufz.de>
Date: Mon, 18 Nov 2024 16:11:23 +0100
Subject: [PATCH] DMs: download data to extra_file_path (#71)

* DMs download data to extra_file_path

* fix zenodo link

* add lint skip for virsorter

* fix plural

* implement test mode for virsorter
---
 .../data_manager/checkv_datamanager.xml       |  5 ++--
 .../data_manager/genomad_datamanager.xml      |  4 ++-
 .../data_manager/iphop_datamanager.xml        |  6 ++---
 .../data_manager/phabox_datamanager.xml       |  4 ++-
 .../data_manager/vibrant_datamanager.xml      |  5 ++--
 .../data_manager_virsorter/.lint_skip         |  1 +
 .../data_manager/virsorter_datamanager.xml    | 27 +++++++++++++++----
 7 files changed, 38 insertions(+), 14 deletions(-)
 create mode 100644 data_managers/data_manager_virsorter/.lint_skip

diff --git a/data_managers/data_manager_checkv/data_manager/checkv_datamanager.xml b/data_managers/data_manager_checkv/data_manager/checkv_datamanager.xml
index a2ba3f09..0b4d82b7 100644
--- a/data_managers/data_manager_checkv/data_manager/checkv_datamanager.xml
+++ b/data_managers/data_manager_checkv/data_manager/checkv_datamanager.xml
@@ -2,7 +2,7 @@
     <description>database builder</description>
     <macros>
         <token name="@TOOL_VERSION@">1.0.3</token>
-        <token name="@VERSION_SUFFIX@">1</token>
+        <token name="@VERSION_SUFFIX@">2</token>
         <token name="@PROFILE@">22.01</token>
     </macros>
     <requirements>
@@ -16,7 +16,8 @@
         cd checkv-db-v${version}/genome_db/ &&
         diamond makedb --in checkv_reps.faa --db checkv_reps &&
         cd - &&
-        mv checkv-db-v${version} checkv-db &&
+        mkdir -p '$out_file.extra_files_path' &&
+        mv checkv-db-v${version} '$out_file.extra_files_path'/checkv-db &&
         cp '$dmjson' '$out_file'
     ]]></command>
     <configfiles>
diff --git a/data_managers/data_manager_genomad/data_manager/genomad_datamanager.xml b/data_managers/data_manager_genomad/data_manager/genomad_datamanager.xml
index 02f6196e..a40f30e8 100644
--- a/data_managers/data_manager_genomad/data_manager/genomad_datamanager.xml
+++ b/data_managers/data_manager_genomad/data_manager/genomad_datamanager.xml
@@ -3,7 +3,7 @@
     <description>database builder</description>
     <macros>
         <token name="@TOOL_VERSION@">1.8.0</token>
-        <token name="@VERSION_SUFFIX@">0</token>
+        <token name="@VERSION_SUFFIX@">1</token>
         <token name="@PROFILE@">22.01</token>
     </macros>
     <requirements>
@@ -18,6 +18,8 @@
                 https://zenodo.org/records/10594875/files/genomad_db_v${version}.tar.gz?download=1
             ## #end if
         | tar -xzv &&
+        mkdir -p '$out_file.extra_files_path' &&
+        mv genomad_db '$out_file.extra_files_path' &&
         cp '$dmjson' '$out_file'
     ]]></command>
     <configfiles>
diff --git a/data_managers/data_manager_iphop/data_manager/iphop_datamanager.xml b/data_managers/data_manager_iphop/data_manager/iphop_datamanager.xml
index e4da3219..409b6726 100644
--- a/data_managers/data_manager_iphop/data_manager/iphop_datamanager.xml
+++ b/data_managers/data_manager_iphop/data_manager/iphop_datamanager.xml
@@ -2,16 +2,16 @@
     <description>database builder</description>
     <macros>
         <token name="@TOOL_VERSION@">1.3.3</token>
-        <token name="@VERSION_SUFFIX@">1</token>
+        <token name="@VERSION_SUFFIX@">2</token>
         <token name="@PROFILE@">22.01</token>
     </macros>
     <requirements>
         <requirement type="package" version="@TOOL_VERSION@">iphop</requirement>
     </requirements>
     <command detect_errors="exit_code"><![CDATA[
-        mkdir db &&
+        mkdir -p '$out_file.extra_files_path'/db &&
         iphop download
-            --db_dir db/
+            --db_dir '$out_file.extra_files_path'/db
             --db_version $db_version
             ## --full_verify  https://bitbucket.org/srouxjgi/iphop/issues/114/db-download-verification-error
             --no_prompt &&
diff --git a/data_managers/data_manager_phabox/data_manager/phabox_datamanager.xml b/data_managers/data_manager_phabox/data_manager/phabox_datamanager.xml
index b29be6ba..725ec000 100644
--- a/data_managers/data_manager_phabox/data_manager/phabox_datamanager.xml
+++ b/data_managers/data_manager_phabox/data_manager/phabox_datamanager.xml
@@ -3,7 +3,7 @@
     <macros>
         <token name="@TOOL_VERSION@">2.1.5</token>
         <token name="@DB_VERSION@">2</token>
-        <token name="@VERSION_SUFFIX@">0</token>
+        <token name="@VERSION_SUFFIX@">1</token>
         <token name="@PROFILE@">22.01</token>
     </macros>
     <requirements>
@@ -13,6 +13,8 @@
     <command detect_errors="exit_code"><![CDATA[
         wget https://github.com/KennthShang/PhaBOX/releases/download/v2/phabox_db_v@DB_VERSION@.zip &&
         unzip phabox_db_v@DB_VERSION@.zip &&
+        mkdir -p '$out_file.extra_files_path' &&
+        mv "phabox_db_v@DB_VERSION@" '$out_file.extra_files_path' &&
         cp '$dmjson' '$out_file'
     ]]></command>
     <configfiles>
diff --git a/data_managers/data_manager_vibrant/data_manager/vibrant_datamanager.xml b/data_managers/data_manager_vibrant/data_manager/vibrant_datamanager.xml
index 84fdbbcf..9cddc24b 100644
--- a/data_managers/data_manager_vibrant/data_manager/vibrant_datamanager.xml
+++ b/data_managers/data_manager_vibrant/data_manager/vibrant_datamanager.xml
@@ -2,7 +2,7 @@
     <description>database builder</description>
     <macros>
         <token name="@TOOL_VERSION@">1.2.1</token>
-        <token name="@VERSION_SUFFIX@">0</token>
+        <token name="@VERSION_SUFFIX@">1</token>
         <token name="@PROFILE@">22.01</token>
     </macros>
     <xrefs>
@@ -13,7 +13,8 @@
     </requirements>
     <command detect_errors="exit_code"><![CDATA[
         set -eo pipefail; 
-        download-db.sh vibrant_db &&
+        mkdir -p '$out_file.extra_files_path'/vibrant_db &&
+        download-db.sh '$out_file.extra_files_path'/vibrant_db &&
         cp '$dmjson' '$out_file'
     ]]></command>
     <configfiles>
diff --git a/data_managers/data_manager_virsorter/.lint_skip b/data_managers/data_manager_virsorter/.lint_skip
new file mode 100644
index 00000000..c28a1b42
--- /dev/null
+++ b/data_managers/data_manager_virsorter/.lint_skip
@@ -0,0 +1 @@
+InputsMissing
\ No newline at end of file
diff --git a/data_managers/data_manager_virsorter/data_manager/virsorter_datamanager.xml b/data_managers/data_manager_virsorter/data_manager/virsorter_datamanager.xml
index eca1cb36..674af9da 100644
--- a/data_managers/data_manager_virsorter/data_manager/virsorter_datamanager.xml
+++ b/data_managers/data_manager_virsorter/data_manager/virsorter_datamanager.xml
@@ -3,9 +3,9 @@
     <description>database builder</description>
     <macros>
         <token name="@TOOL_VERSION@">2.2.4</token>  <!-- when updating make sure to double check the db version -->
-        <token name="@VERSION_SUFFIX@">0</token>
+        <token name="@VERSION_SUFFIX@">1</token>
         <token name="@PROFILE@">22.01</token>
-        <!-- zenodo ID and therebut the version of the reference data is hardcoded in the current version virsorter:
+        <!-- zenodo ID and the version of the reference data is hardcoded in the current version virsorter:
              https://github.com/jiarong/VirSorter2/blob/c0dcac53f3373244bf1bd552a7c2104d53802df7/virsorter/rules/setup.smk#L13 
              check and update the DB version when updating the VS version 
              might also change/improve in the future https://github.com/jiarong/VirSorter2/issues/205#issuecomment-2184184977
@@ -18,7 +18,12 @@
         <requirement type="package" version="2.7.0">pulp</requirement> <!-- needs to be pinned, because the old-ish snakemake used in vs2 struggles with https://github.com/snakemake/snakemake/issues/2607 -->
     </requirements>
     <command detect_errors="exit_code"><![CDATA[
-        virsorter setup -d db -j \${GALAXY_SLOTS:-1} --skip-deps-install &&
+        mkdir -p '$out_file.extra_files_path'/db &&
+        #if $test == "true"
+            touch '$out_file.extra_files_path'/db/test &&
+        #else
+            virsorter setup -d '$out_file.extra_files_path'/db -j \${GALAXY_SLOTS:-1} --skip-deps-install &&
+        #end if
         cp '$dmjson' '$out_file'
     ]]></command>
     <configfiles>
@@ -37,12 +42,15 @@
 }]]>
         </configfile>
     </configfiles>
-    <inputs/>
+    <inputs>
+        <param name="test" type="hidden"/>
+    </inputs>
     <outputs>
         <data name="out_file" format="data_manager_json" />
     </outputs>
     <tests>
         <test expect_num_outputs="1">
+            <param name="test" value="true"/>
             <output name="out_file">
                 <assert_contents>
                     <has_text text='"value":"@VIRSORTER_DB_VERSION@"'/>
@@ -50,9 +58,18 @@
                 </assert_contents>
             </output>
         </test>
+        <!-- 10GB test data seem to be to much for CI. but this test should be executed locally for updates
+        <test expect_num_outputs="1">
+            <output name="out_file">
+                <assert_contents>
+                    <has_text text='"value":"@VIRSORTER_DB_VERSION@"'/>
+                    <has_text text='"name":"Version @VIRSORTER_DB_VERSION@"'/>
+                </assert_contents>
+            </output>
+        </test> -->
     </tests>
     <help><![CDATA[
-Download and extract virsorter database version @VIRSORTER_DB_VERSION@ from zenodo (https://zenodo.org/records/10594875)
+Download and extract virsorter database version @VIRSORTER_DB_VERSION@ from zenodo (https://zenodo.org/records/4269607)
     ]]></help>
     <citations>
         <citation type="doi">10.1186/s40168-020-00990-y</citation>