DMs: download data to extra_file_path (Helmholtz-UFZ#71)

* DMs download data to extra_file_path * fix zenodo link * add lint skip for virsorter * fix plural * implement test mode for virsorter
rmassei · Nov 18, 2024 · c78bd5e · c78bd5e
1 parent 3c8e23e
commit c78bd5e
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 14 deletions.
diff --git a/data_managers/data_manager_checkv/data_manager/checkv_datamanager.xml b/data_managers/data_manager_checkv/data_manager/checkv_datamanager.xml
@@ -2,7 +2,7 @@
     <description>database builder</description>
     <macros>
         <token name="@TOOL_VERSION@">1.0.3</token>
-        <token name="@VERSION_SUFFIX@">1</token>
+        <token name="@VERSION_SUFFIX@">2</token>
         <token name="@PROFILE@">22.01</token>
     </macros>
     <requirements>
@@ -16,7 +16,8 @@
         cd checkv-db-v${version}/genome_db/ &&
         diamond makedb --in checkv_reps.faa --db checkv_reps &&
         cd - &&
-        mv checkv-db-v${version} checkv-db &&
+        mkdir -p '$out_file.extra_files_path' &&
+        mv checkv-db-v${version} '$out_file.extra_files_path'/checkv-db &&
         cp '$dmjson' '$out_file'
     ]]></command>
     <configfiles>

diff --git a/data_managers/data_manager_genomad/data_manager/genomad_datamanager.xml b/data_managers/data_manager_genomad/data_manager/genomad_datamanager.xml
@@ -3,7 +3,7 @@
     <description>database builder</description>
     <macros>
         <token name="@TOOL_VERSION@">1.8.0</token>
-        <token name="@VERSION_SUFFIX@">0</token>
+        <token name="@VERSION_SUFFIX@">1</token>
         <token name="@PROFILE@">22.01</token>
     </macros>
     <requirements>
@@ -18,6 +18,8 @@
                 https://zenodo.org/records/10594875/files/genomad_db_v${version}.tar.gz?download=1
             ## #end if
         | tar -xzv &&
+        mkdir -p '$out_file.extra_files_path' &&
+        mv genomad_db '$out_file.extra_files_path' &&
         cp '$dmjson' '$out_file'
     ]]></command>
     <configfiles>

diff --git a/data_managers/data_manager_iphop/data_manager/iphop_datamanager.xml b/data_managers/data_manager_iphop/data_manager/iphop_datamanager.xml
@@ -2,16 +2,16 @@
     <description>database builder</description>
     <macros>
         <token name="@TOOL_VERSION@">1.3.3</token>
-        <token name="@VERSION_SUFFIX@">1</token>
+        <token name="@VERSION_SUFFIX@">2</token>
         <token name="@PROFILE@">22.01</token>
     </macros>
     <requirements>
         <requirement type="package" version="@TOOL_VERSION@">iphop</requirement>
     </requirements>
     <command detect_errors="exit_code"><![CDATA[
-        mkdir db &&
+        mkdir -p '$out_file.extra_files_path'/db &&
         iphop download
-            --db_dir db/
+            --db_dir '$out_file.extra_files_path'/db
             --db_version $db_version
             ## --full_verify  https://bitbucket.org/srouxjgi/iphop/issues/114/db-download-verification-error
             --no_prompt &&

diff --git a/data_managers/data_manager_phabox/data_manager/phabox_datamanager.xml b/data_managers/data_manager_phabox/data_manager/phabox_datamanager.xml
@@ -3,7 +3,7 @@
     <macros>
         <token name="@TOOL_VERSION@">2.1.5</token>
         <token name="@DB_VERSION@">2</token>
-        <token name="@VERSION_SUFFIX@">0</token>
+        <token name="@VERSION_SUFFIX@">1</token>
         <token name="@PROFILE@">22.01</token>
     </macros>
     <requirements>
@@ -13,6 +13,8 @@
     <command detect_errors="exit_code"><![CDATA[
         wget https://github.com/KennthShang/PhaBOX/releases/download/v2/phabox_db_v@[email protected] &&
         unzip phabox_db_v@[email protected] &&
+        mkdir -p '$out_file.extra_files_path' &&
+        mv "phabox_db_v@DB_VERSION@" '$out_file.extra_files_path' &&
         cp '$dmjson' '$out_file'
     ]]></command>
     <configfiles>

diff --git a/data_managers/data_manager_vibrant/data_manager/vibrant_datamanager.xml b/data_managers/data_manager_vibrant/data_manager/vibrant_datamanager.xml
@@ -2,7 +2,7 @@
     <description>database builder</description>
     <macros>
         <token name="@TOOL_VERSION@">1.2.1</token>
-        <token name="@VERSION_SUFFIX@">0</token>
+        <token name="@VERSION_SUFFIX@">1</token>
         <token name="@PROFILE@">22.01</token>
     </macros>
     <xrefs>
@@ -13,7 +13,8 @@
     </requirements>
     <command detect_errors="exit_code"><![CDATA[
         set -eo pipefail; 
-        download-db.sh vibrant_db &&
+        mkdir -p '$out_file.extra_files_path'/vibrant_db &&
+        download-db.sh '$out_file.extra_files_path'/vibrant_db &&
         cp '$dmjson' '$out_file'
     ]]></command>
     <configfiles>

diff --git a/data_managers/data_manager_virsorter/.lint_skip b/data_managers/data_manager_virsorter/.lint_skip
@@ -0,0 +1 @@
+InputsMissing
diff --git a/data_managers/data_manager_virsorter/data_manager/virsorter_datamanager.xml b/data_managers/data_manager_virsorter/data_manager/virsorter_datamanager.xml
@@ -3,9 +3,9 @@
     <description>database builder</description>
     <macros>
         <token name="@TOOL_VERSION@">2.2.4</token>  <!-- when updating make sure to double check the db version -->
-        <token name="@VERSION_SUFFIX@">0</token>
+        <token name="@VERSION_SUFFIX@">1</token>
         <token name="@PROFILE@">22.01</token>
-        <!-- zenodo ID and therebut the version of the reference data is hardcoded in the current version virsorter:
+        <!-- zenodo ID and the version of the reference data is hardcoded in the current version virsorter:
              https://github.com/jiarong/VirSorter2/blob/c0dcac53f3373244bf1bd552a7c2104d53802df7/virsorter/rules/setup.smk#L13 
              check and update the DB version when updating the VS version 
              might also change/improve in the future https://github.com/jiarong/VirSorter2/issues/205#issuecomment-2184184977
@@ -18,7 +18,12 @@
         <requirement type="package" version="2.7.0">pulp</requirement> <!-- needs to be pinned, because the old-ish snakemake used in vs2 struggles with https://github.com/snakemake/snakemake/issues/2607 -->
     </requirements>
     <command detect_errors="exit_code"><![CDATA[
-        virsorter setup -d db -j \${GALAXY_SLOTS:-1} --skip-deps-install &&
+        mkdir -p '$out_file.extra_files_path'/db &&
+        #if $test == "true"
+            touch '$out_file.extra_files_path'/db/test &&
+        #else
+            virsorter setup -d '$out_file.extra_files_path'/db -j \${GALAXY_SLOTS:-1} --skip-deps-install &&
+        #end if
         cp '$dmjson' '$out_file'
     ]]></command>
     <configfiles>
@@ -37,22 +42,34 @@
 }]]>
         </configfile>
     </configfiles>
-    <inputs/>
+    <inputs>
+        <param name="test" type="hidden"/>
+    </inputs>
     <outputs>
         <data name="out_file" format="data_manager_json" />
     </outputs>
     <tests>
         <test expect_num_outputs="1">
+            <param name="test" value="true"/>
             <output name="out_file">
                 <assert_contents>
                     <has_text text='"value":"@VIRSORTER_DB_VERSION@"'/>
                     <has_text text='"name":"Version @VIRSORTER_DB_VERSION@"'/>
                 </assert_contents>
             </output>
         </test>
+        <!-- 10GB test data seem to be to much for CI. but this test should be executed locally for updates
+        <test expect_num_outputs="1">
+            <output name="out_file">
+                <assert_contents>
+                    <has_text text='"value":"@VIRSORTER_DB_VERSION@"'/>
+                    <has_text text='"name":"Version @VIRSORTER_DB_VERSION@"'/>
+                </assert_contents>
+            </output>
+        </test> -->
     </tests>
     <help><![CDATA[
-Download and extract virsorter database version @VIRSORTER_DB_VERSION@ from zenodo (https://zenodo.org/records/10594875)
+Download and extract virsorter database version @VIRSORTER_DB_VERSION@ from zenodo (https://zenodo.org/records/4269607)
     ]]></help>
     <citations>
         <citation type="doi">10.1186/s40168-020-00990-y</citation>