From af685106b368f9b6c66146c7a7bba1ff748f2392 Mon Sep 17 00:00:00 2001 From: Alex Valcourt Caron Date: Mon, 2 Dec 2024 09:53:04 -0500 Subject: [PATCH 1/9] Revert "Revert "[ENH] Update subworkflow preproc_t1"" --- subworkflows/nf-neuro/preproc_t1/main.nf | 26 +++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/subworkflows/nf-neuro/preproc_t1/main.nf b/subworkflows/nf-neuro/preproc_t1/main.nf index d3b5f5d3..29552a7e 100644 --- a/subworkflows/nf-neuro/preproc_t1/main.nf +++ b/subworkflows/nf-neuro/preproc_t1/main.nf @@ -43,13 +43,19 @@ workflow PREPROC_T1 { ch_versions = ch_versions.mix(PREPROC_N4.out.versions.first()) // ** Resampling ** // - ch_resampling = PREPROC_N4.out.image.join(ch_ref_resample) + ch_resampling = PREPROC_N4.out.image + .join(ch_ref_resample, remainder: true) + .map{ it[0..1] + [it[2] ?: []] } + IMAGE_RESAMPLE ( ch_resampling ) ch_versions = ch_versions.mix(IMAGE_RESAMPLE.out.versions.first()) // ** Brain extraction ** // if ( params.run_synthbet) { - ch_bet = IMAGE_RESAMPLE.out.image.join(ch_weights) + ch_bet = IMAGE_RESAMPLE.out.image + .join(ch_weights, remainder: true) + .map{ it[0..1] + [it[2] ?: []] } + BETCROP_SYNTHBET ( ch_bet ) ch_versions = ch_versions.mix(BETCROP_SYNTHBET.out.versions.first()) @@ -59,7 +65,10 @@ workflow PREPROC_T1 { } else { - ch_bet = IMAGE_RESAMPLE.out.image.join(ch_template).join(ch_probability_map) + ch_bet = IMAGE_RESAMPLE.out.image + .join(ch_template) + .join(ch_probability_map) + BETCROP_ANTSBET ( ch_bet ) ch_versions = ch_versions.mix(BETCROP_ANTSBET.out.versions.first()) @@ -68,13 +77,16 @@ workflow PREPROC_T1 { mask_bet = BETCROP_ANTSBET.out.mask } - // ** crop image ** // - ch_crop = image_bet.map{it + [[]]} + // ** Crop image ** // + ch_crop = image_bet + .map{ it + [[]] } + BETCROP_CROPVOLUME_T1 ( ch_crop ) ch_versions = ch_versions.mix(BETCROP_CROPVOLUME_T1.out.versions.first()) - // ** crop mask ** // - ch_crop_mask = mask_bet.join(BETCROP_CROPVOLUME_T1.out.bounding_box) + // ** Crop mask ** // + ch_crop_mask = mask_bet + .join(BETCROP_CROPVOLUME_T1.out.bounding_box) BETCROP_CROPVOLUME_MASK ( ch_crop_mask ) ch_versions = ch_versions.mix(BETCROP_CROPVOLUME_MASK.out.versions.first()) From b85eb0e0c6b16b50f5408dc2f0a06fb49c1dbb61 Mon Sep 17 00:00:00 2001 From: AlexVCaron Date: Mon, 2 Dec 2024 15:09:42 +0000 Subject: [PATCH 2/9] first attempt at comments --- subworkflows/nf-neuro/preproc_t1/main.nf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/subworkflows/nf-neuro/preproc_t1/main.nf b/subworkflows/nf-neuro/preproc_t1/main.nf index 29552a7e..ae3eba9f 100644 --- a/subworkflows/nf-neuro/preproc_t1/main.nf +++ b/subworkflows/nf-neuro/preproc_t1/main.nf @@ -25,6 +25,9 @@ workflow PREPROC_T1 { ch_versions = Channel.empty() // ** Denoising ** // + // Input : [ meta, image, mask | [] ] + // - join [ meta, image, mask | null ] + // - map [ meta, image, mask | [] ] ch_nlmeans = ch_image .join(ch_mask_nlmeans, remainder: true) .map{ it[0..1] + [it[2] ?: []] } @@ -33,6 +36,11 @@ workflow PREPROC_T1 { ch_versions = ch_versions.mix(DENOISING_NLMEANS.out.versions.first()) // ** N4 correction ** // + // Input : [ meta, image, reference | [], mask | [] ] + // - join [ meta, image, reference | null ] + // - map [ meta, image, reference | [] ] + // - join [ meta, image, reference | [], mask | null ] + // - map [ meta, image, reference | [], mask | [] ] ch_N4 = DENOISING_NLMEANS.out.image .join(ch_ref_n4, remainder: true) .map{ it[0..1] + [it[2] ?: []] } @@ -43,6 +51,9 @@ workflow PREPROC_T1 { ch_versions = ch_versions.mix(PREPROC_N4.out.versions.first()) // ** Resampling ** // + // Input : [ meta, image, reference | [] ] + // - join [ meta, image, reference | null ] + // - map [ meta, image, reference | [] ] ch_resampling = PREPROC_N4.out.image .join(ch_ref_resample, remainder: true) .map{ it[0..1] + [it[2] ?: []] } @@ -52,6 +63,10 @@ workflow PREPROC_T1 { // ** Brain extraction ** // if ( params.run_synthbet) { + // ** SYNTHBET ** // + // Input : [ meta, image, weights | [] ] + // - join [ meta, image, weights | null ] + // - map [ meta, image, weights | [] ] ch_bet = IMAGE_RESAMPLE.out.image .join(ch_weights, remainder: true) .map{ it[0..1] + [it[2] ?: []] } From a043fcadda14f19f38bdce229ed355421aed987d Mon Sep 17 00:00:00 2001 From: AlexVCaron Date: Mon, 2 Dec 2024 16:42:43 +0000 Subject: [PATCH 3/9] prettify and comment more --- subworkflows/nf-neuro/preproc_t1/main.nf | 68 +++++++++++++----------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/subworkflows/nf-neuro/preproc_t1/main.nf b/subworkflows/nf-neuro/preproc_t1/main.nf index ae3eba9f..dc8b40fe 100644 --- a/subworkflows/nf-neuro/preproc_t1/main.nf +++ b/subworkflows/nf-neuro/preproc_t1/main.nf @@ -12,22 +12,23 @@ params.run_synthbet = false workflow PREPROC_T1 { take: - ch_image // channel: [ val(meta), [ image ] ] - ch_template // channel: [ val(meta), [ template ] ] , optional - ch_probability_map // channel: [ val(meta), [ probability_map, mask, initial_affine ] ] , optional - ch_mask_nlmeans // channel: [ val(meta), [ mask ] ] , optional - ch_ref_n4 // channel: [ val(meta), [ ref, ref_mask ] ] , optional - ch_ref_resample // channel: [ val(meta), [ ref ] ] , optional - ch_weights // channel: [ val(meta), [ weights ] ] , optional + ch_image // channel: [ val(meta), image ] + ch_template // channel: [ val(meta), template ] , optional + ch_probability_map // channel: [ val(meta), probability-map, mask, initial-affine ] , optional + ch_mask_nlmeans // channel: [ val(meta), mask ] , optional + ch_ref_n4 // channel: [ val(meta), ref, ref-mask ] , optional + ch_ref_resample // channel: [ val(meta), ref ] , optional + ch_weights // channel: [ val(meta), weights ] , optional main: ch_versions = Channel.empty() // ** Denoising ** // - // Input : [ meta, image, mask | [] ] - // - join [ meta, image, mask | null ] - // - map [ meta, image, mask | [] ] + // Result : [ meta, image, mask | [] ] + // Steps : + // - join [ meta, image, mask | null ] + // - map [ meta, image, mask | [] ] ch_nlmeans = ch_image .join(ch_mask_nlmeans, remainder: true) .map{ it[0..1] + [it[2] ?: []] } @@ -36,24 +37,26 @@ workflow PREPROC_T1 { ch_versions = ch_versions.mix(DENOISING_NLMEANS.out.versions.first()) // ** N4 correction ** // - // Input : [ meta, image, reference | [], mask | [] ] - // - join [ meta, image, reference | null ] - // - map [ meta, image, reference | [] ] - // - join [ meta, image, reference | [], mask | null ] - // - map [ meta, image, reference | [], mask | [] ] + // Result : [ meta, image, reference | [], mask | [] ] + // Steps : + // - join [ meta, image ] + [ reference, mask ] | [ reference, null ] | [ null ] + // - map [ meta, image, reference | [], mask | [] ] + // - join [ meta, image, reference | [], mask | [], nlmeans-mask | null ] + // - map [ meta, image, reference | [], mask | [] ] ch_N4 = DENOISING_NLMEANS.out.image .join(ch_ref_n4, remainder: true) - .map{ it[0..1] + [it[2] ?: []] } + .map{ it[0..1] + [it[2] ?: [], it[3] ?: []] } .join(ch_mask_nlmeans, remainder: true) - .map{ it[0..2] + [it[3] ?: []] } + .map{ it[0..2] + [it[3] ?: it[4] ?: []] } PREPROC_N4 ( ch_N4 ) ch_versions = ch_versions.mix(PREPROC_N4.out.versions.first()) // ** Resampling ** // - // Input : [ meta, image, reference | [] ] - // - join [ meta, image, reference | null ] - // - map [ meta, image, reference | [] ] + // Result : [ meta, image, reference | [] ] + // Steps : + // - join [ meta, image, reference | null ] + // - map [ meta, image, reference | [] ] ch_resampling = PREPROC_N4.out.image .join(ch_ref_resample, remainder: true) .map{ it[0..1] + [it[2] ?: []] } @@ -64,9 +67,10 @@ workflow PREPROC_T1 { // ** Brain extraction ** // if ( params.run_synthbet) { // ** SYNTHBET ** // - // Input : [ meta, image, weights | [] ] - // - join [ meta, image, weights | null ] - // - map [ meta, image, weights | [] ] + // Result : [ meta, image, weights | [] ] + // Steps : + // - join [ meta, image, weights | null ] + // - map [ meta, image, weights | [] ] ch_bet = IMAGE_RESAMPLE.out.image .join(ch_weights, remainder: true) .map{ it[0..1] + [it[2] ?: []] } @@ -107,13 +111,13 @@ workflow PREPROC_T1 { ch_versions = ch_versions.mix(BETCROP_CROPVOLUME_MASK.out.versions.first()) emit: - image_nlmeans = DENOISING_NLMEANS.out.image // channel: [ val(meta), [ image ] ] - image_N4 = PREPROC_N4.out.image // channel: [ val(meta), [ image ] ] - image_resample = IMAGE_RESAMPLE.out.image // channel: [ val(meta), [ image ] ] - image_bet = image_bet // channel: [ val(meta), [ t1 ] ] - mask_bet = mask_bet // channel: [ val(meta), [ mask ] ] - crop_box = BETCROP_CROPVOLUME_T1.out.bounding_box // channel: [ val(meta), [ bounding_box ] ] - mask_final = BETCROP_CROPVOLUME_MASK.out.image // channel: [ val(meta), [ mask ] ] - t1_final = BETCROP_CROPVOLUME_T1.out.image // channel: [ val(meta), [ image ] ] - versions = ch_versions // channel: [ versions.yml ] + t1_final = BETCROP_CROPVOLUME_T1.out.image // channel: [ val(meta), t1-preprocessed ] + mask_final = BETCROP_CROPVOLUME_MASK.out.image // channel: [ val(meta), t1-mask ] + image_nlmeans = DENOISING_NLMEANS.out.image // channel: [ val(meta), t1-after-denoise ] + image_N4 = PREPROC_N4.out.image // channel: [ val(meta), t1-after-unbias ] + image_resample = IMAGE_RESAMPLE.out.image // channel: [ val(meta), t1-after-resample ] + image_bet = image_bet // channel: [ val(meta), t1-after-bet ] + mask_bet = mask_bet // channel: [ val(meta), intermediary-mask ] + crop_box = BETCROP_CROPVOLUME_T1.out.bounding_box // channel: [ val(meta), bounding-box ] + versions = ch_versions // channel: [ versions.yml ] } From 626524c40142a34bd49be478cdb2701fe265d443 Mon Sep 17 00:00:00 2001 From: AlexVCaron Date: Mon, 2 Dec 2024 16:58:04 +0000 Subject: [PATCH 4/9] update snap --- .../nf-neuro/preproc_t1/tests/main.nf.test.snap | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap index bfdd332a..9d601671 100644 --- a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap +++ b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap @@ -70,8 +70,8 @@ "test__denoised.nii.gz:md5:header,a7ee0e819007aba98d14f7a145b550e6,data,2e21433e2bcd4de2a6b6167c6233cd40", "test__image_n4.nii.gz:md5:header,e7cfbd06624321d70cbd667a77315ba3,data,a81e98f32ed963c098ccb07486101898", "test_resampled.nii.gz:md5:header,7628a07204938d640c3530fa3d76d2b7,data,a81e98f32ed963c098ccb07486101898", - "test__t1_bet.nii.gz:md5:header,84eb3a5699de115c7f32b67d18479f4f,data,f8f5d5715026511032df083e27ee83dd", - "test__t1_bet_mask.nii.gz:md5:header,a8b3a7e58c811c1542529a73d1ab977d,data,a31e513e6fdf0ac788001c7da89e71bd", + "test__t1_bet.nii.gz:md5:header,84eb3a5699de115c7f32b67d18479f4f,data,350351a925b241d9d42a77607d1697b5", + "test__t1_bet_mask.nii.gz:md5:header,a8b3a7e58c811c1542529a73d1ab977d,data,12927a92f6e860429ed05cf9a0a2175e", [ [ { @@ -81,8 +81,8 @@ "test_t1_cropped_bbox.pkl:md5,3ad48fac10852756283b88c31b02557d" ] ], - "test_cropped.nii.gz:md5:header,8b4ac0737584269dd69d3fd81ddcbb48,data,9aac180400c21340109c501ce76e2934", - "test_t1_cropped.nii.gz:md5:header,45268f5e9fe9e298e96be4478871e6e8,data,8bb543d9aeec863298a20e9a56c95e56", + "test_cropped.nii.gz:md5:header,8b4ac0737584269dd69d3fd81ddcbb48,data,a36f9bb4d1bf28638abe4ff8b6fefcf0", + "test_t1_cropped.nii.gz:md5:header,45268f5e9fe9e298e96be4478871e6e8,data,9889db71508130cfa45c3e573d4d0242", [ "versions.yml:md5,19aaa24cf046e49a4ad4ce5d507ea8aa", "versions.yml:md5,37229cd5cee6f6788fbc9083d6618c61", @@ -96,6 +96,6 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-11-28T19:51:06.227182958" + "timestamp": "2024-12-02T16:49:45.574227522" } -} +} \ No newline at end of file From 937e5d81fdb41978e886e08290dfc15193147304 Mon Sep 17 00:00:00 2001 From: AlexVCaron Date: Tue, 3 Dec 2024 04:49:01 +0000 Subject: [PATCH 5/9] address non repro of antsBET in tests and in the meta --- subworkflows/nf-neuro/preproc_t1/meta.yml | 7 +++++-- subworkflows/nf-neuro/preproc_t1/tests/main.nf.test | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/subworkflows/nf-neuro/preproc_t1/meta.yml b/subworkflows/nf-neuro/preproc_t1/meta.yml index 64e5abfb..0714dd77 100644 --- a/subworkflows/nf-neuro/preproc_t1/meta.yml +++ b/subworkflows/nf-neuro/preproc_t1/meta.yml @@ -8,7 +8,9 @@ description: | The resulting T1 is corrected, resampled, brain extracted and cropped. You can retrieve the file after each step if you don't want to run the entire subworkflow. The next steps would be to register the resulting T1-corrected image with the DWI-corrected image - with, for example, the REGISTRATION subworkflow. + with, for example, the REGISTRATION subworkflow. IMPORTANT : the module is only reproducible with + ANTs when BET is run using a single thread. For performance and reproducibility, use the other + available methods. ----------- Steps ----------- Denoising (nlmeans, scil). Used to remove the noise induced by the MRI acquisition, @@ -23,7 +25,8 @@ description: | This spatial resolution is modifiable in the configuration file. Brain Extraction (bet, ANTs, freesurfer). Isolates the brain tissue voxels from the remaining image. Also creates a binary brain mask. - This brain extraction is required for the T1 to DWI Registration. + This brain extraction is required for the T1 to DWI Registration. IMPORTANT : when using ANTs, + brain extraction is reproducible only when run using a single thread. Cropping (scil). Crops the empty planes around the brain to optimize the next processing steps. Subworkflow based on Tractoflow : https://www.sciencedirect.com/science/article/pii/S105381192030375X?via%3Dihub diff --git a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test index ae2b8e30..ea8df21c 100644 --- a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test +++ b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test @@ -88,11 +88,11 @@ nextflow_workflow { niftiMD5SUM(workflow.out.image_nlmeans.get(0).get(1)), niftiMD5SUM(workflow.out.image_N4.get(0).get(1)), niftiMD5SUM(workflow.out.image_resample.get(0).get(1)), - niftiMD5SUM(workflow.out.image_bet.get(0).get(1)), - niftiMD5SUM(workflow.out.mask_bet.get(0).get(1)), + file(workflow.out.image_bet.get(0).get(1)).name, + file(workflow.out.mask_bet.get(0).get(1)).name, workflow.out.crop_box, - niftiMD5SUM(workflow.out.mask_final.get(0).get(1)), - niftiMD5SUM(workflow.out.t1_final.get(0).get(1)), + file(workflow.out.mask_final.get(0).get(1)).name, + file(workflow.out.t1_final.get(0).get(1)).name, workflow.out.versions ).match()} ) From ac93041a31111be9480dfa766451f3e14bbd6d05 Mon Sep 17 00:00:00 2001 From: AlexVCaron Date: Tue, 3 Dec 2024 05:04:09 +0000 Subject: [PATCH 6/9] remove redundant test case and improve meta --- subworkflows/nf-neuro/preproc_t1/main.nf | 2 +- subworkflows/nf-neuro/preproc_t1/meta.yml | 12 ++-- .../nf-neuro/preproc_t1/tests/main.nf.test | 71 +------------------ .../preproc_t1/tests/nextflow_2.config | 16 ----- 4 files changed, 8 insertions(+), 93 deletions(-) delete mode 100644 subworkflows/nf-neuro/preproc_t1/tests/nextflow_2.config diff --git a/subworkflows/nf-neuro/preproc_t1/main.nf b/subworkflows/nf-neuro/preproc_t1/main.nf index dc8b40fe..2b474899 100644 --- a/subworkflows/nf-neuro/preproc_t1/main.nf +++ b/subworkflows/nf-neuro/preproc_t1/main.nf @@ -65,7 +65,7 @@ workflow PREPROC_T1 { ch_versions = ch_versions.mix(IMAGE_RESAMPLE.out.versions.first()) // ** Brain extraction ** // - if ( params.run_synthbet) { + if ( params.run_synthbet ) { // ** SYNTHBET ** // // Result : [ meta, image, weights | [] ] // Steps : diff --git a/subworkflows/nf-neuro/preproc_t1/meta.yml b/subworkflows/nf-neuro/preproc_t1/meta.yml index 0714dd77..c06f6f1f 100644 --- a/subworkflows/nf-neuro/preproc_t1/meta.yml +++ b/subworkflows/nf-neuro/preproc_t1/meta.yml @@ -8,9 +8,8 @@ description: | The resulting T1 is corrected, resampled, brain extracted and cropped. You can retrieve the file after each step if you don't want to run the entire subworkflow. The next steps would be to register the resulting T1-corrected image with the DWI-corrected image - with, for example, the REGISTRATION subworkflow. IMPORTANT : the module is only reproducible with - ANTs when BET is run using a single thread. For performance and reproducibility, use the other - available methods. + with, for example, the REGISTRATION subworkflow. IMPORTANT : the subworkflow is only reproducible + with when running ANTs BET using a single thread. ----------- Steps ----------- Denoising (nlmeans, scil). Used to remove the noise induced by the MRI acquisition, @@ -23,7 +22,7 @@ description: | Resamples the T1 to an isotropic spatial resolution. The default is 1mm, a standard in humans which usually facilitate registration with corrected DWI images. This spatial resolution is modifiable in the configuration file. - Brain Extraction (bet, ANTs, freesurfer). + Brain Extraction (ANTs - default, freesurfer). Isolates the brain tissue voxels from the remaining image. Also creates a binary brain mask. This brain extraction is required for the T1 to DWI Registration. IMPORTANT : when using ANTs, brain extraction is reproducible only when run using a single thread. @@ -58,13 +57,14 @@ input: - ch_template: type: file description: | - The input channel containing the anatomical template to perform BET. + The input channel containing the anatomical template for antsBET. Structure: [ val(meta), path(image) ] pattern: "*.{nii,nii.gz}" - ch_probability_map: type: file description: | - The input channel containing the brain probability mask, with intensity range 1 (definitely brain) to 0 (definitely background). + The input channel containing the brain probability mask for antsBET, + with intensity range 1 (definitely brain) to 0 (definitely background). Structure: [ val(meta), path(image) ] pattern: "*.{nii,nii.gz}" - ch_mask_nlmeans: diff --git a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test index ea8df21c..cd1833d7 100644 --- a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test +++ b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test @@ -29,7 +29,7 @@ nextflow_workflow { } } - test("preproc_t1_classic") { + test("preproc_t1_antsbet") { config "./nextflow.config" when { workflow { @@ -99,75 +99,6 @@ nextflow_workflow { } } - test("preproc_t1_option") { - config "./nextflow_2.config" - - when { - workflow { - """ - ch_split_test_data = LOAD_DATA.out.test_data_directory - .branch{ - antsbet: it.simpleName == "antsbet" - t1w: it.simpleName == "T1w" - } - input[0] = ch_split_test_data.t1w.map{ - test_data_directory -> [ - [ id:'test', single_end:false ], - file("\${test_data_directory}/T1w.nii.gz") - ]} - input[1] = ch_split_test_data.antsbet.map{ - test_data_directory -> [ - [ id:'test', single_end:false ], - [] - ]} - input[2] = ch_split_test_data.antsbet.map{ - test_data_directory -> [ - [ id:'test', single_end:false ], - [] - ]} - input[3] = ch_split_test_data.t1w.map{ - test_data_directory -> [ - [ id:'test', single_end:false ], - file("\${test_data_directory}/T1w_mask.nii.gz") - ]} - input[4] = ch_split_test_data.t1w.map{ - test_data_directory -> [ - [ id:'test', single_end:false ], - [], - [] - ]} - input[5] = ch_split_test_data.t1w.map{ - test_data_directory -> [ - [ id:'test', single_end:false ], - file("\${test_data_directory}/T1w.nii.gz") - ]} - input[6] = ch_split_test_data.antsbet.map{ - test_data_directory -> [ - [ id:'test', single_end:false ], - [] - ]} - """ - } - } - - then { - assertAll( - { assert workflow.success}, - { assert snapshot( - niftiMD5SUM(workflow.out.image_nlmeans.get(0).get(1)), - niftiMD5SUM(workflow.out.image_N4.get(0).get(1)), - niftiMD5SUM(workflow.out.image_resample.get(0).get(1)), - niftiMD5SUM(workflow.out.image_bet.get(0).get(1)), - niftiMD5SUM(workflow.out.mask_bet.get(0).get(1)), - workflow.out.crop_box, - niftiMD5SUM(workflow.out.mask_final.get(0).get(1)), - niftiMD5SUM(workflow.out.t1_final.get(0).get(1)), - workflow.out.versions - ).match()} - ) - } - } - test("preproc_t1_synthbet") { config "./nextflow_synthbet.config" diff --git a/subworkflows/nf-neuro/preproc_t1/tests/nextflow_2.config b/subworkflows/nf-neuro/preproc_t1/tests/nextflow_2.config deleted file mode 100644 index 8342c09b..00000000 --- a/subworkflows/nf-neuro/preproc_t1/tests/nextflow_2.config +++ /dev/null @@ -1,16 +0,0 @@ -process { - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - withName: "IMAGE_RESAMPLE" { - ext.interp = "lin" - } - withName: "BETCROP_CROPVOLUME_T1" { - ext.output_bbox = true - ext.first_suffix = "t1" - } - withName: "BETCROP_SYNTHBET" { - memory = "8G" - ext.nocsf = true - } -} - -params.run_synthbet = true From 7c2d7013d9b09ea1abae127b5ef97cb48a0df583 Mon Sep 17 00:00:00 2001 From: AlexVCaron Date: Tue, 3 Dec 2024 05:27:10 +0000 Subject: [PATCH 7/9] update snap --- .../preproc_t1/tests/main.nf.test.snap | 47 +++---------------- 1 file changed, 7 insertions(+), 40 deletions(-) diff --git a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap index 9d601671..a4343081 100644 --- a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap +++ b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap @@ -32,46 +32,13 @@ }, "timestamp": "2024-11-25T18:23:03.990173398" }, - "preproc_t1_option": { - "content": [ - "test__denoised.nii.gz:md5:header,a7ee0e819007aba98d14f7a145b550e6,data,fdd1def4fbd8cf738b0d01bd228a94aa", - "test__image_n4.nii.gz:md5:header,e7cfbd06624321d70cbd667a77315ba3,data,9578bf09f4b639a6ed354bc79a6ec07a", - "test_resampled.nii.gz:md5:header,7628a07204938d640c3530fa3d76d2b7,data,9578bf09f4b639a6ed354bc79a6ec07a", - "test__bet_image.nii.gz:md5:header,38a09a5addf7c1f13dae1121c562f3b5,data,b191a8da504640f7496ac855af08dbcb", - "test__brain_mask.nii.gz:md5:header,8fc2b4ae979d881623dfd34d377d437d,data,91a73cc5677ae72931f812a27f987a42", - [ - [ - { - "id": "test", - "single_end": false - }, - "test_t1_cropped_bbox.pkl:md5,522d2c44d3ad1058ea77457a263e39c8" - ] - ], - "test_cropped.nii.gz:md5:header,efab188f3700b5b29d4b4ef99cec1295,data,3f64b48b555fd40a3dfc73e6c5ccce5e", - "test_t1_cropped.nii.gz:md5:header,efab188f3700b5b29d4b4ef99cec1295,data,f3af6e0383ac738415c578c42969e6a9", - [ - "versions.yml:md5,318cabe934be45528a25f52083d9c90d", - "versions.yml:md5,37229cd5cee6f6788fbc9083d6618c61", - "versions.yml:md5,657fbb224f260392e573b8511c4b798d", - "versions.yml:md5,b979132991d8f72a3585465533bd5730", - "versions.yml:md5,bdd934b4b8456060c36d6d97e4f30740", - "versions.yml:md5,ea32c30f5320f720b2f5dc32ac2535ea" - ] - ], - "meta": { - "nf-test": "0.9.0", - "nextflow": "24.10.0" - }, - "timestamp": "2024-11-25T18:22:27.118657583" - }, - "preproc_t1_classic": { + "preproc_t1_antsbet": { "content": [ "test__denoised.nii.gz:md5:header,a7ee0e819007aba98d14f7a145b550e6,data,2e21433e2bcd4de2a6b6167c6233cd40", "test__image_n4.nii.gz:md5:header,e7cfbd06624321d70cbd667a77315ba3,data,a81e98f32ed963c098ccb07486101898", "test_resampled.nii.gz:md5:header,7628a07204938d640c3530fa3d76d2b7,data,a81e98f32ed963c098ccb07486101898", - "test__t1_bet.nii.gz:md5:header,84eb3a5699de115c7f32b67d18479f4f,data,350351a925b241d9d42a77607d1697b5", - "test__t1_bet_mask.nii.gz:md5:header,a8b3a7e58c811c1542529a73d1ab977d,data,12927a92f6e860429ed05cf9a0a2175e", + "test__t1_bet.nii.gz", + "test__t1_bet_mask.nii.gz", [ [ { @@ -81,8 +48,8 @@ "test_t1_cropped_bbox.pkl:md5,3ad48fac10852756283b88c31b02557d" ] ], - "test_cropped.nii.gz:md5:header,8b4ac0737584269dd69d3fd81ddcbb48,data,a36f9bb4d1bf28638abe4ff8b6fefcf0", - "test_t1_cropped.nii.gz:md5:header,45268f5e9fe9e298e96be4478871e6e8,data,9889db71508130cfa45c3e573d4d0242", + "test_cropped.nii.gz", + "test_t1_cropped.nii.gz", [ "versions.yml:md5,19aaa24cf046e49a4ad4ce5d507ea8aa", "versions.yml:md5,37229cd5cee6f6788fbc9083d6618c61", @@ -96,6 +63,6 @@ "nf-test": "0.9.0", "nextflow": "24.04.4" }, - "timestamp": "2024-12-02T16:49:45.574227522" + "timestamp": "2024-12-03T05:08:46.767747205" } -} \ No newline at end of file +} From e373a0b34ed9ee306b91a618a4dc75df7c226cc3 Mon Sep 17 00:00:00 2001 From: Alex Valcourt Caron Date: Wed, 4 Dec 2024 11:03:10 -0500 Subject: [PATCH 8/9] add back check on template and proba map for ants bet, gives a better error message --- subworkflows/nf-neuro/preproc_t1/main.nf | 8 ++- .../nf-neuro/preproc_t1/tests/main.nf.test | 49 ++++++++++++++++++- .../preproc_t1/tests/main.nf.test.snap | 18 ++----- 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/subworkflows/nf-neuro/preproc_t1/main.nf b/subworkflows/nf-neuro/preproc_t1/main.nf index 2b474899..b923495e 100644 --- a/subworkflows/nf-neuro/preproc_t1/main.nf +++ b/subworkflows/nf-neuro/preproc_t1/main.nf @@ -84,9 +84,13 @@ workflow PREPROC_T1 { } else { + // ** ANTSBET ** // + // The template and probability maps are mandatory if running antsBET. Since the + // error message from nextflow when they are absent is either non-informative or + // missing, we use ifEmpty to provide a more informative one. ch_bet = IMAGE_RESAMPLE.out.image - .join(ch_template) - .join(ch_probability_map) + .join(ch_template.ifEmpty{ error("ANTS BET needs a template") }) + .join(ch_probability_map.ifEmpty{ error("ANTS BET needs a tissue probability map") }) BETCROP_ANTSBET ( ch_bet ) ch_versions = ch_versions.mix(BETCROP_ANTSBET.out.versions.first()) diff --git a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test index cd1833d7..44744117 100644 --- a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test +++ b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test @@ -29,6 +29,53 @@ nextflow_workflow { } } + test("preproc_t1_antsbet_error") { + config "./nextflow.config" + when { + workflow { + """ + ch_split_test_data = LOAD_DATA.out.test_data_directory + .branch{ + antsbet: it.simpleName == "antsbet" + t1w: it.simpleName == "T1w" + } + input[0] = ch_split_test_data.t1w.map{ + test_data_directory -> [ + [ id:'test', single_end:false ], + file("\${test_data_directory}/T1w.nii.gz") + ]} + input[1] = Channel.empty() + input[2] = Channel.empty() + input[3] = ch_split_test_data.t1w.map{ + test_data_directory -> [ + [ id:'test', single_end:false ], + [] + ]} + input[4] = ch_split_test_data.t1w.map{ + test_data_directory -> [ + [ id:'test', single_end:false ], + [], + [] + ]} + input[5] = ch_split_test_data.t1w.map{ + test_data_directory -> [ + [ id:'test', single_end:false ], + [] + ]} + input[6] = ch_split_test_data.antsbet.map{ + test_data_directory -> [ + [ id:'test', single_end:false ], + [] + ]} + """ + } + } + + then { + assert workflow.failed + } + } + test("preproc_t1_antsbet") { config "./nextflow.config" when { @@ -90,7 +137,7 @@ nextflow_workflow { niftiMD5SUM(workflow.out.image_resample.get(0).get(1)), file(workflow.out.image_bet.get(0).get(1)).name, file(workflow.out.mask_bet.get(0).get(1)).name, - workflow.out.crop_box, + file(workflow.out.crop_box.get(0).get(1)).name, file(workflow.out.mask_final.get(0).get(1)).name, file(workflow.out.t1_final.get(0).get(1)).name, workflow.out.versions diff --git a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap index a4343081..7f595ab8 100644 --- a/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap +++ b/subworkflows/nf-neuro/preproc_t1/tests/main.nf.test.snap @@ -39,15 +39,7 @@ "test_resampled.nii.gz:md5:header,7628a07204938d640c3530fa3d76d2b7,data,a81e98f32ed963c098ccb07486101898", "test__t1_bet.nii.gz", "test__t1_bet_mask.nii.gz", - [ - [ - { - "id": "test", - "single_end": false - }, - "test_t1_cropped_bbox.pkl:md5,3ad48fac10852756283b88c31b02557d" - ] - ], + "test_t1_cropped_bbox.pkl", "test_cropped.nii.gz", "test_t1_cropped.nii.gz", [ @@ -60,9 +52,9 @@ ] ], "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.04.3" }, - "timestamp": "2024-12-03T05:08:46.767747205" + "timestamp": "2024-12-04T10:38:12.185789" } -} +} \ No newline at end of file From 22a2a942c2c33c61783f3d25504ea1ef497fcfc7 Mon Sep 17 00:00:00 2001 From: Alex Valcourt Caron Date: Wed, 4 Dec 2024 11:03:47 -0500 Subject: [PATCH 9/9] fix load_test_data, removing incomplete downloads from the test data cache --- subworkflows/nf-neuro/load_test_data/main.nf | 35 +++++++++++++------- tests/nextflow.config | 4 +-- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/subworkflows/nf-neuro/load_test_data/main.nf b/subworkflows/nf-neuro/load_test_data/main.nf index a361313e..612c68a7 100644 --- a/subworkflows/nf-neuro/load_test_data/main.nf +++ b/subworkflows/nf-neuro/load_test_data/main.nf @@ -18,25 +18,36 @@ def fetch_archive ( name, destination, remote, database, data_identifiers ) { def cache_entry = file("$cache_location/$data_id") if ( !cache_entry.exists() ) { - def remote_entry = "${data_id[0..1]}/${data_id[2..-1]}" - file("$remote/$database/$remote_entry").copyTo(cache_entry) + try { + def remote_entry = "${data_id[0..1]}/${data_id[2..-1]}" + file("$remote/$database/$remote_entry").copyTo(cache_entry) + } + catch (Exception e) { + error "Failed to fetch test data archive: $name" + file("$remote/$database/$remote_entry").delete() + } } // Unzip all archive content to destination def content = new java.util.zip.ZipFile("$cache_entry") - content.entries().each{ entry -> - def local_target = file("$destination/${entry.getName()}") - if (entry.isDirectory()) { - local_target.mkdirs(); - } else { - local_target.getParent().mkdirs(); - file("$local_target").withOutputStream{ - out -> out << content.getInputStream(entry) + try { + content.entries().each{ entry -> + def local_target = file("$destination/${entry.getName()}") + if (entry.isDirectory()) { + local_target.mkdirs(); + } else { + local_target.getParent().mkdirs(); + file("$local_target").withOutputStream{ + out -> out << content.getInputStream(entry) + } } } - } - return destination.resolve("${name.take(name.lastIndexOf('.'))}") + return destination.resolve("${name.take(name.lastIndexOf('.'))}") + } + finally { + content.close() + } } workflow LOAD_TEST_DATA { diff --git a/tests/nextflow.config b/tests/nextflow.config index 1a6f6664..c7b2182a 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -65,9 +65,9 @@ def query_container_limits (type) { if (memory_limit) { return "${memory_limit}.GB" } else { - def sysmem = ( + def sysmem = (( (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean() - ).getTotalPhysicalMemorySize() / 1073741824 + ).getTotalPhysicalMemorySize() / 1073741824).toInteger() return "${sysmem}.GB" } } catch (all) {