PR comments on mata.json

MHubAI · Nov 30, 2023 · 4abafd5 · 4abafd5
1 parent 0b31b2f
commit 4abafd5
Show file tree

Hide file tree

Showing 4 changed files with 83 additions and 43 deletions.
diff --git a/models/gc_picai_baseline/config/default.yml b/models/gc_picai_baseline/config/default.yml
@@ -24,8 +24,8 @@ modules:
   ReportExporter:
     format: compact
     includes:
-      - data: prostate_cancer_probability
-        label: prostate_cancer_probability
+      - data: prostate_cancer_likelihood
+        label: prostate_cancer_likelihood
         value: value
 
   DataOrganizer:

diff --git a/models/gc_picai_baseline/config/mha-pipeline.yml b/models/gc_picai_baseline/config/mha-pipeline.yml
@@ -21,8 +21,8 @@ modules:
   ReportExporter:
     format: compact
     includes:
-      - data: prostate_cancer_probability
-        label: prostate_cancer_probability
+      - data: prostate_cancer_likelihood
+        label: prostate_cancer_likelihood
         value: value
 
   DataOrganizer:

diff --git a/models/gc_picai_baseline/meta.json b/models/gc_picai_baseline/meta.json
@@ -3,11 +3,31 @@
   "name": "picai_baseline",
   "title": "PI-CAI challenge baseline",
   "summary": {
-    "description": "The PI-CAI challenge is to validate modern AI algorithms at clinically significant prostate cancer (csPCa) detection and diagnosis. This model algorithm provides the baseline for the challenge.",
+    "description": "This algorithm predicts a detection map for the likelihood of clinically significant prostate cancer (csPCa) using biparametric MRI (bpMRI). The algorithm ensembles 5-fold cross-validation models that were trained on the PI-CAI: Public Training and Development Dataset v2.0. The detection map is at the same spatial resolution and physical dimensions as the input axial T2-weighted image.",
     "inputs": [
       {
-        "label": "Prostate biparametric MRI",
-        "description": "Prostate biparametric MRI exam",
+        "label": "Transverse T2-weighted Prostate biparametric MRI",
+        "description": " Transverse T2-weighted Prostate biparametric MRI exam",
+        "format": "DICOM",
+        "modality": "MR",
+        "bodypartexamined": "Prostate",
+        "slicethickness": "",
+        "non-contrast": false,
+        "contrast": false
+      },
+      {
+        "label": "High b-value diffusion-weighted maps",
+        "description": "High b-value diffusion-weighted maps, with b-value of 1400 or 2000, either acquired or vendor-calculated.",
+        "format": "DICOM",
+        "modality": "MR",
+        "bodypartexamined": "Prostate",
+        "slicethickness": "",
+        "non-contrast": false,
+        "contrast": false
+      },
+      {
+        "label": "ADC map",
+        "description": "ADC map",
         "format": "DICOM",
         "modality": "MR",
         "bodypartexamined": "Prostate",
@@ -19,30 +39,30 @@
     "outputs": [
       {
         "type": "Prediction",
-        "valueType": "Probability",
-        "label": "Prostate cancer probability",
+        "valueType": "Likelihood",
+        "label": "Prostate cancer likelihood",
         "description": "Case-level likelihood of harboring clinically significant prostate cancer, in range [0,1]",
         "classes": []
       },
       {
         "type": "Prediction",
-        "valueType": "Probability map",
+        "valueType": "Likelihood map",
         "label": "Transverse cancer detection map",
         "description": "Detection map of clinically significant prostate cancer lesions in 3D, where each voxel represents a floating point in range [0,1]",
         "classes": []
       }
     ],
     "model": {
       "architecture": "3d fullres nnUNet",
-      "training": "supervised",
+      "training": "semi-supervised",
       "cmpapproach": "3D"
     },
     "data": {
       "training": {
-        "vol_samples": 1200
+        "vol_samples": 1500
       },
       "evaluation": {
-        "vol_samples": 300
+        "vol_samples": 1000
       },
       "public": false,
       "external": false
@@ -61,9 +81,18 @@
     "cite": "",
     "license": {
       "code": "Apache 2.0",
-      "weights": "Apache 2.0"
+      "weights": "CC-BY-NC-4.0"
     },
-    "publications": [],
+    "publications": [
+      {
+        "uri": "https://doi.org/10.5281/zenodo.6667655",
+        "title": "Artificial Intelligence and Radiologists at Prostate Cancer Detection in MRI: The PI-CAI Challenge (Study Protocol)"
+      },
+      {
+        "uri": "https://pubs.rsna.org/doi/10.1148/ryai.230031",
+        "title": "Semisupervised Learning with Report-guided Pseudo Labels for Deep Learning–based Prostate Cancer Detection Using Biparametric MRI"
+      }
+    ],
     "github": "https://github.com/DIAGNijmegen/picai_nnunet_semi_supervised_gc_algorithm",
     "zenodo": "",
     "colab": "",
@@ -72,7 +101,7 @@
   "info": {
     "use": {
       "title": "Intended use",
-      "text": "Prediction of the likelihood of harboring clinically significant prostate cancer (csPCa) in prostate biparametric MRI exams.",
+      "text": "This algorithm is a deep learning-based detection/diagnosis model, which ensembles 5 independent nnU-Net models (5-fold cross-validation). To predict the likelihood of harboring clinically significant prostate cancer (csPCa), the transversal T2-weighted, apparent diffusion coefficient (ADC) and high b-value diffusion maps are required. The input sequences should be co-registered or aligned reasonably well and the prostate gland should be localized within a volume of 460 cm³ from the centre coordinate. To train these models, a total of 1500 prostate biparametric MRI (bpMRI) scans paired with human-annotated or AI-derived ISUP ≥ 2 delineations were used.",
       "references": [
         {
           "label": "PI-CAI baseline algorithm on grand-challenge",
@@ -83,18 +112,30 @@
     },
     "analyses": {
       "title": "Evaluation",
-      "text": "Patient-level diagnosis performance is evaluated using the Area Under Receiver Operating Characteristic (AUROC) metric. Lesion-level detection performance is evaluated using the Average Precision (AP) metric. Overall score used to rank each AI algorithm is the average of both task-specific metrics: Overall Ranking Score = (AP + AUROC) / 2",
+      "text": "Patient-level diagnosis performance is evaluated using the Area Under Receiver Operating Characteristic (AUROC) metric. Lesion-level detection performance is evaluated using the Average Precision (AP) metric.",
       "references": [
         {
           "label": "PI-CAI AI challenge details",
           "uri": "https://pi-cai.grand-challenge.org/AI/"
+        },
+        {
+          "label": "PI-CAI baseline algorithm evaluation results on grand-challenge.",
+          "uri": "https://pi-cai.grand-challenge.org/evaluation/fe187cdb-cb61-4cbb-ab63-2de483a52d60/"
         }
       ],
-      "tables": []
+      "tables": [
+        {
+          "label": "Evaluation results on the PI-CAI testing cohort of 1000 cases.",
+          "entries": {
+              "AUROC": "0.865",
+              "AP": "0.576"
+          }
+        }
+      ]
     },
     "evaluation": {
       "title": "Evaluation data",
-      "text": "The test sets are two private cohorts of 100 and 1000 biparametric MRI exams respectively. The first was used to tune the algorithms in a public leaderboard, the second was used to determine the top 5 AI algorithms.",
+      "text": "The PI-CAI Hidden Testing Cohort (1000 cases) includes internal testing data (unseen cases from seen centers) and external testing data (unseen cases from an unseen center).",
       "references": [
         {
           "label": "PI-CAI data section",
@@ -105,19 +146,19 @@
     },
     "training": {
       "title": "Training data",
-      "text": "For the PI-CAI a publicly available training datasets of 1500 biparametric MRI exams including 328 cases from the ProstateX challenge were made available.",
+      "text": "The publicly available PI-CAI training and development dataset of 1500 biparametric MRI exams was used for training [1]. AI-derived annotations were created for cases without manual annotations [2]. This model was trained using a custom preprocessing step followed by the standard nnU-Net pipeline. The default nnU-Net loss-function was changed to Cross-Entropy + Focal loss. [3]",
       "references": [
         {
-          "label": "PI-CAI publicly available training data",
+          "label": "PI-CAI publicly available training and development dataset",
           "uri": "https://zenodo.org/record/6624726"
         },
         {
-          "label": "PI-CAI publicly available training data annotations",
-          "uri": "https://github.com/DIAGNijmegen/picai_labels"
+          "label": "Method to obtain AI-derived annotations",
+          "uri": "https://fastmri.eu/research/bosma22a.html"
         },
         {
-          "label": "ProstateX challenge",
-          "uri": "https://prostatex.grand-challenge.org/"
+          "label": "Detailed description of training method",
+          "uri": "https://github.com/DIAGNijmegen/picai_baseline/blob/main/nnunet_baseline.md"
         }
       ],
       "tables": []

diff --git a/models/gc_picai_baseline/utils/PicaiBaselineRunner.py b/models/gc_picai_baseline/utils/PicaiBaselineRunner.py
@@ -18,12 +18,11 @@
 from process import csPCaAlgorithm as PicaiClassifier
 
 
-@ValueOutput.Name('prostate_cancer_probability')
-@ValueOutput.Meta(Meta(key="value"))
-@ValueOutput.Label('ProstateCancerProbability')
+@ValueOutput.Name('prostate_cancer_likelihood')
+@ValueOutput.Label('ProstateCancerLikelihood')
 @ValueOutput.Type(float)
-@ValueOutput.Description('Probability of case-level prostate cancer.')
-class ProstateCancerProbability(ValueOutput):
+@ValueOutput.Description('Likelihood of case-level prostate cancer.')
+class ProstateCancerLikelihood(ValueOutput):
     pass
 
 
@@ -33,10 +32,10 @@ class PicaiBaselineRunner(Module):
     @IO.Input('in_data_t2', 'mha:mod=mr:type=t2w', the='input T2 weighted prostate MR image')
     @IO.Input('in_data_adc', 'mha:mod=mr:type=adc', the='input ADC prostate MR image')
     @IO.Input('in_data_hbv', 'mha:mod=mr:type=hbv', the='input HBV prostate MR image')
-    @IO.Output('cancer_probability_json', 'cspca-case-level-likelihood.json', "json", bundle='model', the='output JSON file with PICAI baseline prostate cancer probability')
-    @IO.Output('cancer_detection_heatmap', 'cspca_detection_map.mha', "mha:mod=hm", bundle='model', the='output heatmap indicating prostate cancer probability')
-    @IO.OutputData('cancer_probability', ProstateCancerProbability, the='PICAI baseline prostate cancer probability')
-    def task(self, instance: Instance, in_data_t2: InstanceData, in_data_adc: InstanceData, in_data_hbv: InstanceData, cancer_probability_json: InstanceData, cancer_detection_heatmap: InstanceData, cancer_probability: ProstateCancerProbability) -> None:
+    @IO.Output('cancer_likelihood_json', 'cspca-case-level-likelihood.json', "json", bundle='model', the='output JSON file with PICAI baseline prostate cancer likelihood')
+    @IO.Output('cancer_detection_heatmap', 'cspca_detection_map.mha', "mha:mod=hm", bundle='model', the='output heatmap indicating prostate cancer likelihood')
+    @IO.OutputData('cancer_likelihood', ProstateCancerLikelihood, the='PICAI baseline prostate cancer likelihood')
+    def task(self, instance: Instance, in_data_t2: InstanceData, in_data_adc: InstanceData, in_data_hbv: InstanceData, cancer_likelihood_json: InstanceData, cancer_detection_heatmap: InstanceData, cancer_likelihood: ProstateCancerLikelihood) -> None:
         # Initialize classifier object
         classifier = PicaiClassifier()
 
@@ -49,20 +48,20 @@ def task(self, instance: Instance, in_data_t2: InstanceData, in_data_adc: Instan
 
         # Specify output files
         classifier.cspca_detection_map_path = Path(cancer_detection_heatmap.abspath)
-        classifier.case_confidence_path = Path(cancer_probability_json.abspath)
+        classifier.case_confidence_path = Path(cancer_likelihood_json.abspath)
 
         # Run the classifier on the input images
         classifier.process()
 
-        # Extract cancer probability value from cancer_probability_file
-        if not Path(cancer_probability_json.abspath).is_file():
-            raise FileNotFoundError(f"Output file {cancer_probability_json.abspath} could not be found!")
+        # Extract cancer likelihood value from cancer_likelihood_file
+        if not Path(cancer_likelihood_json.abspath).is_file():
+            raise FileNotFoundError(f"Output file {cancer_likelihood_json.abspath} could not be found!")
 
-        with open(cancer_probability_json.abspath, "r") as f:
-            cancer_prob = float(json.load(f))
+        with open(cancer_likelihood_json.abspath, "r") as f:
+            cancer_lh = float(json.load(f))
 
-        if not (isinstance(cancer_prob, (float, int)) and (0.0 <= cancer_prob <= 1.0)):
-            raise ValueError(f"Cancer probability value should be a probability value, found: {cancer_prob}")
+        if not (isinstance(cancer_lh, (float, int)) and (0.0 <= cancer_lh <= 1.0)):
+            raise ValueError(f"Cancer likelihood value should be between 0 and 1, found: {cancer_lh}")
 
         # Output the predicted values
-        cancer_probability.value = cancer_prob
+        cancer_likelihood.value = cancer_lh