[WIP] Domain adaptation

icu_benchmarks/run.py

@@ -22,6 +22,7 @@
     setup_logging,
 )
 from icu_benchmarks.contants import RunMode
+from icu_benchmarks.models import domain_adaptation
 
 
 @gin.configurable("Run")
@@ -116,6 +117,15 @@ def main(my_args=tuple(sys.argv[1:])):
         run_dir = create_run_dir(log_dir)
         source_dir = args.source_dir
         gin.parse_config_file(source_dir / "train_config.gin")
+    if args.command == "da":
+        gin_config_files = (
+            [Path(f"configs/experiments/{args.experiment}.gin")]
+            if args.experiment
+            else [Path(f"configs/models/{model}.gin"), Path(f"configs/tasks/{task}.gin")]
+        )
+        gin.parse_config_files_and_bindings(gin_config_files, args.gin_bindings, finalize_config=False)
+        domain_adaptation(name, args.data_dir, args.log_dir, args.seed, args.task_name, model, debug=args.debug)
+        return
     else:
         # Train
         checkpoint = log_dir / args.checkpoint if args.checkpoint else None

icu_benchmarks/run_utils.py

@@ -90,6 +90,9 @@ def build_parser() -> ArgumentParser:
     evaluate.add_argument("-sn", "--source-name", required=True, type=Path, help="Name of the source dataset.")
     evaluate.add_argument("--source-dir", required=True, type=Path, help="Directory containing gin and model weights.")
 
+    # DOMAIN ADAPTATION ARGUMENTS
+    prep_and_train = subparsers.add_parser("da", help="Run DA experiment.", parents=[parent_parser])
+
     return parser
 
 

scripts/results/da_results_to_latex.py

@@ -0,0 +1,69 @@
+import csv
+
+rawNamesMap = {
+  "target": "Target",
+  "aumc": "AUMCdb",
+  "eicu": "eICU",
+  "hirid": "HiRID",
+  "miiv": "MIMIC-IV",
+  "convex_combination_without_target": "Convex UDA",
+  "max_prediction": "Max Pooling",
+  "target_weight_0.5": "Weighted $\\alpha=1/3$",
+  "target_weight_2": "Weighted $\\alpha=2/3$",
+  "loss_weighted": "Weighted Loss",
+  "bayes_opt": "Weighted Bayes",
+  "target_with_predictions": "Prediction-Feature",
+  "cc_with_preds": "Combined",
+}
+
+def csv_to_dict(file_name):
+    with open(file_name, 'r') as file:
+        reader = csv.DictReader(file)
+        data = [row for row in reader]
+    tables = {}
+    for row in data:
+        row_without_target = {key: value for key, value in row.items() if key != 'target' and key != 'target_size' and key != 'model'}
+        tables.setdefault((row['target'], row['target_size']), {})[row['model']] = row_without_target
+    return tables
+
+
+def dict_to_latex(combination, data, metric):
+    table = '\\begin{table}[h]\n'
+    table += '\\centering\n'
+    table += '\\footnotesize'
+    table += '\\caption{{Sepsis prediction on {0} with target size {1}, {2} with standard deviation.}}\n'.format(rawNamesMap[combination[0]], combination[1], "AUROC" if metric == "auc" else "AUPRC")
+    headers = ['Model']
+    for model, scores in data.items():
+        headers += [model]
+
+    table += '\\begin{tabular}{l|' + ''.join(['c'] * (len(headers) - 1)) + '}\n'
+    table += '\\textbf{' + '} & \\textbf{'.join(headers) + '}\\\\\n'
+    table += '\\hline\n'
+
+    for score_name, score in data[model].items():
+        if "_avg" in score_name:
+            raw_name = score_name.split("_avg")[0]
+            if raw_name == combination[0] or not raw_name in rawNamesMap:
+                continue
+            clean_name = rawNamesMap[raw_name]
+            values = [clean_name]
+            for model in headers[1:]:
+                scores = data[model]
+                avg = "{:.2f}".format(float(scores[score_name]))
+                std = "{:.2f}".format(float(scores[f"{raw_name}_std"]))
+                values.append(f"${avg} \pm {std}$")
+            table += ' & '.join(values) + '\\\\\n'
+
+    table += '\\end{tabular}\n'
+    table += '\\end{table}\n'
+    return table
+
+if __name__ == '__main__':
+    for metric in ["auc", "pr"]:
+        file_name = f'../yaib_logs/sep_{metric}.csv'
+        data = csv_to_dict(file_name)
+        for key, row in data.items():
+            table = dict_to_latex(key, row, metric)
+            print(table)
+        print('\n' * 5)
+

scripts/results/da_results_to_latex_sep.py

@@ -0,0 +1,69 @@
+import csv
+
+rawNamesMap = {
+  "target": "Target",
+  "aumc": "AUMCdb",
+  "eicu": "eICU",
+  "hirid": "HiRID",
+  "miiv": "MIMIC-IV",
+  "convex_combination_without_target": "Convex UDA",
+  "max_prediction": "Max Pooling",
+  "target_weight_0.5": "Weighted $\\alpha=1/3$",
+  "target_weight_2": "Weighted $\\alpha=2/3$",
+  "loss_weighted": "Weighted Loss",
+  "bayes_opt": "Weighted Bayes",
+  "target_with_predictions": "Prediction-Feature",
+  "cc_with_preds": "Combined",
+}
+
+def csv_to_dict(file_name):
+    with open(file_name, 'r') as file:
+        reader = csv.DictReader(file)
+        data = [row for row in reader]
+    tables = {}
+    for row in data:
+        row_without_target = {key: value for key, value in row.items() if key != 'target' and key != 'target_size' and key != 'model'}
+        tables.setdefault((row['target']), {})[row['target_size']] = row_without_target
+    return tables
+
+
+def dict_to_latex(combination, data, metric):
+    table = '\\begin{table}[h]\n'
+    table += '\\centering\n'
+    table += '\\footnotesize'
+    table += '\\caption{{Sepsis prediction on {0} with LGBM, {1} with standard deviation.}}\n'.format(rawNamesMap[combination], "AUROC" if metric == "auc" else "AUPRC")
+    headers = ['Target Size']
+    for target_size, scores in data.items():
+        headers += [target_size]
+
+    table += '\\begin{tabular}{l|' + ''.join(['c'] * (len(headers) - 1)) + '}\n'
+    table += '\\textbf{' + '} & \\textbf{'.join(headers) + '}\\\\\n'
+    table += '\\hline\n'
+
+    for score_name, score in data[target_size].items():
+        if "_avg" in score_name:
+            raw_name = score_name.split("_avg")[0]
+            if raw_name == combination[0] or not raw_name in rawNamesMap:
+                continue
+            clean_name = rawNamesMap[raw_name]
+            values = [clean_name]
+            for target_size in headers[1:]:
+                scores = data[target_size]
+                avg = "{:.2f}".format(float(scores[score_name]))
+                std = "{:.2f}".format(float(scores[f"{raw_name}_std"]))
+                values.append(f"${avg} \pm {std}$")
+            table += ' & '.join(values) + '\\\\\n'
+
+    table += '\\end{tabular}\n'
+    table += '\\end{table}\n'
+    return table
+
+if __name__ == '__main__':
+    for metric in ["auc", "pr"]:
+        file_name = f'../yaib_logs/sep_{metric}.csv'
+        data = csv_to_dict(file_name)
+        for key, row in data.items():
+            table = dict_to_latex(key, row, metric)
+            print(table)
+        print('\n' * 5)
+

scripts/results/mortality_to_csv.py

@@ -0,0 +1,53 @@
+import json
+from pathlib import Path
+import csv
+
+models_dir = Path("../DA_new")
+for metric in ["AUC", "PR"]:
+    for endpoint in models_dir.iterdir():
+        if endpoint.is_dir():
+            with open(models_dir / f"{endpoint.name}_{metric}_results.csv", "w") as csv_file:
+                writer = csv.writer(csv_file)
+                info = ["model", "target", "target_size"]
+                source_names = [
+                    "target",
+                    "aumc",
+                    "eicu",
+                    "hirid",
+                    "miiv",
+                    "convex_combination_without_target",
+                    "max_prediction", 
+                    "target_weight_0.5",
+                    "target_weight_1",
+                    "target_weight_2",
+                    "loss_weighted",
+                    "squared_loss_weighted",
+                    "bayes_opt",
+                    "target_with_predictions",
+                    "cc_with_preds",
+                ]
+                stats_basis = ["avg", "std"]
+                stats = ["avg", "std"]
+                # combine fieldnames and stats
+                full_fields = [f"{source}_{stat}" for source in source_names for stat in stats]
+                writer = csv.DictWriter(csv_file, fieldnames=info + full_fields)
+
+                writer.writeheader()
+                for model in endpoint.iterdir():
+                    for target in ["aumc", "eicu", "hirid", "miiv"]:
+                        target_sizes = [500, 1000, 2000]
+                        for target_size in target_sizes:
+                            target_str = f"target_{target_size}"
+                            if (model / target / target_str).exists():
+                                with open(model / target / target_str / "averaged_source_metrics.json", "r") as f:
+                                    results = json.load(f)
+
+                                    row_data = {"model": model.name, "target": target, "target_size": target_size}
+                                    for stat in stats_basis:
+                                        for source, source_metrics in results.items():
+                                            if stat == "CI_0.95":
+                                                row_data[f"{source}_{stat}_min"] = source_metrics[metric][0][stat][0] * 100
+                                                row_data[f"{source}_{stat}_max"] = source_metrics[metric][0][stat][1] * 100
+                                            else:
+                                                row_data[f"{source}_{stat}"] = source_metrics[metric][0][stat] * 100
+                                    writer.writerow(row_data)

scripts/results/sepsis_to_csv.py

@@ -0,0 +1,52 @@
+import json
+from pathlib import Path
+import csv
+
+models_dir = Path("../DA_sep_new")
+for metric in ["AUC", "PR"]:
+    for endpoint in models_dir.iterdir():
+        if endpoint.is_dir():
+            with open(models_dir / f"{endpoint.name}_{metric}_results.csv", "w") as csv_file:
+                writer = csv.writer(csv_file)
+                info = ["model", "target", "target_size"]
+                source_names = [
+                    "target",
+                    "aumc",
+                    "hirid",
+                    "miiv",
+                    "convex_combination_without_target",
+                    "max_prediction", 
+                    "target_weight_0.5",
+                    "target_weight_1",
+                    "target_weight_2",
+                    "loss_weighted",
+                    "squared_loss_weighted",
+                    "bayes_opt",
+                    "target_with_predictions",
+                    "cc_with_preds",
+                ]
+                stats_basis = ["avg", "std"]
+                stats = ["avg", "std"]
+                # combine fieldnames and stats
+                full_fields = [f"{source}_{stat}" for source in source_names for stat in stats]
+                writer = csv.DictWriter(csv_file, fieldnames=info + full_fields)
+
+                writer.writeheader()
+                for model in endpoint.iterdir():
+                    for target in ["aumc", "hirid", "miiv"]:
+                        target_sizes = [500, 1000, 2000]
+                        for target_size in target_sizes:
+                            target_str = f"target_{target_size}"
+                            if (model / target / target_str).exists():
+                                with open(model / target / target_str / "averaged_source_metrics.json", "r") as f:
+                                    results = json.load(f)
+
+                                    row_data = {"model": model.name, "target": target, "target_size": target_size}
+                                    for stat in stats_basis:
+                                        for source, source_metrics in results.items():
+                                            if stat == "CI_0.95":
+                                                row_data[f"{source}_{stat}_min"] = source_metrics[metric][0][stat][0] * 100
+                                                row_data[f"{source}_{stat}_max"] = source_metrics[metric][0][stat][1] * 100
+                                            else:
+                                                row_data[f"{source}_{stat}"] = source_metrics[metric][0][stat] * 100
+                                    writer.writerow(row_data)