Create README.md

Fix slurm job submission update README update README Add layer mask
mackelab · Dec 1, 2023 · 8541f0c · 8541f0c
1 parent e7b6c6b
commit 8541f0c
Show file tree

Hide file tree

Showing 40 changed files with 273 additions and 231 deletions.
diff --git a/.gitignore b/.gitignore
@@ -157,4 +157,10 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+#.idea/
+
+out/
+*.DS_Store
+data/*
+!data/Ekstrom/IRH_data/LayerData_flowline_mask.mat
+*sbi-logs/
diff --git a/sbi_ice/README.md → README.md b/sbi_ice/README.md → README.md
@@ -1,9 +1,10 @@
 # Simulation-Based Inference of Surface Accumulation and Basal Melt Rates of Antarctic Ice Shelves from Isochronal Layers
 
-This repository contains research code for Simulation-based inference of surface accumulation and basal melt rates of Antarctic Ice Shelves from Isochronal Layers [preprint link]. It contains scripts for the forward model, along with the inference workflow. For code used to preprocess ice shelf data, and generate synthetic ice shelves using icepack[icepack link] - go to [preprocessing repository link]. 
+This repository contains research code for Simulation-based inference of surface accumulation and basal melt rates of Antarctic Ice Shelves from Isochronal Layers [preprint link]. It contains scripts for the forward model, along with the inference workflow. For code used to preprocess ice shelf data, and generate synthetic ice shelves using [firedrake](https://www.firedrakeproject.org) and [icepack](https://icepack.github.io) - see [preprocessing_ice_data](https://github.com/mackelab/preprocessing_ice_data/). 
 Maintenance is ongoing! Expect improvements in terms of extended functionality and improved usability in the future.
 
 Some small data and output files are already included in this repository for quicker start-up. Some parts of the workflow require bigger files. These will clearly be marked in the **workflow** section, along with links to where these files can be downloaded.
+The picked IRH data for Ekström Ice Shelf can be found [here](https://doi.org/doi:10.5281/zenodo.10231043).
 
 ## Installation
 Activate a virtual environment, e.g. using conda. Install dependencies:
@@ -47,18 +48,18 @@ First, we look at the calibration simulations. We find (and ignore) any anomalou
     gt_version
     anomalies_overwrite = True
     layer_bounds_overwrite = True
-    noise_dist_overwrite = True or False
+    noise_dist_overwrite = True/False
+    sim_or_calib = "layer_sims"/"calib_sims"
 ```
 
-We then find any anomalous results in `.layer_sims`. Keep everything the same, and make sure `layer_bounds_overwrite=False` and `noise_dist_overwite = False`. This can be sped up given access to multiple CPU cores by running `submit_calibrate_sims.sh`
+We then find any anomalous results in `.layer_sims`. Keep everything the same, and make sure `layer_bounds_overwrite=False` and `noise_dist_overwite = False`. This can be sped up given access to multiple CPU cores by running `sbatch submit_calibrate_sims.sh` on SLURM or `source submit_calibrate_sims.sh` locally.
 
 ### 3. Noise model and best fitting layer selection
 
 Our inference workflow requires only one layer out of each simulation to be the observed value, and so we select the best fitting layer (by MSE). This is done for all simulations in a single job using the script `sbi_ice/runs/select_layers.py`. This can be parallelized across all the simulation jobs using `sbi_ice_runs/submit_select_layers_local.sh` or `sbi_ice/runs/submit_select_layers_slurm.sh`
 The rest of the workflow takes in one processed layer file for all simulations - the per-job files can be combined and saved into one file using `notebooks/workflow/combine_pickles.ipynb`
 
-For the processed simulation data for both the synthetic and Ekström ice shelves used used in our experiments, you can find the file at [link to data repository].
-
+To reproduce the results reported in the paper, you can [download the posteriors and processed simulation data](https://doi.org/doi:10.5281/zenodo.10245153), and save the respective files in `out`.
 
 
 ### 4. Training a Density Estimator

diff --git a/configs/post_predictive/Ekstrom_post_pred.yaml b/configs/post_predictive/Ekstrom_post_pred.yaml
@@ -1,12 +1,12 @@
 defaults:
   - _self_
-  - launcher: cpu_local
+  - override hydra/launcher: cpu_slurm #=basic for local, serial job submission (debugging)
 
 
 
 shelf: Ekstrom
 exp_path: exp2
-post_name: all_final
+post_name: test
 
 posterior_config_fol: ../../../posteriors/${post_name}
 n_post_samples: 1000
@@ -33,5 +33,5 @@ hydra:
   sweeper:
     params:
       layer_idx: range(0,4)
-      +posterior_seed: range(100,105)
+      +posterior_seed: range(1000,1001)
 
diff --git a/configs/post_predictive/Synthetic_long_post_pred.yaml b/configs/post_predictive/Synthetic_long_post_pred.yaml
@@ -1,6 +1,6 @@
 defaults:
   - _self_
-  - launcher: cpu_local
+  - override hydra/launcher: cpu_slurm #=basic for local, serial job submission (debugging)
 
 
 shelf: Synthetic_long
@@ -9,7 +9,7 @@ post_name: test
 
 posterior_config_fol: ../../../posteriors/${post_name}
 n_post_samples: 1000
-n_predictive_sims: 1
+n_predictive_sims: 100
 overwrite_saved_sims: True
 layer_idx: 0
 seed: 0

diff --git a/configs/post_predictive/hydra/launcher/cpu_local.yaml b/configs/post_predictive/hydra/launcher/cpu_local.yaml
@@ -0,0 +1,9 @@
+_target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+submitit_folder: ${hydra.sweep.dir}/.submitit/%j
+timeout_min: 60
+cpus_per_task: 1
+gpus_per_node: 0
+tasks_per_node: 1
+mem_gb: 4
+nodes: 1
+name: ${hydra.job.name}
diff --git a/configs/post_predictive/hydra/launcher/cpu_slurm.yaml b/configs/post_predictive/hydra/launcher/cpu_slurm.yaml
@@ -0,0 +1,27 @@
+submitit_folder: ${hydra.sweep.dir}/.submitit/%j
+timeout_min: 1200
+mem_gb: null
+nodes: 1
+gpus_per_node: null
+tasks_per_node: 1
+cpus_per_task: 1
+name: ${hydra.job.name}
+
+
+_target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher
+partition: cpu-preemptable
+qos: null
+comment: null
+constraint: null
+exclude: null
+gres: null
+cpus_per_gpu: null
+gpus_per_task: null
+mem_per_gpu: null
+mem_per_cpu: null
+account: null
+signal_delay_s: 120
+max_num_timeout: 0
+additional_parameters: {}
+setup: null
+array_parallelism: 100
diff --git a/configs/post_predictive/hydra/launcher/gpu_slurm.yaml b/configs/post_predictive/hydra/launcher/gpu_slurm.yaml
@@ -0,0 +1,24 @@
+submitit_folder: ${hydra.sweep.dir}/.submitit/%j
+timeout_min: 300
+mem_gb: 32
+nodes: 1
+gpus_per_node: 1
+name: ${hydra.job.name}
+
+
+_target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher
+partition: null
+qos: null
+comment: null
+constraint: null
+exclude: null
+gres: null
+cpus_per_gpu: null
+gpus_per_task: 1
+mem_per_gpu: null
+mem_per_cpu: null
+account: null
+signal_delay_s: 120
+max_num_timeout: 0
+additional_parameters: {}
+setup: null
diff --git a/configs/post_predictive/launcher/cpu_local.yaml b/configs/post_predictive/launcher/cpu_local.yaml
diff --git a/configs/post_predictive/launcher/cpu_slurm.yaml b/configs/post_predictive/launcher/cpu_slurm.yaml
diff --git a/configs/post_predictive/launcher/gpu_slurm.yaml b/configs/post_predictive/launcher/gpu_slurm.yaml
diff --git a/configs/run_sims/Ekstrom.yaml b/configs/run_sims/Ekstrom.yaml
@@ -5,7 +5,7 @@ defaults:
   - random_seed: 0_seed
   - prior: profile_and_transform
   - scheds: two_phases
-  - launcher: cpu_local
+  - override hydra/launcher: basic #=basic for local, serial job submission (debugging)
 
 hydra:
   job:
@@ -24,4 +24,4 @@ hydra:
 
   sweeper:
       params:
-        random_seed.seed: range(0,1)
+        random_seed.seed: range(0,10)
diff --git a/configs/run_sims/Synthetic_long.yaml b/configs/run_sims/Synthetic_long.yaml
@@ -5,7 +5,7 @@ defaults:
   - random_seed: 0_seed
   - prior: profile_and_transform
   - scheds: two_phases
-  - launcher: cpu_local
+  - override hydra/launcher: basic #=basic for local, serial job submission (debugging)
 
 hydra:
   job:
@@ -24,4 +24,4 @@ hydra:
 
   sweeper:
       params:
-        random_seed.seed: range(1,5)
+        random_seed.seed: range(0,10)
diff --git a/configs/run_sims/hydra/launcher/cpu_local.yaml b/configs/run_sims/hydra/launcher/cpu_local.yaml
@@ -0,0 +1,9 @@
+_target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+submitit_folder: ${hydra.sweep.dir}/.submitit/%j
+timeout_min: 60
+cpus_per_task: 1
+gpus_per_node: 0
+tasks_per_node: 1
+mem_gb: 4
+nodes: 1
+name: ${hydra.job.name}
diff --git a/configs/run_sims/hydra/launcher/cpu_slurm.yaml b/configs/run_sims/hydra/launcher/cpu_slurm.yaml
@@ -0,0 +1,27 @@
+submitit_folder: ${hydra.sweep.dir}/.submitit/%j
+timeout_min: 1200
+mem_gb: null
+nodes: 1
+gpus_per_node: null
+tasks_per_node: 1
+cpus_per_task: 1
+name: ${hydra.job.name}
+
+
+_target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher
+partition: cpu-preemptable
+qos: null
+comment: null
+constraint: null
+exclude: null
+gres: null
+cpus_per_gpu: null
+gpus_per_task: null
+mem_per_gpu: null
+mem_per_cpu: null
+account: null
+signal_delay_s: 120
+max_num_timeout: 0
+additional_parameters: {}
+setup: null
+array_parallelism: 100
diff --git a/configs/run_sims/hydra/launcher/gpu_slurm.yaml b/configs/run_sims/hydra/launcher/gpu_slurm.yaml
@@ -0,0 +1,24 @@
+submitit_folder: ${hydra.sweep.dir}/.submitit/%j
+timeout_min: 300
+mem_gb: 32
+nodes: 1
+gpus_per_node: 1
+name: ${hydra.job.name}
+
+
+_target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher
+partition: null
+qos: null
+comment: null
+constraint: null
+exclude: null
+gres: null
+cpus_per_gpu: null
+gpus_per_task: 1
+mem_per_gpu: null
+mem_per_cpu: null
+account: null
+signal_delay_s: 120
+max_num_timeout: 0
+additional_parameters: {}
+setup: null
diff --git a/configs/run_sims/launcher/cpu_local.yaml b/configs/run_sims/launcher/cpu_local.yaml
diff --git a/configs/run_sims/launcher/cpu_slurm.yaml b/configs/run_sims/launcher/cpu_slurm.yaml
diff --git a/configs/run_sims/launcher/gpu_slurm.yaml b/configs/run_sims/launcher/gpu_slurm.yaml
diff --git a/configs/training/.DS_Store b/configs/training/.DS_Store