Add error handling for payu checkout with non-writable laboratory (#528)

* Initialise Laboratory directories in payu checkout and in the Experiment class.
* Add info for error messaging when laboratory directories are non-writable.
* payu checkout: Fix when old archive symlink points to a deleted directory
* payu checkout tests: Fix un-caught metadata warning

payu/branch.py

@@ -21,6 +21,14 @@
 from payu.metadata import Metadata, UUID_FIELD, METADATA_FILENAME
 from payu.git_utils import GitRepository, git_clone, PayuBranchError
 
+LAB_WRITE_ACCESS_ERROR = """
+Failed to initialise laboratory directories. Skipping creating metadata,
+setting up restart configuration, and archive/work symlinks.
+
+To fix, first modify/remove the config.yaml options that determine laboratory
+path. Then either run 'payu setup' or rerun checkout command with the current
+git branch, e.g. `payu checkout -r  <RESTART_PATH> <CURRENT_BRANCH>`
+"""
 
 NO_CONFIG_FOUND_MESSAGE = """No configuration file found on this branch.
 Skipping adding new metadata file and creating archive/work symlinks.
@@ -171,8 +179,15 @@ def checkout_branch(branch_name: str,
     # Check config file exists on checked out branch
     config_path = check_config_path(config_path)
 
-    # Initialise Lab and Metadata
+    # Initialise Lab
     lab = Laboratory(model_type, config_path, lab_path)
+    try:
+        lab.initialize()
+    except PermissionError:
+        print(LAB_WRITE_ACCESS_ERROR)
+        raise
+
+    # Initialise metadata
     metadata = Metadata(Path(lab.archive_path),
                         branch=branch_name,
                         config_path=config_path)
@@ -211,7 +226,7 @@ def switch_symlink(lab_dir_path: Path, control_path: Path,
     sym_path = control_path / sym_dir
 
     # Remove symlink if it already exists
-    if sym_path.exists() and sym_path.is_symlink:
+    if sym_path.is_symlink():
         previous_path = sym_path.resolve()
         sym_path.unlink()
         print(f"Removed {sym_dir} symlink to {previous_path}")

payu/experiment.py

@@ -52,6 +52,8 @@ class Experiment(object):
 
     def __init__(self, lab, reproduce=False, force=False, metadata_off=False):
         self.lab = lab
+        # Check laboratory directories are writable
+        self.lab.initialize()
 
         if not force:
             # check environment for force flag under PBS

payu/laboratory.py

@@ -10,6 +10,17 @@
 
 from payu.fsops import mkdir_p, read_config
 
+LAB_INITIALIZE_ERROR = """
+The configured laboratory directory may not have write access. Edit/remove one
+(or more) of the following config.yaml options that determine the laboratory
+path:
+    - 'project': The project to use for payu PBS jobs. Default: ${PROJECT}
+    - 'shortpath' Top-level directory for laboratory
+        Default: /scratch/${PROJECT}
+    - 'laboratory': Top-level directory for the model laboratory
+        Default: /scratch/${PROJECT}/${USER}/${MODEL}
+"""
+
 
 class Laboratory(object):
     """Interface to the numerical model's laboratory."""
@@ -84,7 +95,13 @@ def get_default_lab_path(self, config):
 
     def initialize(self):
         """Create the laboratory directories."""
-        mkdir_p(self.archive_path)
-        mkdir_p(self.bin_path)
-        mkdir_p(self.codebase_path)
-        mkdir_p(self.input_basepath)
+        try:
+            mkdir_p(self.archive_path)
+            mkdir_p(self.bin_path)
+            mkdir_p(self.codebase_path)
+            mkdir_p(self.input_basepath)
+        except PermissionError as e:
+            print(LAB_INITIALIZE_ERROR)
+            raise PermissionError(
+                f"Failed to initialise laboratory directories. Error: {e}"
+            )

test/test_branch.py

@@ -215,6 +215,27 @@ def test_switch_symlink_when_no_symlink_exists_and_no_archive():
     assert not archive_symlink.is_symlink()
 
 
+def test_switch_symkink_when_previous_symlink_dne():
+    # Point archive symlink to a directory that does not exist anymore
+    lab_archive = labdir / "archive"
+    previous_archive_dir = lab_archive / "ExperimentDNE"
+
+    archive_symlink = ctrldir / "archive"
+    archive_symlink.symlink_to(previous_archive_dir)
+
+    # New Experiment
+    experiment_name = "Experiment1"
+    archive_dir = lab_archive / experiment_name
+    archive_dir.mkdir(parents=True)
+
+    # Test Function
+    switch_symlink(lab_archive, ctrldir, experiment_name, "archive")
+
+    # Assert new symlink is created
+    assert archive_symlink.exists() and archive_symlink.is_symlink()
+    assert archive_symlink.resolve() == archive_dir
+
+
 def check_metadata(expected_uuid,
                    expected_experiment,
                    expected_parent_uuid=None,
@@ -342,12 +363,16 @@ def test_checkout_existing_branch_with_no_metadata(mock_uuid):
     expected_no_uuid_msg = (
         "No experiment uuid found in metadata. Generating a new uuid"
     )
+    expected_no_archive_msg = (
+        "No pre-existing archive found. Generating a new uuid"
+    )
 
     with cd(ctrldir):
         # Test checkout existing branch with no existing metadata
         with pytest.warns(MetadataWarning, match=expected_no_uuid_msg):
-            checkout_branch(branch_name="Branch1",
-                            lab_path=labdir)
+            with pytest.warns(MetadataWarning, match=expected_no_archive_msg):
+                checkout_branch(branch_name="Branch1",
+                                lab_path=labdir)
 
     # Check metadata was created and commited
     branch1_experiment_name = f"{ctrldir_basename}-Branch1-574ea2c9"
@@ -489,6 +514,39 @@ def test_checkout_branch_with_restart_path(mock_uuid):
                           expected_parent_uuid=uuid1)
 
 
+@patch("payu.laboratory.Laboratory.initialize")
+def test_checkout_laboratory_path_error(mock_lab_initialise):
+    mock_lab_initialise.side_effect = PermissionError
+
+    repo = setup_control_repository()
+    current_commit = repo.active_branch.object.hexsha
+
+    with cd(ctrldir):
+        # Test raises a permission error
+        with pytest.raises(PermissionError):
+            checkout_branch(branch_name="Branch1",
+                            is_new_branch=True,
+                            lab_path=labdir)
+
+    # Assert new commit has not been added
+    assert repo.active_branch.object.hexsha == current_commit
+
+    assert str(repo.active_branch) == "Branch1"
+    assert not (ctrldir / "metadata.yaml").exists()
+
+    # Test removing lab directory
+    shutil.rmtree(labdir)
+    mock_lab_initialise.side_effect = None
+    with cd(ctrldir):
+        # Test runs without an error - directory is initialised
+        checkout_branch(branch_name="Branch2",
+                        is_new_branch=True,
+                        lab_path=labdir)
+
+    # Assert new commit has been added
+    assert repo.active_branch.object.hexsha != current_commit
+
+
 @patch("uuid.uuid4")
 def test_clone(mock_uuid):
     # Create a repo to clone