Configuration for crop cli

.github/workflows/ci.yaml

@@ -17,11 +17,11 @@ env:
 
 on:
   pull_request:
-    branches: ['main', 'cache-cpm-for-hads', ]
+    branches: ['main',]
     paths-ignore: ['docs/**']
 
   push:
-    branches: ['main', 'cache-cpm-for-hads', ]
+    branches: ['main', 'crop-cli']
     paths-ignore: ['docs/**']
 
 concurrency:

_quarto.yml

@@ -73,22 +73,36 @@ website:
             text: "CEDA Data Access"
           - href: "docs/reference/clim_recal.data_loader.qmd"
             text: "Data Loading"
-          - href: "docs/reference/clim_recal.resample.qmd"
+          - href: "docs/reference/clim_recal.convert.qmd"
             text: "Data Resampling"
           - section: "Debiasing"
             contents:
-              - href: "docs/reference/clim_recal.debiasing.debias_wrapper.qmd"
-                text: "Wrapper"
-          - section: "Utilities"
-            contents:
-              - href: "docs/reference/clim_recal.utils.core.qmd"
-                text: "core"
-              - href: "docs/reference/clim_recal.utils.server.qmd"
-                text: "server"
-              - href: "docs/reference/clim_recal.utils.xarray.qmd"
-                text: "xarray"
-              - href: "docs/reference/clim_recal.utils.data.qmd"
-                text: "data"
+              - href: "docs/reference/clim_recal.pipeline.qmd"
+                text: "Pipeline"
+              - href: "docs/reference/clim_recal.config.qmd"
+                text: "Configure"
+              - href: "docs/reference/clim_recal.ceda_ftp_download.qmd"
+                text: "CEDA Data Access"
+              - href: "docs/reference/clim_recal.data_loader.qmd"
+                text: "Data Loading"
+              - href: "docs/reference/clim_recal.convert.qmd"
+                text: "Data Resampling"
+              - section: "Debiasing"
+                contents:
+                  - href: "docs/reference/clim_recal.debiasing.debias_wrapper.qmd"
+                    text: "Wrapper"
+              - section: "Utilities"
+                contents:
+                  - href: "docs/reference/clim_recal.utils.core.qmd"
+                    text: "core"
+                  - href: "docs/reference/clim_recal.utils.server.qmd"
+                    text: "server"
+                  - href: "docs/reference/clim_recal.utils.xarray.qmd"
+                    text: "xarray"
+                  - href: "docs/reference/clim_recal.utils.data.qmd"
+                    text: "data"
+      - text: "Docker"
+        href: "docs/docker-configurations.qmd"
       - text: "Contributing"
         href: "docs/contributing.md"
 
@@ -130,7 +144,7 @@ quartodoc:
         - clim_recal.ceda_ftp_download
         - clim_recal.data_loader
         - clim_recal.config
-        - clim_recal.resample
+        - clim_recal.convert
         - clim_recal.debiasing.debias_wrapper
         - clim_recal.utils.core
         - clim_recal.utils.server

bash/run-pipeline-iteratively.sh

@@ -68,8 +68,10 @@ for year in $(seq $cpm_start_year $cpm_end_year); do
 
   {
     clim-recal \
-    --hads-input-path $hads_working_dir \
-    --cpm-input-path $cpm_working_dir \
+    --resample-start-index $i \
+    --total-from-index 1 \
+    --hads-input-path $hads_input_path \
+    --cpm-input-path $cpm_input_path \
     --output-path $output_path \
     --all-variables \
     --all-regions \

python/clim_recal/ceda_ftp_download.py

@@ -3,8 +3,26 @@
 import ftplib
 import os
 import random
+from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
+from typing import Final, Sequence
+
+HADS_FTP_PATH: Final[str] = (
+    "/badc/ukmo-hadobs/data/insitu/MOHC/HadOBS/HadUK-Grid/v1.2.0.ceda/1km/"
+)
+CPM_FTP_PATH: Final[str] = "/badc/ukcp18/data/land-cpm/uk/2.2km/rcp85/"
+
+DEFAULT_SAVE_PATH: Final[Path] = Path("ceda")
+CEDA_ENV_USER_NAME_KEY: Final[str] = "CLIM_RECAL_CEDA_USER_NAME"
+CEDA_ENV_PASSWORD_KEY: Final[str] = "CLIM_RECAL_CEDA_PASSWORD"
+
+
+def check_env_auth() -> bool:
+    """Test if CEDA `user_name` and `password` available."""
+    user_name: str | None = os.getenv(CEDA_ENV_USER_NAME_KEY)
+    password: str | None = os.getenv(CEDA_ENV_PASSWORD_KEY)
+    return True if user_name and password else False
 
 
 def download_ftp(
@@ -77,19 +95,91 @@ def download_ftp(
 
             if size_ftp == size_local:
                 download = False
-                print("File exist, will not dowload")
+                print("File exists, will not download")
 
         if download:
             f.retrbinary("RETR %s" % file, open(file, "wb").write)
 
         counter += 1
         print(counter, "file downloaded out of", len(filelist))
 
-    print("Finished: ", counter, " files dowloaded from ", input)
+    print("Finished: ", counter, " files downloaded from ", input)
     # Close FTP connection
     f.close()
 
 
+@dataclass(kw_only=True)
+class HADsCEDADownloadManager:
+
+    user_name: str | None
+    password: str | None
+    variables: Sequence[str] | None = None
+    save_path: os.PathLike = DEFAULT_SAVE_PATH
+    reverse: bool = False
+    shuffle: bool = False
+    change_hierarchy: bool = False
+    ftp_path: str = HADS_FTP_PATH
+    order: int = 0
+
+    def __post_init__(self) -> None:
+        self.user_name = self.user_name or os.getenv(CEDA_ENV_USER_NAME_KEY)
+        self.password = self.password or os.getenv(CEDA_ENV_PASSWORD_KEY)
+        if self.reverse:
+            self.order = 1
+        # reverse precedes shuffle
+        elif self.shuffle:
+            self.order = 2
+        if not self.user_name or not self.password:
+            raise ValueError(f"Both 'user_name' and 'password' needed.")
+
+    def download(self) -> None:
+        if self.change_hierarchy:
+            for v in self.variables:
+                download_ftp(
+                    os.path.join(self.ftp_path, n, v, "day", "latest"),
+                    os.path.join(self.save_path, v, n, "latest"),
+                    username=self.user_name,
+                    password=self.password,
+                    order=self.order,
+                )
+        else:
+            download_ftp(
+                self.ftp_path,
+                str(self.save_path),
+                username=self.user_name,
+                password=self.password,
+                order=self.order,
+            )
+
+
+@dataclass(kw_only=True, repr=False)
+class CPMCEDADownloadManager(HADsCEDADownloadManager):
+    """Manage downloading raw CPM data."""
+
+    runs: Sequence[str] | None = None
+    ftp_path: str = CPM_FTP_PATH
+
+    def download(self) -> None:
+        if self.change_hierarchy:
+            for n in self.runs:
+                for v in self.variables:
+                    download_ftp(
+                        os.path.join(self.ftp_path, n, v, "day", "latest"),
+                        os.path.join(self.save_path, v, n, "latest"),
+                        username=self.user_name,
+                        password=self.password,
+                        order=self.order,
+                    )
+        else:
+            download_ftp(
+                self.ftp_path,
+                str(self.save_path),
+                username=self.user_name,
+                password=self.password,
+                order=self.order,
+            )
+
+
 if __name__ == "__main__":
     """
     Script to download CEDA data from the command line.