feat: allow to define multiple storages at once

gisaia · Jan 9, 2025 · 209f9f3 · 209f9f3
1 parent 30f0db1
commit 209f9f3
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 30 deletions.
diff --git a/aproc/core/settings.py b/aproc/core/settings.py
@@ -21,7 +21,7 @@ class Settings(BaseModel, extra='allow'):
     celery_result_backend: str | None = Field(title="Celery's backend")
     processes: list[ProcessSettings] = Field(title="List of processes available")
     airs_endpoint: str | None = Field(title="ARLAS Item Registration Service endpoint")
-    storage: StorageSettings = Field(title="Configuration of available storage")
+    storages: list[StorageSettings] = Field(title="List of configurations for the available storages")
 
 
 class Configuration:

diff --git a/conf/aproc.yaml b/conf/aproc.yaml
@@ -23,15 +23,18 @@ processes:
     configuration:
       drivers: conf/enrich_drivers.yaml
 
-storage:
-  type: $APROC_INPUT_STORAGE_TYPE
-  # Google Storage
-  bucket: $APROC_INPUT_STORAGE_BUCKET
-  # api_key:
-  #   project_id: $APROC_INPUT_STORAGE_API_KEY_PROJECT
-  #   private_key_id: $APROC_INPUT_STORAGE_API_KEY_PRIVATE_KEY_ID
-  #   private_key: $APROC_INPUT_STORAGE_API_KEY_PRIVATE_KEY
-  # # HTTPS
-  # headers: $APROC_INPUT_STORAGE_HEADERS
-  # domain: $APROC_INPUT_STORAGE_DOMAIN
-  # force_download: $APROC_INPUT_STORAGE_FORCE_DOWNLOAD
+storages:
+  -
+    type: file
+  -
+    type: gs
+    bucket: gisaia-public
+    # api_key:
+    #   project_id: $APROC_INPUT_STORAGE_API_KEY_PROJECT
+    #   private_key_id: $APROC_INPUT_STORAGE_API_KEY_PRIVATE_KEY_ID
+    #   private_key: $APROC_INPUT_STORAGE_API_KEY_PRIVATE_KEY
+  # -
+  #   type: https
+  #   headers: $APROC_INPUT_STORAGE_HEADERS
+  #   domain: $APROC_INPUT_STORAGE_DOMAIN
+  #   force_download: $APROC_INPUT_STORAGE_FORCE_DOWNLOAD
diff --git a/extensions/aproc/proc/access/manager.py b/extensions/aproc/proc/access/manager.py
@@ -20,34 +20,36 @@
 
 
 class AccessManager:
-    storage: AnyStorage | None = Field(None)
+    storages: list[AnyStorage]
     tmp_dir = tempfile.gettempdir()
 
     @staticmethod
     def init():
         LOGGER.info("Initializing APROC storages")
-        storage_settings = Configuration.settings.storage.model_dump()
-
-        match Configuration.settings.storage.type:
-            case "file":
-                AccessManager.storage = FileStorage(**storage_settings)
-            case "gs":
-                AccessManager.storage = GoogleStorage(**storage_settings)
-            case "http":
-                AccessManager.storage = HttpStorage(**storage_settings)
-            case "https":
-                AccessManager.storage = HttpsStorage(**storage_settings)
-            case _:
-                raise NotImplementedError(f"Specified storage {Configuration.settings.storage.type} is not implemented")
+        AccessManager.storages = []
+
+        for s in Configuration.settings.storages:
+            match s.type:
+                case "file":
+                    AccessManager.storages.append(FileStorage(**s.model_dump()))
+                case "gs":
+                    AccessManager.storages.append(GoogleStorage(**s.model_dump()))
+                case "http":
+                    AccessManager.storages.append(HttpStorage(**s.model_dump()))
+                case "https":
+                    AccessManager.storages.append(HttpsStorage(**s.model_dump()))
+                case _:
+                    raise NotImplementedError(f"Specified storage {s.type} is not implemented")
 
     @staticmethod
     def resolve_storage(href: str) -> AnyStorage:
         """
-        Based on the defined storages (TODO), returns the one matching the input href
+        Based on the defined storages, returns the one matching the input href
         """
 
-        if AccessManager.storage.supports(href):
-            return AccessManager.storage
+        for s in AccessManager.storages:
+            if s.supports(href):
+                return s
 
         raise NotImplementedError(f"Storage for {href} is not configured")