diff --git a/aproc/core/settings.py b/aproc/core/settings.py index de3addf..e907b46 100644 --- a/aproc/core/settings.py +++ b/aproc/core/settings.py @@ -21,7 +21,7 @@ class Settings(BaseModel, extra='allow'): celery_result_backend: str | None = Field(title="Celery's backend") processes: list[ProcessSettings] = Field(title="List of processes available") airs_endpoint: str | None = Field(title="ARLAS Item Registration Service endpoint") - storage: StorageSettings = Field(title="Configuration of available storage") + storages: list[StorageSettings] = Field(title="List of configurations for the available storages") class Configuration: diff --git a/conf/aproc.yaml b/conf/aproc.yaml index 380cad1..b6aa07e 100644 --- a/conf/aproc.yaml +++ b/conf/aproc.yaml @@ -23,15 +23,18 @@ processes: configuration: drivers: conf/enrich_drivers.yaml -storage: - type: $APROC_INPUT_STORAGE_TYPE - # Google Storage - bucket: $APROC_INPUT_STORAGE_BUCKET - # api_key: - # project_id: $APROC_INPUT_STORAGE_API_KEY_PROJECT - # private_key_id: $APROC_INPUT_STORAGE_API_KEY_PRIVATE_KEY_ID - # private_key: $APROC_INPUT_STORAGE_API_KEY_PRIVATE_KEY - # # HTTPS - # headers: $APROC_INPUT_STORAGE_HEADERS - # domain: $APROC_INPUT_STORAGE_DOMAIN - # force_download: $APROC_INPUT_STORAGE_FORCE_DOWNLOAD \ No newline at end of file +storages: + - + type: file + - + type: gs + bucket: gisaia-public + # api_key: + # project_id: $APROC_INPUT_STORAGE_API_KEY_PROJECT + # private_key_id: $APROC_INPUT_STORAGE_API_KEY_PRIVATE_KEY_ID + # private_key: $APROC_INPUT_STORAGE_API_KEY_PRIVATE_KEY + # - + # type: https + # headers: $APROC_INPUT_STORAGE_HEADERS + # domain: $APROC_INPUT_STORAGE_DOMAIN + # force_download: $APROC_INPUT_STORAGE_FORCE_DOWNLOAD \ No newline at end of file diff --git a/extensions/aproc/proc/access/manager.py b/extensions/aproc/proc/access/manager.py index 7c22e3c..6dea2cc 100644 --- a/extensions/aproc/proc/access/manager.py +++ b/extensions/aproc/proc/access/manager.py @@ -20,34 +20,36 @@ class AccessManager: - storage: AnyStorage | None = Field(None) + storages: list[AnyStorage] tmp_dir = tempfile.gettempdir() @staticmethod def init(): LOGGER.info("Initializing APROC storages") - storage_settings = Configuration.settings.storage.model_dump() - - match Configuration.settings.storage.type: - case "file": - AccessManager.storage = FileStorage(**storage_settings) - case "gs": - AccessManager.storage = GoogleStorage(**storage_settings) - case "http": - AccessManager.storage = HttpStorage(**storage_settings) - case "https": - AccessManager.storage = HttpsStorage(**storage_settings) - case _: - raise NotImplementedError(f"Specified storage {Configuration.settings.storage.type} is not implemented") + AccessManager.storages = [] + + for s in Configuration.settings.storages: + match s.type: + case "file": + AccessManager.storages.append(FileStorage(**s.model_dump())) + case "gs": + AccessManager.storages.append(GoogleStorage(**s.model_dump())) + case "http": + AccessManager.storages.append(HttpStorage(**s.model_dump())) + case "https": + AccessManager.storages.append(HttpsStorage(**s.model_dump())) + case _: + raise NotImplementedError(f"Specified storage {s.type} is not implemented") @staticmethod def resolve_storage(href: str) -> AnyStorage: """ - Based on the defined storages (TODO), returns the one matching the input href + Based on the defined storages, returns the one matching the input href """ - if AccessManager.storage.supports(href): - return AccessManager.storage + for s in AccessManager.storages: + if s.supports(href): + return s raise NotImplementedError(f"Storage for {href} is not configured") @@ -97,7 +99,7 @@ def is_download_required(href: str): storage = AccessManager.resolve_storage(href) return storage.type in ["http", "https"] \ - and AccessManager.storage.force_download + and storage.force_download @staticmethod def prepare(href: str):