feat: minio service

docker-compose.yaml

@@ -14,6 +14,7 @@ include:
       - lib/gohan/docker-compose.gohan.yaml  # Optional feature; controlled by a compose profile
       - lib/katsu/docker-compose.katsu.yaml
       - lib/logs/docker-compose.logs.yaml
+      - lib/minio/docker-compose.minio.yaml
       - lib/notification/docker-compose.notification.yaml
       - lib/public/docker-compose.public.yaml  # Optional feature; controlled by a compose profile
       - lib/redis/docker-compose.redis.yaml

docs/deployment.md

@@ -9,6 +9,7 @@ BENTOV2_DOMAIN=bento.example.com
 BENTOV2_PORTAL_DOMAIN=portal.${BENTOV2_DOMAIN}
 BENTOV2_AUTH_DOMAIN=auth.${BENTOV2_DOMAIN}
 BENTOV2_CBIOPORTAL_DOMAIN=cbioportal.${BENTOV2_DOMAIN}
+BENTO_MINIO_DOMAIN=minio.${BENTOV2_DOMAIN}
 ```
 
 For a real deployment, make sure that your `local.env` file uses valid domain names for which SSL certificates 

docs/migrating_to_18.md

@@ -19,6 +19,27 @@ TODO
   above location, or copy `branding.png` to `branding.lightbg.png`
 
 
+## 3. Enabling MinIO
+
+Enable MinIO by setting the feature flag in local.env
+```bash
+BENTO_MINIO_ENABLED='true'
+```
+
+After enabling the MinIO feature flag for the first time, 
+you must initialize the Docker networks, mounted directories and certs.
+```bash
+./bentoctl.bash init-certs -f
+./bentoctl.bash init-docker # new network for MinIO
+./bentoctl.bash init-dirs 
+```
+
+Also set root user and password in local.env:
+```bash
+BENTO_MINIO_ROOT_USER=root # (default value, could be change)
+BENTO_MINIO_ROOT_PASSWORD=secure-password
+```
+
 TODO
 
 

docs/object-store.md

@@ -0,0 +1,25 @@
+# Bento object store
+
+![MinIO service logs querying](./img/minio_object_store.png)
+
+## Configuration
+
+Enable MinIO by setting the feature flag in local.env
+
+```bash
+BENTO_MINIO_ENABLED='true'
+```
+
+After enabling the MinIO feature flag for the first time, 
+you must initialize the Docker networks, mounted directories and certs.
+```bash
+./bentoctl.bash init-certs -f
+./bentoctl.bash init-docker # new network for MinIO
+./bentoctl.bash init-dirs 
+```
+
+Also set root user and password in local.env:
+```bash
+BENTO_MINIO_ROOT_USER=root # (default value, could be change)
+BENTO_MINIO_ROOT_PASSWORD=secure-password
+```

etc/bento.env

@@ -34,7 +34,7 @@ BENTOV2_GATEWAY_INTERNAL_CERTS_DIR=/usr/local/openresty/nginx/certs
 
 # Gateway
 BENTOV2_GATEWAY_IMAGE=ghcr.io/bento-platform/bento_gateway
-BENTOV2_GATEWAY_VERSION=0.13.2
+BENTOV2_GATEWAY_VERSION=edge
 BENTOV2_GATEWAY_VERSION_DEV=${BENTOV2_GATEWAY_VERSION}-dev
 BENTOV2_GATEWAY_CONTAINER_NAME=${BENTOV2_PREFIX}-gateway
 
@@ -472,3 +472,13 @@ BENTO_GRAFANA_SIGNOUT_REDIRECT_URL=https://${BENTOV2_AUTH_DOMAIN}/realms/${BENTO
 BENTO_PROMTAIL_IMAGE=grafana/promtail
 BENTO_PROMTAIL_IMAGE_VERSION=3.1.2
 BENTO_PROMTAIL_CONTAINER_NAME=${BENTOV2_PREFIX}-promtail
+
+# MinIO
+BENTO_MINIO_IMAGE=quay.io/minio/minio
+BENTO_MINIO_IMAGE_VERSION=RELEASE.2024-12-13T22-19-12Z
+BENTO_MINIO_CONTAINER_NAME=${BENTOV2_PREFIX}-minio
+BENTO_MINIO_DATA_DIR=${BENTO_SLOW_DATA_DIR}/minio/data
+BENTO_MINIO_NETWORK=${BENTOV2_PREFIX}-minio-net
+BENTO_MINIO_ROOT_USER=root
+BENTO_MINIO_INTERNAL_PORT=9000
+BENTO_MINIO_CONSOLE_PORT=9001

etc/bento_deploy.env

@@ -14,6 +14,7 @@ BENTO_BEACON_NETWORK_ENABLED='false'
 BENTO_CBIOPORTAL_ENABLED='false'
 BENTO_GOHAN_ENABLED='true'
 BENTO_MONITORING_ENABLED='false'
+BENTO_MINIO_ENABLED='false'
 
 #  - Switch to enable French translation in Bento Public
 BENTO_PUBLIC_TRANSLATED='true'
@@ -36,6 +37,8 @@ BENTOV2_PORTAL_DOMAIN=portal.${BENTOV2_DOMAIN}
 BENTOV2_AUTH_DOMAIN=bentov2auth.local
 # Unused if cBioPortal is disabled:
 BENTOV2_CBIOPORTAL_DOMAIN=cbioportal.${BENTOV2_DOMAIN}
+# Unused if MinIO is disabled
+BENTO_MINIO_DOMAIN=minio.${BENTOV2_DOMAIN}
 # ---------------------------------------------------------------------
 
 # Authn/z -------------------------------------------------------------
@@ -78,6 +81,9 @@ BENTOV2_KATSU_APP_SECRET=  # TODO: SET ME WHEN DEPLOYING!
 # Reference
 BENTO_REFERENCE_DB_PASSWORD=  # TODO: SET ME WHEN DEPLOYING!
 
+# MinIO
+BENTO_MINIO_ROOT_PASSWORD= # TODO: SET ME WHEN DEPLOYING!
+
 # HTTPS Certificates --------------------------------------------------
 BENTOV2_CERTS_DIR=${PWD}/certs
 BENTOV2_AUTH_CERTS_DIR=${BENTOV2_CERTS_DIR}/auth

etc/bento_dev.env

@@ -14,6 +14,7 @@ BENTO_BEACON_NETWORK_ENABLED='false'
 BENTO_CBIOPORTAL_ENABLED='false'
 BENTO_GOHAN_ENABLED='true'
 BENTO_MONITORING_ENABLED='false'
+BENTO_MINIO_ENABLED='false'
 
 #  - Switch to enable French translation in Bento Public
 BENTO_PUBLIC_TRANSLATED='true'
@@ -32,6 +33,8 @@ BENTOV2_PORTAL_DOMAIN=portal.${BENTOV2_DOMAIN}
 BENTOV2_AUTH_DOMAIN=bentov2auth.local
 # Unused if cBioPortal is disabled:
 BENTOV2_CBIOPORTAL_DOMAIN=cbioportal.${BENTOV2_DOMAIN}
+# Unused if MinIO is disabled
+BENTO_MINIO_DOMAIN=minio.${BENTOV2_DOMAIN}
 # ---------------------------------------------------------------------
 
 # Authn/z -------------------------------------------------------------
@@ -79,6 +82,9 @@ BENTO_REFERENCE_DB_PASSWORD=devpassword456
 # BENTO_CBIOPORTAL_DATABASE_PASSWORD=somepassword  Required for CBIOPORTAL
 # BENTO_CBIOPORTAL_DATABASE_ROOT_PASSWORD=somepassword  Required for CBIOPORTAL
 
+# MinIO
+BENTO_MINIO_ROOT_PASSWORD=devpassword789
+
 # Development settings ------------------------------------------------
 
 # - Git configuration

etc/default_config.env

@@ -19,6 +19,7 @@ BENTO_BEACON_NETWORK_ENABLED='false'
 BENTO_CBIOPORTAL_ENABLED='false'
 BENTO_GOHAN_ENABLED='true'
 BENTO_MONITORING_ENABLED='false'
+BENTO_MINIO_ENABLED='false'
 
 #  - Switch to enable French translation in Bento Public
 BENTO_PUBLIC_TRANSLATED='true'
@@ -45,6 +46,8 @@ BENTOV2_PORTAL_DOMAIN=portal.${BENTOV2_DOMAIN}
 BENTOV2_AUTH_DOMAIN=bentov2auth.local
 # Unused if cBioPortal is disabled:
 BENTOV2_CBIOPORTAL_DOMAIN=cbioportal.${BENTOV2_DOMAIN}
+# Unused if MinIO is disabled
+BENTO_MINIO_DOMAIN=minio.${BENTOV2_DOMAIN}
 # ---------------------------------------------------------------------
 
 # Certificates --------------------------------------------------------
@@ -62,6 +65,8 @@ BENTOV2_GATEWAY_INTERNAL_CBIOPORTAL_FULLCHAIN_RELATIVE_PATH=/cbioportal_fullchai
 BENTOV2_GATEWAY_INTERNAL_CBIOPORTAL_PRIVKEY_RELATIVE_PATH=/cbioportal_privkey1.key
 BENTO_GATEWAY_INTERNAL_REDIRECT_FULLCHAIN_RELATIVE_PATH=/redirect_fullchain1.crt
 BENTO_GATEWAY_INTERNAL_REDIRECT_PRIVKEY_RELATIVE_PATH=/redirect_privkey1.key
+BENTO_GATEWAY_INTERNAL_MINIO_FULLCHAIN_RELATIVE_PATH=/minio_fullchain1.crt
+BENTO_GATEWAY_INTERNAL_MINIO_PRIVKEY_RELATIVE_PATH=/minio_privkey1.key
 # ---------------------------------------------------------------------
 
 # Authn/z -------------------------------------------------------------
@@ -105,6 +110,9 @@ BENTOV2_KATSU_DB_PASSWORD=
 BENTO_CBIOPORTAL_DATABASE_PASSWORD=
 BENTO_CBIOPORTAL_DATABASE_ROOT_PASSWORD=
 
+# MinIO
+BENTO_MINIO_ROOT_PASSWORD=
+
 # Development settings ------------------------------------------------
 
 # - Git configuration

lib/gateway/docker-compose.gateway.yaml

@@ -22,6 +22,7 @@ services:
       - BENTO_BEACON_ENABLED
       - BENTO_CBIOPORTAL_ENABLED
       - BENTO_GOHAN_ENABLED
+      - BENTO_MINIO_ENABLED
       - BENTO_MONITORING_ENABLED
 
       - BENTOV2_GATEWAY_CONTAINER_NAME
@@ -35,6 +36,8 @@ services:
       - BENTOV2_GATEWAY_INTERNAL_CBIOPORTAL_PRIVKEY_RELATIVE_PATH
       - BENTO_GATEWAY_INTERNAL_REDIRECT_FULLCHAIN_RELATIVE_PATH
       - BENTO_GATEWAY_INTERNAL_REDIRECT_PRIVKEY_RELATIVE_PATH
+      - BENTO_GATEWAY_INTERNAL_MINIO_FULLCHAIN_RELATIVE_PATH
+      - BENTO_GATEWAY_INTERNAL_MINIO_PRIVKEY_RELATIVE_PATH
 
       - BENTOV2_GATEWAY_PUBLIC_ALLOW_FRAME_DOMAINS
 
@@ -78,6 +81,10 @@ services:
       - BENTO_CBIOPORTAL_CONTAINER_NAME
       - BENTO_CBIOPORTAL_INTERNAL_PORT
       - BENTO_GRAFANA_CONTAINER_NAME
+      - BENTO_MINIO_CONTAINER_NAME
+      - BENTO_MINIO_INTERNAL_PORT
+      - BENTO_MINIO_CONSOLE_PORT 
+      - BENTO_MINIO_DOMAIN
     networks:
       - aggregation-net
       - auth-net
@@ -89,6 +96,7 @@ services:
       - event-relay-net
       - gohan-api-net
       - katsu-net
+      - minio-net
       - monitoring-net
       - notification-net
       - public-net
@@ -171,6 +179,9 @@ networks:
   katsu-net:
     external: true
     name: ${BENTO_KATSU_NETWORK}
+  minio-net:
+    external: true
+    name: ${BENTO_MINIO_NETWORK}    
   monitoring-net:
     external: true
     name: ${BENTO_MONITORING_NETWORK}

lib/minio/docker-compose.minio.yaml

@@ -0,0 +1,30 @@
+services:
+  minio:
+    image: ${BENTO_MINIO_IMAGE}:${BENTO_MINIO_IMAGE_VERSION}
+    container_name: ${BENTO_MINIO_CONTAINER_NAME}
+    expose:
+      - ${BENTO_MINIO_INTERNAL_PORT} 
+      - ${BENTO_MINIO_CONSOLE_PORT} 
+    networks:
+      - minio-net
+    volumes:
+      - ${BENTO_MINIO_DATA_DIR}:/data
+    environment:
+      - MINIO_ROOT_USER=${BENTO_MINIO_ROOT_USER}
+      - MINIO_ROOT_PASSWORD=${BENTO_MINIO_ROOT_PASSWORD}
+      - MINIO_DOMAIN=https://${BENTO_MINIO_DOMAIN}
+      - MINIO_BROWSER_REDIRECT_URL=https://${BENTO_MINIO_DOMAIN}/minio/ui
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      timeout: ${BENTO_HEALTHCHECK_TIMEOUT}
+      interval: ${BENTO_HEALTHCHECK_START_INTERVAL}
+      start_period: ${BENTO_HEALTHCHECK_START_PERIOD}
+      start_interval: ${BENTO_HEALTHCHECK_START_INTERVAL}      
+    profiles:
+      - minio
+    command: server /data --console-address ":${BENTO_MINIO_CONSOLE_PORT}"
+
+networks:
+  minio-net:
+    external: true
+    name: ${BENTO_MINIO_NETWORK}

py_bentoctl/config.py

@@ -103,6 +103,8 @@ def __init__(self, enabled: bool, profile: str):
     enabled=_env_get_bool("BENTO_CBIOPORTAL_ENABLED", default=False), profile="cbioportal")
 BENTO_FEATURE_GOHAN = BentoOptionalFeature(
     enabled=_env_get_bool("BENTO_GOHAN_ENABLED", default=False), profile="gohan")
+BENTO_FEATURE_MINIO = BentoOptionalFeature(
+    enabled=_env_get_bool("BENTO_MINIO_ENABLED", default=False), profile="minio")
 BENTO_FEATURE_MONITORING = BentoOptionalFeature(
     enabled=_env_get_bool("BENTO_MONITORING_ENABLED", default=False), profile="monitoring")
 

py_bentoctl/other_helpers.py

@@ -147,6 +147,14 @@ def init_self_signed_certs(force: bool):
             "dir": auth_certs_dir,
         },
 
+        # MinIO
+        **({"minio": {
+            "var": "BENTO_MINIO_DOMAIN",
+            "priv_key_name": "minio_privkey1.key",
+            "crt": "minio_fullchain1.crt",
+            "dir": gateway_certs_dir,
+        }} if c.BENTO_FEATURE_MINIO.enabled else {}),
+
         # If cBioPortal is enabled, generate a cBioPortal self-signed certificate as well
         **({"cbioportal": {
             "var": "BENTOV2_CBIOPORTAL_DOMAIN",
@@ -155,6 +163,7 @@ def init_self_signed_certs(force: bool):
             "dir": gateway_certs_dir,
         }} if c.BENTO_FEATURE_CBIOPORTAL.enabled else {}),
 
+        # If a domain is configured for redirect (e.g. preserve a published reference)
         **({"redirect": {
             "var": "BENTO_DOMAIN_REDIRECT",
             "priv_key_name": "redirect_privkey1.key",
@@ -231,6 +240,8 @@ def init_dirs():
         **({"auth": "BENTOV2_AUTH_VOL_DIR"} if not c.BENTOV2_USE_EXTERNAL_IDP else {}),
         #  - cBioPortal
         **({"cbioportal": "BENTO_CBIOPORTAL_STUDY_DIR"} if c.BENTO_FEATURE_CBIOPORTAL.enabled else {}),
+        #  - MinIO
+        **({"minio": "BENTO_MINIO_DATA_DIR"} if c.BENTO_FEATURE_MINIO.enabled else {}),
         #  - Monitoring: Grafana/Loki
         **({"grafana": "BENTO_GRAFANA_LIB_DIR"} if c.BENTO_FEATURE_MONITORING else {}),
         **({"loki": "BENTO_LOKI_TEMP_DIR"} if c.BENTO_FEATURE_MONITORING else {}),
@@ -299,6 +310,7 @@ def init_docker(client: docker.DockerClient):
         ("BENTO_GOHAN_ES_NETWORK", dict(driver="bridge", internal=True)),  # Does not need to access the web
         ("BENTO_KATSU_NETWORK", dict(driver="bridge")),
         ("BENTO_KATSU_DB_NETWORK", dict(driver="bridge", internal=True)),  # Does not need to access the web
+        ("BENTO_MINIO_NETWORK", dict(driver="bridge")),
         ("BENTO_MONITORING_NETWORK", dict(driver="bridge")),
         ("BENTO_NOTIFICATION_NETWORK", dict(driver="bridge")),
         ("BENTO_PUBLIC_NETWORK", dict(driver="bridge")),