From fadbb19eb8c2875ad2d499765507106dda524578 Mon Sep 17 00:00:00 2001 From: eternaltyro Date: Thu, 22 Feb 2024 09:17:38 +0000 Subject: [PATCH] Update config.py to allow for remote credentials Deployment of RAW Data API on cloud services offers the advantages of passing credentials (Database, OSM App OAuth2, HDX, etc.) securely via environment variables to the container. The credentials are passed from AWS Secrets Manager, Azure Key Vault or equivalent cloud services. To make management of secrets easier, credentials that relate to a single service are grouped into a single JSON formatted entry. For example, PostgreSQL hostname, port, user, password can be supplied via a single envvar. This change adds support for passing the following credentials: 1. OSM App OAuth2 credentials 2. PostgreSQL database credentials 3. HDX credentials The envvar key is similar to the section headers in the .env file with `REMOTE_` prefix. --- src/config.py | 152 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 112 insertions(+), 40 deletions(-) diff --git a/src/config.py b/src/config.py index 89bc6d68..9f1ab3f9 100644 --- a/src/config.py +++ b/src/config.py @@ -87,7 +87,7 @@ def not_raises(func, *args, **kwargs): #################### -### EXPORT_UPLOAD CONFIG BLOCK +# EXPORT_UPLOAD CONFIG BLOCK FILE_UPLOAD_METHOD = os.environ.get("FILE_UPLOAD_METHOD") or config.get( "EXPORT_UPLOAD", "FILE_UPLOAD_METHOD", fallback="disk" ) @@ -173,7 +173,7 @@ def not_raises(func, *args, **kwargs): config.getboolean("API_CONFIG", "USE_CONNECTION_POOLING", fallback=False), ) -## Queue +# Queue DEFAULT_QUEUE_NAME = os.environ.get("DEFAULT_QUEUE_NAME") or config.get( "API_CONFIG", "DEFAULT_QUEUE_NAME", fallback="raw_ondemand" @@ -182,7 +182,7 @@ def not_raises(func, *args, **kwargs): "API_CONFIG", "DAEMON_QUEUE_NAME", fallback="raw_daemon" ) -### Polygon statistics which will deliver the stats of approx buildings/ roads in the area +# Polygon statistics which will deliver the stats of approx buildings/ roads in the area ENABLE_POLYGON_STATISTICS_ENDPOINTS = get_bool_env_var( "ENABLE_POLYGON_STATISTICS_ENDPOINTS", @@ -198,7 +198,7 @@ def not_raises(func, *args, **kwargs): "POLYGON_STATISTICS_API_RATE_LIMIT" ) or config.get("API_CONFIG", "POLYGON_STATISTICS_API_RATE_LIMIT", fallback=5) -## task limit +# task limit DEFAULT_SOFT_TASK_LIMIT = os.environ.get("DEFAULT_SOFT_TASK_LIMIT") or config.get( "API_CONFIG", "DEFAULT_SOFT_TASK_LIMIT", fallback=2 * 60 * 60 @@ -207,7 +207,7 @@ def not_raises(func, *args, **kwargs): "API_CONFIG", "DEFAULT_HARD_TASK_LIMIT", fallback=3 * 60 * 60 ) -## duckdb +# duckdb USE_DUCK_DB_FOR_CUSTOM_EXPORTS = get_bool_env_var( "USE_DUCK_DB_FOR_CUSTOM_EXPORTS", @@ -226,7 +226,7 @@ def not_raises(func, *args, **kwargs): "API_CONFIG", "DUCK_DB_THREAD_LIMIT", fallback=None ) -## hdx and custom exports +# hdx and custom exports ENABLE_CUSTOM_EXPORTS = get_bool_env_var( "ENABLE_CUSTOM_EXPORTS", config.getboolean("API_CONFIG", "ENABLE_CUSTOM_EXPORTS", fallback=False), @@ -255,18 +255,38 @@ def not_raises(func, *args, **kwargs): if ENABLE_HDX_EXPORTS: - HDX_SITE = os.environ.get("HDX_SITE") or config.get( - "HDX", "HDX_SITE", fallback="demo" - ) - HDX_API_KEY = os.environ.get("HDX_API_KEY") or config.get( - "HDX", "HDX_API_KEY", fallback=None - ) - HDX_OWNER_ORG = os.environ.get("HDX_OWNER_ORG") or config.get( - "HDX", "HDX_OWNER_ORG", fallback="225b9f7d-e7cb-4156-96a6-44c9c58d31e3" - ) - HDX_MAINTAINER = os.environ.get("HDX_MAINTAINER") or config.get( - "HDX", "HDX_MAINTAINER", fallback=None - ) + try: + hdx_credentials = os.environ["REMOTE_HDX"] + + except KeyError: + print("EnvVar: REMOTE_HDX not supplied; Falling back to other means") + HDX_SITE = os.environ.get("HDX_SITE") or config.get( + "HDX", "HDX_SITE", fallback="demo" + ) + HDX_API_KEY = os.environ.get("HDX_API_KEY") or config.get( + "HDX", "HDX_API_KEY", fallback=None + ) + HDX_OWNER_ORG = os.environ.get("HDX_OWNER_ORG") or config.get( + "HDX", "HDX_OWNER_ORG", fallback="225b9f7d-e7cb-4156-96a6-44c9c58d31e3" + ) + HDX_MAINTAINER = os.environ.get("HDX_MAINTAINER") or config.get( + "HDX", "HDX_MAINTAINER", fallback=None + ) + + else: + import json + + hdx_credentials_json = json.loads(hdx_credentials) + + HDX_SITE = hdx_credentials_json["HDX_SITE"] + HDX_API_KEY = hdx_credentials_json["HDX_API_KEY"] + HDX_OWNER_ORG = hdx_credentials_json["HDX_OWNER_ORG"] + HDX_MAINTAINER = hdx_credentials_json["HDX_MAINTAINER"] + + if None in (HDX_SITE, HDX_API_KEY, HDX_OWNER_ORG, HDX_MAINTAINER): + raise ValueError("HDX Remote Credentials Malformed") + logging.error("HDX Remote Credentials Malformed") + from hdx.api.configuration import Configuration try: @@ -312,50 +332,100 @@ def get_db_connection_params() -> dict: """Return a python dict that can be passed to psycopg2 connections to authenticate to Postgres Databases - Returns: connection_params (dict): PostgreSQL connection parameters corresponding to the configuration section. """ + # This block fetches PostgreSQL (database) credentials passed as + # environment variables as a JSON object, from AWS Secrets Manager or + # Azure Key Vault. try: - connection_params = { - "host": os.environ.get("PGHOST") or config.get("DB", "PGHOST"), - "port": os.environ.get("PGPORT") + db_credentials = os.environ["REMOTE_DB"] + + except KeyError: + print("EnvVar: REMOTE_DB not supplied; Falling back to other means") + + connection_params = dict( + host=os.environ.get("PGHOST") or config.get("DB", "PGHOST"), + port=os.environ.get("PGPORT") or config.get("DB", "PGPORT", fallback="5432"), - "dbname": os.environ.get("PGDATABASE") or config.get("DB", "PGDATABASE"), - "user": os.environ.get("PGUSER") or config.get("DB", "PGUSER"), - "password": os.environ.get("PGPASSWORD") or config.get("DB", "PGPASSWORD"), - } - if any(value is None for value in connection_params.values()): - raise ValueError( - "Connection Params Value Error : Couldn't be Loaded , Check DB Credentials" - ) - except Exception as ex: + dbname=os.environ.get("PGDATABASE") or config.get("DB", "PGDATABASE"), + user=os.environ.get("PGUSER") or config.get("DB", "PGUSER"), + password=os.environ.get("PGPASSWORD") or config.get("DB", "PGPASSWORD"), + ) + + else: + import json + + connection_params = json.loads(db_credentials) + + connection_params["username"] = connection_params["user"] + for k in ("dbinstanceidentifier", "engine", "user"): + connection_params.pop(k, None) + + if None in connection_params.values(): + raise ValueError( + "Connection Params Value Error : Couldn't be Loaded , Check DB Credentials" + ) logging.error( "Can't find database credentials , Either export them as env variable or include in config Block DB" ) - raise ex + return connection_params def get_oauth_credentials() -> tuple: - """Gets oauth credentials from the env file and returns a config dict""" - osm_url = os.environ.get("OSM_URL") or config.get( - "OAUTH", "OSM_URL", fallback="https://www.openstreetmap.org" - ) + """Get OAuth2 credentials from env file and return a config dict + + Return an ordered python tuple that can be passed to functions that + authenticate to OSM. + + Order of precedence: + 1. Environment Variables + 2. Config File + 3. Default fallback + + Returns: oauth2_credentials (tuple): Tuple containing OAuth2 client + secret, client ID, and redirect URL. + + """ + client_id = os.environ.get("OSM_CLIENT_ID") or config.get("OAUTH", "OSM_CLIENT_ID") client_secret = os.environ.get("OSM_CLIENT_SECRET") or config.get( "OAUTH", "OSM_CLIENT_SECRET" ) - secret_key = os.environ.get("APP_SECRET_KEY") or config.get( - "OAUTH", "APP_SECRET_KEY", fallback="development" - ) login_redirect_uri = os.environ.get("LOGIN_REDIRECT_URI") or config.get( "OAUTH", "LOGIN_REDIRECT_URI", fallback="http://127.0.0.1:8000/v1/auth/callback" ) scope = os.environ.get("OSM_PERMISSION_SCOPE") or config.get( "OAUTH", "OSM_PERMISSION_SCOPE", fallback="read_prefs" ) + osm_url = os.environ.get("OSM_URL") or config.get( + "OAUTH", "OSM_URL", fallback="https://www.openstreetmap.org" + ) + secret_key = os.environ.get("APP_SECRET_KEY") or config.get( + "OAUTH", "APP_SECRET_KEY" + ) + + # This block fetches OSM OAuth2 app credentials passed as + # environment variables as a JSON object, from AWS Secrets Manager or + # Azure Key Vault. + try: + oauth2_credentials = os.environ["REMOTE_OAUTH"] + + except KeyError: + print("EnvVar: REMOTE_OAUTH not supplied; Falling back to other means") + + else: + import json + + oauth2_credentials_json = json.loads(oauth2_credentials) + + client_id = oauth2_credentials_json["OSM_CLIENT_ID"] + client_secret = oauth2_credentials_json["OSM_CLIENT_SECRET"] + login_redirect_uri = oauth2_credentials_json["LOGIN_REDIRECT_URI"] + scope = oauth2_credentials_json["OSM_PERMISSION_SCOPE"] + oauth_cred = ( osm_url, client_id, @@ -364,7 +434,9 @@ def get_oauth_credentials() -> tuple: login_redirect_uri, scope, ) - if any(item is None for item in oauth_cred): + + if None in oauth_cred: raise ValueError("Oauth Credentials can't be loaded") + logging.error("Malformed OSM OAuth2 App credentials") return oauth_cred