Add descriptions to all configuration variables (#702)

* Add descriptions to all configuration variables

Related to #40

Add descriptions to missing configuration variables in `opal-common`, `opal-client`, and `opal-server` configuration files.

* **opal-common/config.py**:
  - Add descriptions to `LOG_FORMAT_INCLUDE_PID`, `FETCH_PROVIDER_MODULES`, `FETCHING_WORKER_COUNT`, `FETCHING_CALLBACK_TIMEOUT`, `FETCHING_ENQUEUE_TIMEOUT`, `GIT_SSH_KEY_FILE`, `AUTH_PUBLIC_KEY_FORMAT`, `AUTH_PUBLIC_KEY`, `AUTH_JWT_AUDIENCE`, `AUTH_JWT_ISSUER`, `ENABLE_METRICS`.

* **opal-client/config.py**:
  - Add descriptions to `POLICY_STORE_TYPE`, `POLICY_STORE_URL`, `POLICY_STORE_AUTH_TYPE`, `POLICY_STORE_AUTH_TOKEN`, `POLICY_STORE_AUTH_OAUTH_SERVER`, `POLICY_STORE_AUTH_OAUTH_CLIENT_ID`, `POLICY_STORE_AUTH_OAUTH_CLIENT_SECRET`, `POLICY_STORE_CONN_RETRY`, `POLICY_UPDATER_CONN_RETRY`, `DATA_STORE_CONN_RETRY`, `DATA_UPDATER_CONN_RETRY`, `POLICY_STORE_POLICY_PATHS_TO_IGNORE`, `POLICY_STORE_TLS_CLIENT_CERT`, `POLICY_STORE_TLS_CLIENT_KEY`, `POLICY_STORE_TLS_CA`, `EXCLUDE_POLICY_STORE_SECRETS`, `INLINE_OPA_ENABLED`, `INLINE_OPA_CONFIG`, `INLINE_OPA_LOG_FORMAT`, `INLINE_CEDAR_ENABLED`, `INLINE_CEDAR_CONFIG`, `INLINE_CEDAR_LOG_FORMAT`, `KEEP_ALIVE_INTERVAL`, `SERVER_URL`, `SERVER_WS_URL`, `SERVER_PUBSUB_URL`, `CLIENT_TOKEN`, `POLICY_SUBSCRIPTION_DIRS`, `DATA_UPDATER_ENABLED`, `DEFAULT_UPDATE_CALLBACK`, `DEFAULT_UPDATE_CALLBACKS`, `SCOPE_ID`.

* **opal-server/config.py**:
  - Add descriptions to `WS_LOCAL_URL`, `WS_TOKEN`, `BROADCAST_URI`, `BROADCAST_CHANNEL_NAME`, `BROADCAST_CONN_LOSS_BUGFIX_EXPERIMENT_ENABLED`, `AUTH_PRIVATE_KEY_FORMAT`, `AUTH_PRIVATE_KEY_PASSPHRASE`, `AUTH_PRIVATE_KEY`, `AUTH_JWKS_URL`, `AUTH_JWKS_STATIC_DIR`, `AUTH_MASTER_TOKEN`, `POLICY_SOURCE_TYPE`, `POLICY_REPO_REUSE_CLONE_PATH`, `POLICY_REPO_MAIN_BRANCH`, `POLICY_REPO_SSH_KEY`, `POLICY_REPO_MANIFEST_PATH`, `POLICY_REPO_CLONE_TIMEOUT`, `LEADER_LOCK_FILE_PATH`, `POLICY_BUNDLE_SERVER_TYPE`, `REPO_WATCHER_ENABLED`, `PUBLISHER_ENABLED`, `BROADCAST_KEEPALIVE_INTERVAL`, `ALL_DATA_TOPIC`, `ALL_DATA_ROUTE`, `ALL_DATA_URL`, `POLICY_REPO_WEBHOOK_SECRET`, `POLICY_REPO_WEBHOOK_ENFORCE_BRANCH`, `POLICY_REPO_WEBHOOK_PARAMS`, `POLICY_REPO_POLLING_INTERVAL`, `ALLOWED_ORIGINS`, `FILTER_FILE_EXTENSIONS`, `BUNDLE_IGNORE`, `NO_RPC_LOGS`, `SERVER_WORKER_COUNT`, `ENABLE_DATADOG_APM`, `SCOPES`, `SCOPES_REPO_CLONES_SHARDS`, `REDIS_URL`, `BASE_DIR`.

* Add a test in `opal-common/tests/test_config.py` to check for missing descriptions in configuration variables.

* Add descriptions to missing configuration variables and update tests

* **opal-server/config.py**
  - Add example to `FILTER_FILE_EXTENSIONS` description

* **opal-common/tests/test_config.py**
  - Convert tests to use pytest
  - Add tests to check for missing descriptions in configuration variables
  - Cover `opal_common_config`, `opal_client_config`, and `opal_server_config`

packages/opal-client/opal_client/config.py

@@ -20,41 +20,46 @@ class EngineLogFormat(str, Enum):
 class OpalClientConfig(Confi):
     # opa client (policy store) configuration
     POLICY_STORE_TYPE = confi.enum(
-        "POLICY_STORE_TYPE", PolicyStoreTypes, PolicyStoreTypes.OPA
+        "POLICY_STORE_TYPE", PolicyStoreTypes, PolicyStoreTypes.OPA,
+        description="The type of policy store to use (e.g., OPA, Cedar, etc.)"
+    )
+    POLICY_STORE_URL = confi.str(
+        "POLICY_STORE_URL", "http://localhost:8181",
+        description="The URL of the policy store (e.g., OPA agent)."
     )
-    POLICY_STORE_URL = confi.str("POLICY_STORE_URL", "http://localhost:8181")
 
     POLICY_STORE_AUTH_TYPE = confi.enum(
-        "POLICY_STORE_AUTH_TYPE", PolicyStoreAuth, PolicyStoreAuth.NONE
+        "POLICY_STORE_AUTH_TYPE", PolicyStoreAuth, PolicyStoreAuth.NONE,
+        description="The authentication type to use for the policy store (e.g., NONE, TOKEN, etc.)"
     )
     POLICY_STORE_AUTH_TOKEN = confi.str(
         "POLICY_STORE_AUTH_TOKEN",
         None,
-        description="the authentication (bearer) token OPAL client will use to "
+        description="The authentication (bearer) token OPAL client will use to "
         "authenticate against the policy store (i.e: OPA agent).",
     )
     POLICY_STORE_AUTH_OAUTH_SERVER = confi.str(
         "POLICY_STORE_AUTH_OAUTH_SERVER",
         None,
-        description="the authentication server OPAL client will use to authenticate against for retrieving the access_token.",
+        description="The authentication server OPAL client will use to authenticate against for retrieving the access_token.",
     )
     POLICY_STORE_AUTH_OAUTH_CLIENT_ID = confi.str(
         "POLICY_STORE_AUTH_OAUTH_CLIENT_ID",
         None,
-        description="the client_id OPAL will use to authenticate against the OAuth server.",
+        description="The client_id OPAL will use to authenticate against the OAuth server.",
     )
     POLICY_STORE_AUTH_OAUTH_CLIENT_SECRET = confi.str(
         "POLICY_STORE_AUTH_OAUTH_CLIENT_SECRET",
         None,
-        description="the client secret OPAL will use to authenticate against the OAuth server.",
+        description="The client secret OPAL will use to authenticate against the OAuth server.",
     )
 
     POLICY_STORE_CONN_RETRY: ConnRetryOptions = confi.model(
         "POLICY_STORE_CONN_RETRY",
         ConnRetryOptions,
         # defaults are being set according to ConnRetryOptions pydantic definitions (see class)
         {},
-        description="retry options when connecting to the policy store (i.e. the agent that handles the policy, e.g. OPA)",
+        description="Retry options when connecting to the policy store (i.e. the agent that handles the policy, e.g. OPA)",
     )
     POLICY_UPDATER_CONN_RETRY: ConnRetryOptions = confi.model(
         "POLICY_UPDATER_CONN_RETRY",
@@ -65,14 +70,14 @@ class OpalClientConfig(Confi):
             "attempts": 5,
             "wait_time": 1,
         },
-        description="retry options when connecting to the policy source (e.g. the policy bundle server)",
+        description="Retry options when connecting to the policy source (e.g. the policy bundle server)",
     )
 
     DATA_STORE_CONN_RETRY: ConnRetryOptions = confi.model(
         "DATA_STORE_CONN_RETRY",
         ConnRetryOptions,
         None,
-        description="DEPTRECATED - The old confusing name for DATA_UPDATER_CONN_RETRY, kept for backwards compatibilit (for now)",
+        description="DEPRECATED - The old confusing name for DATA_UPDATER_CONN_RETRY, kept for backwards compatibility (for now)",
     )
 
     DATA_UPDATER_CONN_RETRY: ConnRetryOptions = confi.model(
@@ -84,7 +89,7 @@ class OpalClientConfig(Confi):
             "attempts": 5,
             "wait_time": 1,
         },
-        description="retry options when connecting to the base data source (e.g. an external API server which returns data snapshot)",
+        description="Retry options when connecting to the base data source (e.g. an external API server which returns data snapshot)",
     )
 
     POLICY_STORE_POLICY_PATHS_TO_IGNORE = confi.list(
@@ -102,17 +107,17 @@ class OpalClientConfig(Confi):
     POLICY_STORE_TLS_CLIENT_CERT = confi.str(
         "POLICY_STORE_TLS_CLIENT_CERT",
         None,
-        description="path to the client certificate used for tls authentication with the policy store",
+        description="Path to the client certificate used for TLS authentication with the policy store",
     )
     POLICY_STORE_TLS_CLIENT_KEY = confi.str(
         "POLICY_STORE_TLS_CLIENT_KEY",
         None,
-        description="path to the client key used for tls authentication with the policy store",
+        description="Path to the client key used for TLS authentication with the policy store",
     )
     POLICY_STORE_TLS_CA = confi.str(
         "POLICY_STORE_TLS_CA",
         None,
-        description="path to the file containing the ca certificate(s) used for tls authentication with the policy store",
+        description="Path to the file containing the CA certificate(s) used for TLS authentication with the policy store",
     )
 
     EXCLUDE_POLICY_STORE_SECRETS = confi.bool(
@@ -132,7 +137,10 @@ def load_policy_store():
     # opa runner configuration (OPA can optionally be run by OPAL) ----------------
 
     # whether or not OPAL should run OPA by itself in the same container
-    INLINE_OPA_ENABLED = confi.bool("INLINE_OPA_ENABLED", True)
+    INLINE_OPA_ENABLED = confi.bool(
+        "INLINE_OPA_ENABLED", True,
+        description="Whether or not OPAL should run OPA by itself in the same container"
+    )
 
     INLINE_OPA_EXEC_PATH = confi.str(
         "INLINE_OPA_EXEC_PATH",
@@ -146,17 +154,21 @@ def load_policy_store():
         "INLINE_OPA_CONFIG",
         OpaServerOptions,
         {},  # defaults are being set according to OpaServerOptions pydantic definitions (see class)
-        description="cli options used when running `opa run --server` inline",
+        description="CLI options used when running `opa run --server` inline",
     )
 
     INLINE_OPA_LOG_FORMAT: EngineLogFormat = confi.enum(
-        "INLINE_OPA_LOG_FORMAT", EngineLogFormat, EngineLogFormat.NONE
+        "INLINE_OPA_LOG_FORMAT", EngineLogFormat, EngineLogFormat.NONE,
+        description="The log format to use for inline OPA logs"
     )
 
     # Cedar runner configuration (Cedar-engine can optionally be run by OPAL) ----------------
 
     # whether or not OPAL should run the Cedar agent by itself in the same container
-    INLINE_CEDAR_ENABLED = confi.bool("INLINE_CEDAR_ENABLED", True)
+    INLINE_CEDAR_ENABLED = confi.bool(
+        "INLINE_CEDAR_ENABLED", True,
+        description="Whether or not OPAL should run the Cedar agent by itself in the same container"
+    )
 
     INLINE_CEDAR_EXEC_PATH = confi.str(
         "INLINE_CEDAR_EXEC_PATH",
@@ -170,23 +182,30 @@ def load_policy_store():
         "INLINE_CEDAR_CONFIG",
         CedarServerOptions,
         {},  # defaults are being set according to CedarServerOptions pydantic definitions (see class)
-        description="cli options used when running the Cedar agent inline",
+        description="CLI options used when running the Cedar agent inline",
     )
 
     INLINE_CEDAR_LOG_FORMAT: EngineLogFormat = confi.enum(
-        "INLINE_CEDAR_LOG_FORMAT", EngineLogFormat, EngineLogFormat.NONE
+        "INLINE_CEDAR_LOG_FORMAT", EngineLogFormat, EngineLogFormat.NONE,
+        description="The log format to use for inline Cedar logs"
     )
 
     # configuration for fastapi routes
     ALLOWED_ORIGINS = ["*"]
 
     # general configuration for pub/sub clients
-    KEEP_ALIVE_INTERVAL = confi.int("KEEP_ALIVE_INTERVAL", 0)
+    KEEP_ALIVE_INTERVAL = confi.int(
+        "KEEP_ALIVE_INTERVAL", 0,
+        description="The interval (in seconds) for sending keep-alive messages"
+    )
 
     # Opal Server general configuration -------------------------------------------
 
     # opal server url
-    SERVER_URL = confi.str("SERVER_URL", "http://localhost:7002", flags=["-s"])
+    SERVER_URL = confi.str(
+        "SERVER_URL", "http://localhost:7002", flags=["-s"],
+        description="The URL of the OPAL server"
+    )
     # opal server pubsub url
     OPAL_WS_ROUTE = "/ws"
     SERVER_WS_URL = confi.str(
@@ -196,16 +215,18 @@ def load_policy_store():
                 "http", "ws"
             )
         ),
+        description="The WebSocket URL of the OPAL server"
     )
     SERVER_PUBSUB_URL = confi.str(
-        "SERVER_PUBSUB_URL", confi.delay("{SERVER_WS_URL}" + f"{OPAL_WS_ROUTE}")
+        "SERVER_PUBSUB_URL", confi.delay("{SERVER_WS_URL}" + f"{OPAL_WS_ROUTE}"),
+        description="The Pub/Sub URL of the OPAL server"
     )
 
     # opal server auth token
     CLIENT_TOKEN = confi.str(
         "CLIENT_TOKEN",
         "THIS_IS_A_DEV_SECRET",
-        description="opal server auth token",
+        description="The authentication token for the OPAL server",
         flags=["-t"],
     )
 
@@ -241,7 +262,7 @@ def load_policy_store():
         "POLICY_SUBSCRIPTION_DIRS",
         ["."],
         delimiter=":",
-        description="directories in policy repo we should subscribe to",
+        description="Directories in the policy repo to subscribe to for policy code (rego) modules",
     )
 
     # Data updater configuration --------------------------------------------------
@@ -284,6 +305,7 @@ def load_policy_store():
             "headers": {"content-type": "application/json"},
             "process_data": False,
         },
+        description="Configuration for the default update callback",
     )
 
     DEFAULT_UPDATE_CALLBACKS = confi.model(
@@ -317,7 +339,10 @@ def load_policy_store():
 
     OPA_HEALTH_CHECK_POLICY_PATH = "engine/healthcheck/opal.rego"
 
-    SCOPE_ID = confi.str("SCOPE_ID", "default", description="OPAL Scope ID")
+    SCOPE_ID = confi.str(
+        "SCOPE_ID", "default",
+        description="OPAL Scope ID"
+    )
 
     STORE_BACKUP_PATH = confi.str(
         "STORE_BACKUP_PATH",

packages/opal-common/opal_common/config.py

@@ -16,7 +16,9 @@ class OpalCommonConfig(Confi):
     PROCESS_NAME = ""
     # Logging
     # - Log formatting
-    LOG_FORMAT_INCLUDE_PID = confi.bool("LOG_FORMAT_INCLUDE_PID", False)
+    LOG_FORMAT_INCLUDE_PID = confi.bool(
+        "LOG_FORMAT_INCLUDE_PID", False, description="Include process ID in log format"
+    )
     LOG_FORMAT = confi.str(
         "LOG_FORMAT",
         confi.delay(
@@ -106,19 +108,27 @@ class OpalCommonConfig(Confi):
     # Fetching Providers
     # - where to load providers from
     FETCH_PROVIDER_MODULES = confi.list(
-        "FETCH_PROVIDER_MODULES", ["opal_common.fetcher.providers"]
+        "FETCH_PROVIDER_MODULES", ["opal_common.fetcher.providers"],
+        description="List of modules to load fetch providers from"
     )
 
     # Fetching engine
     # Max number of worker tasks handling fetch events concurrently
-    FETCHING_WORKER_COUNT = confi.int("FETCHING_WORKER_COUNT", 6)
+    FETCHING_WORKER_COUNT = confi.int(
+        "FETCHING_WORKER_COUNT", 6, description="Max number of worker tasks handling fetch events concurrently"
+    )
     # Time in seconds to wait on the queued fetch task.
-    FETCHING_CALLBACK_TIMEOUT = confi.int("FETCHING_CALLBACK_TIMEOUT", 10)
+    FETCHING_CALLBACK_TIMEOUT = confi.int(
+        "FETCHING_CALLBACK_TIMEOUT", 10, description="Time in seconds to wait on the queued fetch task"
+    )
     # Time in seconds to wait for queuing a new task (if the queue is full)
-    FETCHING_ENQUEUE_TIMEOUT = confi.int("FETCHING_ENQUEUE_TIMEOUT", 10)
+    FETCHING_ENQUEUE_TIMEOUT = confi.int(
+        "FETCHING_ENQUEUE_TIMEOUT", 10, description="Time in seconds to wait for queuing a new task (if the queue is full)"
+    )
 
     GIT_SSH_KEY_FILE = confi.str(
-        "GIT_SSH_KEY_FILE", str(Path.home() / ".ssh/opal_repo_ssh_key")
+        "GIT_SSH_KEY_FILE", str(Path.home() / ".ssh/opal_repo_ssh_key"),
+        description="Path to the SSH key file for Git"
     )
 
     # Trust self signed certificates (Advanced Usage - only affects OPAL client) -----------------------------
@@ -139,11 +149,13 @@ class OpalCommonConfig(Confi):
 
     # security
     AUTH_PUBLIC_KEY_FORMAT = confi.enum(
-        "AUTH_PUBLIC_KEY_FORMAT", EncryptionKeyFormat, EncryptionKeyFormat.ssh
+        "AUTH_PUBLIC_KEY_FORMAT", EncryptionKeyFormat, EncryptionKeyFormat.ssh,
+        description="Format of the public key for authentication"
     )
     AUTH_PUBLIC_KEY = confi.delay(
         lambda AUTH_PUBLIC_KEY_FORMAT=None: confi.public_key(
-            "AUTH_PUBLIC_KEY", default=None, key_format=AUTH_PUBLIC_KEY_FORMAT
+            "AUTH_PUBLIC_KEY", default=None, key_format=AUTH_PUBLIC_KEY_FORMAT,
+            description="Public key for authentication"
         )
     )
     AUTH_JWT_ALGORITHM = confi.enum(
@@ -152,15 +164,23 @@ class OpalCommonConfig(Confi):
         getattr(JWTAlgorithm, "RS256"),
         description="jwt algorithm, possible values: see: https://pyjwt.readthedocs.io/en/stable/algorithms.html",
     )
-    AUTH_JWT_AUDIENCE = confi.str("AUTH_JWT_AUDIENCE", "https://api.opal.ac/v1/")
-    AUTH_JWT_ISSUER = confi.str("AUTH_JWT_ISSUER", f"https://opal.ac/")
+    AUTH_JWT_AUDIENCE = confi.str(
+        "AUTH_JWT_AUDIENCE", "https://api.opal.ac/v1/",
+        description="Audience for JWT authentication"
+    )
+    AUTH_JWT_ISSUER = confi.str(
+        "AUTH_JWT_ISSUER", f"https://opal.ac/",
+        description="Issuer for JWT authentication"
+    )
     POLICY_REPO_POLICY_EXTENSIONS = confi.list(
         "POLICY_REPO_POLICY_EXTENSIONS",
         [".rego"],
         description="List of extensions to serve as policy modules",
     )
 
-    ENABLE_METRICS = confi.bool("ENABLE_METRICS", False)
+    ENABLE_METRICS = confi.bool(
+        "ENABLE_METRICS", False, description="Enable metrics collection"
+    )
 
     # optional APM tracing with datadog
     ENABLE_DATADOG_APM = confi.bool(

packages/opal-common/opal_common/tests/test_config.py

@@ -0,0 +1,16 @@
+import pytest
+from opal_common.config import opal_common_config
+from opal_client.config import opal_client_config
+from opal_server.config import opal_server_config
+
+def test_opal_common_config_descriptions():
+    for name, entry in opal_common_config.entries.items():
+        assert entry.description is not None, f"{name} is missing a description"
+
+def test_opal_client_config_descriptions():
+    for name, entry in opal_client_config.entries.items():
+        assert entry.description is not None, f"{name} is missing a description"
+
+def test_opal_server_config_descriptions():
+    for name, entry in opal_server_config.entries.items():
+        assert entry.description is not None, f"{name} is missing a description"