From f2388b5cfe7cef9a9ada8f6f6843a3c51062761f Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Wed, 20 Sep 2023 17:42:08 +0200 Subject: [PATCH 1/6] Add environment variables for JWT arguments and auth token --- rasa/cli/arguments/run.py | 49 ++++++--- rasa/env.py | 5 + tests/cli/arguments/__init__.py | 0 tests/cli/arguments/test_run.py | 188 ++++++++++++++++++++++++++++++++ 4 files changed, 229 insertions(+), 13 deletions(-) create mode 100644 rasa/env.py create mode 100644 tests/cli/arguments/__init__.py create mode 100644 tests/cli/arguments/test_run.py diff --git a/rasa/cli/arguments/run.py b/rasa/cli/arguments/run.py index f982672700d1..2f807ad74718 100644 --- a/rasa/cli/arguments/run.py +++ b/rasa/cli/arguments/run.py @@ -1,8 +1,11 @@ +import os + import argparse from typing import Union from rasa.cli.arguments.default_arguments import add_model_param, add_endpoint_param from rasa.core import constants +from rasa.env import DEFAULT_JWT_METHOD, JWT_METHOD_ENV, JWT_SECRET_ENV, JWT_PRIVATE_KEY_ENV, AUTH_TOKEN_ENV def set_run_arguments(parser: argparse.ArgumentParser) -> None: @@ -82,18 +85,27 @@ def add_server_arguments(parser: argparse.ArgumentParser) -> None: "yml file.", ) + add_server_settings_arguments(parser) + + +def add_server_settings_arguments(parser: argparse.ArgumentParser) -> None: + """Add arguments for the API server. + + Args: + parser: Argument parser. + """ server_arguments = parser.add_argument_group("Server Settings") add_interface_argument(server_arguments) - add_port_argument(server_arguments) server_arguments.add_argument( "-t", "--auth-token", type=str, + default=os.getenv(AUTH_TOKEN_ENV), help="Enable token based authentication. Requests need to provide " - "the token to be accepted.", + "the token to be accepted.", ) server_arguments.add_argument( "--cors", @@ -132,13 +144,13 @@ def add_server_arguments(parser: argparse.ArgumentParser) -> None: server_arguments.add_argument( "--ssl-ca-file", help="If your SSL certificate needs to be verified, " - "you can specify the CA file " - "using this parameter.", + "you can specify the CA file " + "using this parameter.", ) server_arguments.add_argument( "--ssl-password", help="If your ssl-keyfile is protected by a password, you can specify it " - "using this paramer.", + "using this paramer.", ) channel_arguments = parser.add_argument_group("Channels") channel_arguments.add_argument( @@ -150,26 +162,37 @@ def add_server_arguments(parser: argparse.ArgumentParser) -> None: "--connector", type=str, help="Service to connect to." ) + add_jwt_arguments(parser) + + +def add_jwt_arguments(parser: argparse.ArgumentParser) -> None: + """Adds arguments related to JWT authentication. + + Args: + parser: Argument parser. + """ jwt_auth = parser.add_argument_group("JWT Authentication") jwt_auth.add_argument( "--jwt-secret", type=str, + default=os.getenv(JWT_SECRET_ENV), help="Public key for asymmetric JWT methods or shared secret" - "for symmetric methods. Please also make sure to use " - "--jwt-method to select the method of the signature, " - "otherwise this argument will be ignored." - "Note that this key is meant for securing the HTTP API.", + "for symmetric methods. Please also make sure to use " + "--jwt-method to select the method of the signature, " + "otherwise this argument will be ignored." + "Note that this key is meant for securing the HTTP API.", ) jwt_auth.add_argument( "--jwt-method", type=str, - default="HS256", + default=os.getenv(JWT_METHOD_ENV, DEFAULT_JWT_METHOD), help="Method used for the signature of the JWT authentication payload.", ) jwt_auth.add_argument( "--jwt-private-key", type=str, + default=os.getenv(JWT_PRIVATE_KEY_ENV), help="A private key used for generating web tokens, dependent upon " - "which hashing algorithm is used. It must be used together with " - "--jwt-secret for providing the public key.", - ) + "which hashing algorithm is used. It must be used together with " + "--jwt-secret for providing the public key.", + ) \ No newline at end of file diff --git a/rasa/env.py b/rasa/env.py new file mode 100644 index 000000000000..3415487c3807 --- /dev/null +++ b/rasa/env.py @@ -0,0 +1,5 @@ +AUTH_TOKEN_ENV = "AUTH_TOKEN" +JWT_SECRET_ENV = "JWT_SECRET" +JWT_METHOD_ENV = "JWT_METHOD" +DEFAULT_JWT_METHOD = "HS256" +JWT_PRIVATE_KEY_ENV = "JWT_PRIVATE_KEY" diff --git a/tests/cli/arguments/__init__.py b/tests/cli/arguments/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/cli/arguments/test_run.py b/tests/cli/arguments/test_run.py new file mode 100644 index 000000000000..5d555e6db2b2 --- /dev/null +++ b/tests/cli/arguments/test_run.py @@ -0,0 +1,188 @@ +from typing import List, Dict + +import argparse +import pytest +from _pytest.monkeypatch import MonkeyPatch + +from rasa.cli.arguments.run import add_jwt_arguments, add_server_settings_arguments +from rasa.env import ( + JWT_SECRET_ENV, + JWT_METHOD_ENV, + JWT_PRIVATE_KEY_ENV, + DEFAULT_JWT_METHOD, + AUTH_TOKEN_ENV, +) + + +@pytest.mark.parametrize( + "env_variables, input_args, expected", + [ + ( + # all env variables are set + { + JWT_SECRET_ENV: "secret", + JWT_METHOD_ENV: "HS256", + JWT_PRIVATE_KEY_ENV: "private_key", + }, + [], + argparse.Namespace( + jwt_secret="secret", + jwt_method="HS256", + jwt_private_key="private_key", + ), + ), + ( + # no JWT_SECRET_ENV and --jwt-secret is set + { + JWT_METHOD_ENV: "HS256", + JWT_PRIVATE_KEY_ENV: "private_key", + }, + ["--jwt-secret", "secret"], + argparse.Namespace( + jwt_secret="secret", + jwt_method="HS256", + jwt_private_key="private_key", + ), + ), + ( + # no JWT_METHOD_ENV and --jwt-method is set + { + JWT_SECRET_ENV: "secret", + JWT_PRIVATE_KEY_ENV: "private_key", + }, + ["--jwt-method", "HS256"], + argparse.Namespace( + jwt_secret="secret", + jwt_method="HS256", + jwt_private_key="private_key", + ), + ), + ( + # no JWT_PRIVATE_KEY_ENV and --jwt-private-key is set + { + JWT_SECRET_ENV: "secret", + JWT_METHOD_ENV: "HS256", + }, + ["--jwt-private-key", "private_key"], + argparse.Namespace( + jwt_secret="secret", + jwt_method="HS256", + jwt_private_key="private_key", + ), + ), + ( + # no JWT_SECRET_ENV and no --jwt-secret + { + JWT_METHOD_ENV: "HS256", + JWT_PRIVATE_KEY_ENV: "private_key", + }, + [], + argparse.Namespace( + jwt_secret=None, + jwt_method="HS256", + jwt_private_key="private_key", + ), + ), + ( + # no JWT_METHOD_ENV and no --jwt-method + { + JWT_SECRET_ENV: "secret", + JWT_PRIVATE_KEY_ENV: "private_key", + }, + [], + argparse.Namespace( + jwt_secret="secret", + jwt_method=DEFAULT_JWT_METHOD, + jwt_private_key="private_key", + ), + ), + ( + # no JWT_PRIVATE_KEY_ENV and no --jwt-private-key + { + JWT_SECRET_ENV: "secret", + JWT_METHOD_ENV: "HS256", + }, + [], + argparse.Namespace( + jwt_secret="secret", + jwt_method="HS256", + jwt_private_key=None, + ), + ), + ( + # no env variables and no arguments + {}, + [], + argparse.Namespace( + jwt_secret=None, + jwt_method="HS256", + jwt_private_key=None, + ), + ), + ], +) +def test_jwt_argument_parsing( + env_variables: Dict[str, str], + input_args: List[str], + expected: argparse.Namespace, + monkeypatch: MonkeyPatch, +) -> None: + """Tests parsing of the JWT arguments.""" + parser = argparse.ArgumentParser() + + for env_name, env_value in env_variables.items(): + monkeypatch.setenv(env_name, env_value) + + add_jwt_arguments(parser) + args = parser.parse_args(input_args) + + assert args.jwt_secret == expected.jwt_secret + assert args.jwt_method == expected.jwt_method + assert args.jwt_private_key == expected.jwt_private_key + + +@pytest.mark.parametrize( + "env_variables, input_args, expected", + [ + ( + { + AUTH_TOKEN_ENV: "secret", + }, + [], + argparse.Namespace( + auth_token="secret", + ), + ), + ( + {}, + ["--auth-token", "secret"], + argparse.Namespace( + auth_token="secret", + ), + ), + ( + {}, + [], + argparse.Namespace( + auth_token=None, + ), + ), + ], +) +def test_add_server_settings_arguments( + env_variables: Dict[str, str], + input_args: List[str], + expected: argparse.Namespace, + monkeypatch: MonkeyPatch, +) -> None: + """Tests parsing of the server settings arguments.""" + parser = argparse.ArgumentParser() + + for env_name, env_value in env_variables.items(): + monkeypatch.setenv(env_name, env_value) + + add_server_settings_arguments(parser) + + args = parser.parse_args(input_args) + + assert args.auth_token == expected.auth_token From 8f136822706909eab13cdbac7c4cf7c86a238270 Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Wed, 20 Sep 2023 18:39:27 +0200 Subject: [PATCH 2/6] Add changelog for JWT token and auth token environment variables --- changelog/1557.improvement.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changelog/1557.improvement.md diff --git a/changelog/1557.improvement.md b/changelog/1557.improvement.md new file mode 100644 index 000000000000..87f4b1e62a88 --- /dev/null +++ b/changelog/1557.improvement.md @@ -0,0 +1,8 @@ +Added environment variables to configure JWT and auth token. +For JWT the following environment variables are available: +- JWT_SECRET +- JWT_METHOD +- JWT_PRIVATE_KEY + +For auth token the following environment variable is available: +- AUTH_TOKEN From 28f295f5a300693354277c5bcb8f05ede68a1b8a Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Wed, 20 Sep 2023 18:54:07 +0200 Subject: [PATCH 3/6] Format run.py --- rasa/cli/arguments/run.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/rasa/cli/arguments/run.py b/rasa/cli/arguments/run.py index 2f807ad74718..14ecba19ad8e 100644 --- a/rasa/cli/arguments/run.py +++ b/rasa/cli/arguments/run.py @@ -5,7 +5,13 @@ from rasa.cli.arguments.default_arguments import add_model_param, add_endpoint_param from rasa.core import constants -from rasa.env import DEFAULT_JWT_METHOD, JWT_METHOD_ENV, JWT_SECRET_ENV, JWT_PRIVATE_KEY_ENV, AUTH_TOKEN_ENV +from rasa.env import ( + DEFAULT_JWT_METHOD, + JWT_METHOD_ENV, + JWT_SECRET_ENV, + JWT_PRIVATE_KEY_ENV, + AUTH_TOKEN_ENV, +) def set_run_arguments(parser: argparse.ArgumentParser) -> None: @@ -105,7 +111,7 @@ def add_server_settings_arguments(parser: argparse.ArgumentParser) -> None: type=str, default=os.getenv(AUTH_TOKEN_ENV), help="Enable token based authentication. Requests need to provide " - "the token to be accepted.", + "the token to be accepted.", ) server_arguments.add_argument( "--cors", @@ -144,13 +150,13 @@ def add_server_settings_arguments(parser: argparse.ArgumentParser) -> None: server_arguments.add_argument( "--ssl-ca-file", help="If your SSL certificate needs to be verified, " - "you can specify the CA file " - "using this parameter.", + "you can specify the CA file " + "using this parameter.", ) server_arguments.add_argument( "--ssl-password", help="If your ssl-keyfile is protected by a password, you can specify it " - "using this paramer.", + "using this paramer.", ) channel_arguments = parser.add_argument_group("Channels") channel_arguments.add_argument( @@ -177,10 +183,10 @@ def add_jwt_arguments(parser: argparse.ArgumentParser) -> None: type=str, default=os.getenv(JWT_SECRET_ENV), help="Public key for asymmetric JWT methods or shared secret" - "for symmetric methods. Please also make sure to use " - "--jwt-method to select the method of the signature, " - "otherwise this argument will be ignored." - "Note that this key is meant for securing the HTTP API.", + "for symmetric methods. Please also make sure to use " + "--jwt-method to select the method of the signature, " + "otherwise this argument will be ignored." + "Note that this key is meant for securing the HTTP API.", ) jwt_auth.add_argument( "--jwt-method", @@ -193,6 +199,6 @@ def add_jwt_arguments(parser: argparse.ArgumentParser) -> None: type=str, default=os.getenv(JWT_PRIVATE_KEY_ENV), help="A private key used for generating web tokens, dependent upon " - "which hashing algorithm is used. It must be used together with " - "--jwt-secret for providing the public key.", - ) \ No newline at end of file + "which hashing algorithm is used. It must be used together with " + "--jwt-secret for providing the public key.", + ) From f25dff0819b5e9fe6eb07f2254bb8ef3a10485c2 Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Wed, 20 Sep 2023 18:54:33 +0200 Subject: [PATCH 4/6] Document environment variables for JWT and auth token --- docs/docs/http-api.mdx | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/docs/http-api.mdx b/docs/docs/http-api.mdx index 794e66bd2d30..58ddb79af538 100644 --- a/docs/docs/http-api.mdx +++ b/docs/docs/http-api.mdx @@ -66,6 +66,21 @@ rasa run \ --auth-token thisismysecret ``` +or via environment variable: +:::tip Security best practice + +We recommend that you use environment variables to store +sensitive information such as tokens and secrets +as they will not be stored in your shell history. + +::: + +```bash +export AUTH_TOKEN=thisismysecret +rasa run \ + --enable-api +``` + Any clients sending requests to the server must pass the token as a query parameter, or the request will be rejected. For example, to fetch a tracker from the server: @@ -85,6 +100,22 @@ rasa run \ --jwt-secret thisismysecret ``` +or via environment variable: +:::tip Security best practice + +We recommend that you use environment variables to store +sensitive information such as tokens and secrets +as they will not be stored in your shell history. + +::: + + +```bash +export JWT_SECRET=thisismysecret +rasa run \ + --enable-api +``` + If you want to sign a JWT token with asymmetric algorithms, you can specify the JWT private key to the `--jwt-private-key` CLI argument. You must pass the public key to the `--jwt-secret` argument, and also specify the algorithm to the `--jwt-method` argument: @@ -97,6 +128,23 @@ rasa run \ --jwt-method RS512 ``` +or via environment variables: +:::tip Security best practice + +We recommend that you use environment variables to store +sensitive information such as tokens and secrets +as they will not be stored in your shell history. + +::: + +```bash +export JWT_SECRET= +export JWT_PRIVATE_KEY= +export JWT_METHOD=RS512 +rasa run \ + --enable-api +``` + Client requests to the server will need to contain a valid JWT token in the `Authorization` header that is signed using this secret and the `HS256` algorithm e.g. From 1e3d019411b0e0e957648610876093662b21d1ca Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Thu, 21 Sep 2023 18:25:44 +0200 Subject: [PATCH 5/6] Revert back change about mitie image hash in CI --- .github/workflows/continous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index 8813c17b9006..daec3ee4ce83 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -906,7 +906,7 @@ jobs: # Base MITIE image BASE_MITIE_IMAGE_HASH=${{ hashFiles('docker/Dockerfile.base-mitie') }} MAKEFILE_MITIE_HASH=${{ hashFiles('Makefile') }} - echo "base_mitie_image_hash=${BASE_MITIE_IMAGE_HASH}" >> $GITHUB_OUTPUT + echo "base_mitie_image_hash=${BASE_MITIE_IMAGE_HASH:0:50}-${MAKEFILE_MITIE_HASH:0:50}" >> $GITHUB_OUTPUT BASE_IMAGE_MITIE_EXISTS=$((docker manifest inspect rasa/rasa:base-mitie-${BASE_MITIE_IMAGE_HASH:0:50}-${MAKEFILE_MITIE_HASH:0:50} &> /dev/null && echo true || echo false) || true) echo "base_mitie_exists=${BASE_IMAGE_MITIE_EXISTS}" >> $GITHUB_OUTPUT From 0f9b5c8ac3bb4bd0a8909aa3e7585a6f7686f15c Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Thu, 21 Sep 2023 18:42:27 +0200 Subject: [PATCH 6/6] Sorty excluded workd in .typo-ci.yaml and add thisismysecret to the list --- .typo-ci.yml | 105 +++++++++++++++++++++-------------------- docs/docs/http-api.mdx | 56 +++++++++------------- 2 files changed, 76 insertions(+), 85 deletions(-) diff --git a/.typo-ci.yml b/.typo-ci.yml index bcb9c4c75be6..9a6e5205558d 100644 --- a/.typo-ci.yml +++ b/.typo-ci.yml @@ -62,27 +62,43 @@ excluded_files: # # Any typos we should ignore? excluded_words: + - CDD + - Comerica + - ConveRTFeaturizer + - ConveRTTokenizer + - HookimplMarker + - Juste + - NLG + - README + - Tanja + - Vova - analytics + - anonymization + - anonymized - asyncio + - backends - bot - bot's - cdd - - CDD - cmdline + - conftest - conveRT - - ConveRTFeaturizer - - ConveRTTokenizer + - crf + - crfentityextractor - crfsuite + - crypto - custom-nlg-service + - customizable - daksh + - dataset - db's - - deque - - docusaurus - - non-latin - deduplicate - deduplication + - deque + - docusaurus - donath - - matplotlib + - dslim + - entitysynonymmapper - extractor - fbmessenger - featurization @@ -95,13 +111,17 @@ excluded_words: - forni - gzip - gzipped + - hallo - hftransformersnlp + - hookimpl - initializer - instaclient - - jwt - - jwt's + - ish + - jieba - jupyter - jupyterhub + - jwt + - jwt's - karpathy - keras - knowledgebase @@ -110,101 +130,82 @@ excluded_words: - llm - luis - matmul + - matplotlib - mattermost - memoization + - memoizationpolicy - miniconda - mitie - - mitiefeaturizer - mitie's + - mitiefeaturizer - mitienlp - - dataset - mongod - mrkdown - mrkdwn - myio - mymodelname - myuser - - numpy - networkx + - ngram + - nlg - nlu - nlu's + - non-latin + - numpy - perceptron + - pii-management - pika - pika's - - jieba + - pluggy + - pre - pretrained - prototyper + - prototyper - pycodestyle - pykwalify - pymessenger - pyobject - python-engineio - - pre - - customizable - quickstart - rasa - rasa's - readthedocs + - regexes + - regexfeaturizer - regularizer - repo - rst + - ruamel + - rustc + - rustup + - rustup-init - sanic - sanitization - scipy - sklearn - socketio + - spaCy + - spaCy's - spacy - spacyfeaturizer - spacynlp - - ish - - spaCy - - spaCy's - - README - - crf - - backends - - whitespaced - - ngram - subsampled - testagent + - thisismysecret + - tokenization - tokenize - tokenized - - tokenization - tokenizer - tokenizers - tokenizing - typoci - unfeaturized - unschedule - - wsgi - - ruamel - - prototyper - - hallo - - crypto - - regexes + - venv - walkthroughs - webexteams - - venv - - regexfeaturizer - - crfentityextractor - - Comerica - - entitysynonymmapper - - memoizationpolicy - - NLG - - nlg - - Juste - - Tanja - - Vova - - rustup - - rustup-init - - rustc - - conftest + - whitespaced - winpty - - pii-management - - anonymization - - anonymized - - dslim - - pluggy - - HookimplMarker - - hookimpl + - wsgi spellcheck_filenames: false diff --git a/docs/docs/http-api.mdx b/docs/docs/http-api.mdx index 58ddb79af538..d96e7476018d 100644 --- a/docs/docs/http-api.mdx +++ b/docs/docs/http-api.mdx @@ -66,21 +66,18 @@ rasa run \ --auth-token thisismysecret ``` -or via environment variable: +You can also use environment variable `AUTH_TOKEN` to set the auth token: +``` +AUTH_TOKEN=thisismysecret +``` + :::tip Security best practice We recommend that you use environment variables to store -sensitive information such as tokens and secrets -as they will not be stored in your shell history. - +and share sensitive information such as tokens and secrets +when deploying Rasa as Docker container as they will not be stored in your shell history. ::: -```bash -export AUTH_TOKEN=thisismysecret -rasa run \ - --enable-api -``` - Any clients sending requests to the server must pass the token as a query parameter, or the request will be rejected. For example, to fetch a tracker from the server: @@ -100,22 +97,18 @@ rasa run \ --jwt-secret thisismysecret ``` -or via environment variable: +You can also use environment variable `JWT_SECRET` to set the JWT secret: +``` +JWT_SECRET=thisismysecret +``` + :::tip Security best practice We recommend that you use environment variables to store -sensitive information such as tokens and secrets -as they will not be stored in your shell history. - +and share sensitive information such as tokens and secrets +when deploying Rasa as Docker container as they will not be stored in your shell history. ::: - -```bash -export JWT_SECRET=thisismysecret -rasa run \ - --enable-api -``` - If you want to sign a JWT token with asymmetric algorithms, you can specify the JWT private key to the `--jwt-private-key` CLI argument. You must pass the public key to the `--jwt-secret` argument, and also specify the algorithm to the `--jwt-method` argument: @@ -128,23 +121,20 @@ rasa run \ --jwt-method RS512 ``` -or via environment variables: +You can also use environment variables to configure JWT: +``` +JWT_SECRET= +JWT_PRIVATE_KEY= +JWT_METHOD=RS512 +``` + :::tip Security best practice We recommend that you use environment variables to store -sensitive information such as tokens and secrets -as they will not be stored in your shell history. - +and share sensitive information such as tokens and secrets +when deploying Rasa as Docker container as they will not be stored in your shell history. ::: -```bash -export JWT_SECRET= -export JWT_PRIVATE_KEY= -export JWT_METHOD=RS512 -rasa run \ - --enable-api -``` - Client requests to the server will need to contain a valid JWT token in the `Authorization` header that is signed using this secret and the `HS256` algorithm e.g.