From 5c08d9e69816277602128668cbc5e6441f4fdca3 Mon Sep 17 00:00:00 2001 From: Steve Clarke <84364906+s7clarke10@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:29:33 +1200 Subject: [PATCH 1/6] Adding support for OAuth Headers --- pyproject.toml | 1 + tap_rest_api_msdk/auth.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5c6469f..f139c17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ readme = "README.md" python = "<3.12,>=3.7.1" requests = "^2.25.1" singer-sdk = "^0.26.0" +singer-sdk = { git = "https://github.com/s7clarke10/sdk.git", branch = "feature/support_headers_when_requesting_oauth_token" } genson = "^1.2.2" atomicwrites = "^1.4.0" requests-aws4auth = "^1.2.3" diff --git a/tap_rest_api_msdk/auth.py b/tap_rest_api_msdk/auth.py index dd6815e..91b9dcd 100644 --- a/tap_rest_api_msdk/auth.py +++ b/tap_rest_api_msdk/auth.py @@ -5,7 +5,12 @@ import boto3 from requests_aws4auth import AWS4Auth -from singer_sdk.authenticators import APIKeyAuthenticator, BasicAuthenticator, BearerTokenAuthenticator, OAuthAuthenticator +from singer_sdk.authenticators import ( + APIKeyAuthenticator, + BasicAuthenticator, + BearerTokenAuthenticator, + OAuthAuthenticator +) class AWSConnectClient: """A connection class to AWS Resources""" @@ -195,6 +200,11 @@ def select_authenticator(self) -> Any: api_keys = my_config.get('api_keys', '') self.http_auth = None + # Set http headers if headers are supplied + # Some OAUTH2 API's require headers to be supplied + # In the OAUTH request. + auth_headers = my_config.get('headers',None) + # Using API Key Authenticator, keys are extracted from api_keys dict if auth_method == "api_key": if api_keys: @@ -220,6 +230,7 @@ def select_authenticator(self) -> Any: auth_endpoint=my_config.get('access_token_url', ''), oauth_scopes=my_config.get('scope', ''), default_expiration=my_config.get('oauth_expiration_secs', ''), + auth_headers=auth_headers, ) # Using Bearer Token Authenticator elif auth_method == "bearer_token": From 303a71f8522997fbdf813bd47deb34b08ea54159 Mon Sep 17 00:00:00 2001 From: Steve Clarke <84364906+s7clarke10@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:42:27 +1200 Subject: [PATCH 2/6] Commenting out current SDK version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f139c17..2d87041 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ readme = "README.md" [tool.poetry.dependencies] python = "<3.12,>=3.7.1" requests = "^2.25.1" -singer-sdk = "^0.26.0" +#singer-sdk = "^0.26.0" singer-sdk = { git = "https://github.com/s7clarke10/sdk.git", branch = "feature/support_headers_when_requesting_oauth_token" } genson = "^1.2.2" atomicwrites = "^1.4.0" From 08014b10dff48da775a31c687e275467732a9bab Mon Sep 17 00:00:00 2001 From: Steve Clarke <84364906+s7clarke10@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:53:38 +1200 Subject: [PATCH 3/6] Correcting parameter name --- tap_rest_api_msdk/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_rest_api_msdk/auth.py b/tap_rest_api_msdk/auth.py index 91b9dcd..79661af 100644 --- a/tap_rest_api_msdk/auth.py +++ b/tap_rest_api_msdk/auth.py @@ -230,7 +230,7 @@ def select_authenticator(self) -> Any: auth_endpoint=my_config.get('access_token_url', ''), oauth_scopes=my_config.get('scope', ''), default_expiration=my_config.get('oauth_expiration_secs', ''), - auth_headers=auth_headers, + oauth_headers=auth_headers, ) # Using Bearer Token Authenticator elif auth_method == "bearer_token": From 8b72d0075e7d8f0a5111e7561845267898147982 Mon Sep 17 00:00:00 2001 From: Steve Clarke <84364906+s7clarke10@users.noreply.github.com> Date: Mon, 10 Jul 2023 17:57:54 +1200 Subject: [PATCH 4/6] Add support for expirying OAuth Tokens --- tap_rest_api_msdk/client.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tap_rest_api_msdk/client.py b/tap_rest_api_msdk/client.py index 5d472c6..93ee056 100644 --- a/tap_rest_api_msdk/client.py +++ b/tap_rest_api_msdk/client.py @@ -16,6 +16,8 @@ class RestApiStream(RESTStream): # Intialise self.http_auth used by prepare_request http_auth = None + # Cache the authenticator using a Smart Singleton pattern + _authenticator = None @property def url_base(self) -> Any: @@ -29,7 +31,6 @@ def url_base(self) -> Any: @property - @cached def authenticator(self) -> Any: """Calls an appropriate SDK Authentication method based on the the set auth_method which is set via the config. @@ -39,9 +40,8 @@ def authenticator(self) -> Any: Note 1: Each auth method requires certain configuration to be present see README.md for each auth methods configuration requirements. - Note 2: The cached decorator will have all calls to the authenticator use the - same instance logic to speed up processing. TODO: Examine if this affects OAuth2 - callbacks and if logic is required for callback for expiring AWS STS tokens. + Note 2: Using Singleton Pattern on the autenticator for caching with a check + if an OAuth Token has expired and needs to be refreshed. Raises: ValueError: if the auth_method is unknown. @@ -50,11 +50,16 @@ def authenticator(self) -> Any: A SDK Authenticator or APIAuthenticatorBase if no auth_method supplied. """ - stream_authenticator = select_authenticator(self) - - if stream_authenticator: - return stream_authenticator - else: - return APIAuthenticatorBase(stream=self) - + auth_method = self.config.get("auth_method",None) + + if not self._authenticator: + self._authenticator = select_authenticator(self) + if not self._authenticator: + # No Auth Method, use default Authenticator + self._authenticator = APIAuthenticatorBase(stream=self) + elif auth_method == 'oauth': + if not self._authenticator.is_token_valid(): + # Obtain a new OAuth token as it has expired + self._authenticator = select_authenticator(self) + return self._authenticator From d9322ef3d79734e3b89d9c5affe0dc4530e9c54d Mon Sep 17 00:00:00 2001 From: Steve Clarke <84364906+s7clarke10@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:08:28 +1200 Subject: [PATCH 5/6] Pointing to Meltano SDK with OAuth Header support --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2d87041..419aafb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ readme = "README.md" python = "<3.12,>=3.7.1" requests = "^2.25.1" #singer-sdk = "^0.26.0" -singer-sdk = { git = "https://github.com/s7clarke10/sdk.git", branch = "feature/support_headers_when_requesting_oauth_token" } +singer-sdk = { git = "https://github.com/meltano/sdk.git", rev = "1637b48" } genson = "^1.2.2" atomicwrites = "^1.4.0" requests-aws4auth = "^1.2.3" From 6bcf8aaf5108da7879370df19eaa319e62567b40 Mon Sep 17 00:00:00 2001 From: Steve Clarke <84364906+s7clarke10@users.noreply.github.com> Date: Mon, 24 Jul 2023 16:16:08 +1200 Subject: [PATCH 6/6] SDK update to support oauth headers --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 419aafb..b4c34e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,8 +20,7 @@ readme = "README.md" [tool.poetry.dependencies] python = "<3.12,>=3.7.1" requests = "^2.25.1" -#singer-sdk = "^0.26.0" -singer-sdk = { git = "https://github.com/meltano/sdk.git", rev = "1637b48" } +singer-sdk = "^0.30.0" genson = "^1.2.2" atomicwrites = "^1.4.0" requests-aws4auth = "^1.2.3"