Add ACTIVATE_VERSION message support

.bumpversion.cfg

@@ -24,4 +24,4 @@ replace = singer-sdk = "^{new_version}"
 
 [bumpversion:file:cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml]
 search = singer-sdk = "^{current_version}"
-replace = singer-sdk = "^{new_version}"
+replace = singer-sdk = "^{new_version}"

cookiecutter/tap-template/cookiecutter.json

@@ -6,6 +6,7 @@
     "stream_type": [
         "REST",
         "GraphQL",
+        "SQL",
         "Other"
     ],
     "auth_method": [

cookiecutter/tap-template/cookiecutter.tests.yml

@@ -41,6 +41,11 @@ tests:
     auth_method: JWT
     include_cicd_sample_template: "None (Skip)"
 
+  - source_name: SQLTemplateTest
+    tap_id: test-tap-sql-type
+    stream_type: SQL
+    auth_method: Custom or N/A
+
   - source_name: OtherCustomTemplateTest
     tap_id: test-tap-other-custom-type
     stream_type: Other

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py

@@ -2,9 +2,12 @@
 
 from typing import List
 
-from singer_sdk import Tap, Stream
+from singer_sdk import {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Tap, {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Stream
 from singer_sdk import typing as th  # JSON schema typing helpers
 
+{%- if cookiecutter.stream_type == "SQL" %}
+from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream
+{%- else %}
 # TODO: Import your custom stream types here:
 from {{ cookiecutter.library_name }}.streams import (
     {{ cookiecutter.source_name }}Stream,
@@ -13,6 +16,7 @@
     GroupsStream,
 {%- endif %}
 )
+{%- endif %}
 
 {%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %}
 # TODO: Compile a list of custom stream types here
@@ -24,7 +28,7 @@
 {%- endif %}
 
 
-class Tap{{ cookiecutter.source_name }}(Tap):
+class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.source_name == 'SQL' else '' }}Tap):
     """{{ cookiecutter.source_name }} tap class."""
     name = "{{ cookiecutter.tap_id }}"
 

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%}

@@ -0,0 +1,68 @@
+"""SQL client handling.
+
+This includes {{ cookiecutter.source_name }}Stream and {{ cookiecutter.source_name }}Connector.
+"""
+
+import sqlalchemy
+
+from singer_sdk import SQLConnector, SQLStream
+
+class {{ cookiecutter.source_name }}Connector(SQLConnector):
+    """Connects to the {{ cookiecutter.source_name }} SQL source."""
+
+    def get_sqlalchemy_url(cls, config: dict) -> str:
+        """Concatenate a SQLAlchemy URL for use in connecting to the source."""
+        # TODO: Replace this with a valid connection string for your source:
+        return (
+            f"awsathena+rest://{config['aws_access_key_id']}:"
+            f"{config['aws_secret_access_key']}@athena"
+            f".{config['aws_region']}.amazonaws.com:443/"
+            f"{config['schema_name']}?"
+            f"s3_staging_dir={config['s3_staging_dir']}"
+        )
+
+    @staticmethod
+    def to_jsonschema_type(sql_type: sqlalchemy.types.TypeEngine) -> dict:
+        """Returns a JSON Schema equivalent for the given SQL type.
+
+        Developers may optionally add custom logic before calling the default implementation
+        inherited from the base class.
+        """
+        # Optionally, add custom logic before calling the super().
+        # You may delete this method if overrides are not needed.
+        return super().to_jsonschema_type(sql_type)
+
+    @staticmethod
+    def to_sql_type(jsonschema_type: dict) -> sqlalchemy.types.TypeEngine:
+        """Returns a JSON Schema equivalent for the given SQL type.
+
+        Developers may optionally add custom logic before calling the default implementation
+        inherited from the base class.
+        """
+        # Optionally, add custom logic before calling the super().
+        # You may delete this method if overrides are not needed.
+        return super().to_sql_type(jsonschema_type)
+
+
+class {{ cookiecutter.source_name }}Stream(SQLStream):
+    """Stream class for {{ cookiecutter.source_name }} streams."""
+
+    connector_class = {{ cookiecutter.source_name }}Connector
+
+
+    def get_records(self, partition: Optional[dict]) -> Iterable[Dict[str, Any]]:
+        """Return a generator of row-type dictionary objects.
+
+        Developers may optionally add custom logic before calling the default implementation
+        inherited from the base class.
+
+        Args:
+            partition: If provided, will read specifically from this data slice.
+
+        Yields:
+            One dict per record.
+        """
+        # Optionally, add custom logic instead of calling the super().
+        # This is helpful if the source database provides batch-optimized record retreival.
+        # If no overrides or optimizations are needed, you may delete this method.
+        yield from super().get_records(partition)

cookiecutter/target-template/cookiecutter.json

@@ -3,5 +3,5 @@
     "admin_name": "FirstName LastName",
     "target_id": "target-{{ cookiecutter.destination_name.lower() }}",
     "library_name": "{{ cookiecutter.target_id.replace('-', '_') }}",
-    "serialization_method": ["Per record", "Per batch"]
+    "serialization_method": ["Per record", "Per batch", "SQL"]
 }

cookiecutter/target-template/cookiecutter.tests.yml

@@ -5,3 +5,6 @@ tests:
   - destination_name: TargetRecordSinkTest
     target_id: target-recordsink-test
     serialization_method: Per record
+  - destination_name: SQLSinkTest
+    target_id: target-sqlsink-test
+    serialization_method: SQL

docs/classes/singer_sdk.SQLConnector.rst

@@ -0,0 +1,7 @@
+singer\_sdk.SQLConnector
+======================
+
+.. currentmodule:: singer_sdk
+
+.. autoclass:: SQLConnector
+    :members:

docs/classes/singer_sdk.SQLStream.rst

@@ -0,0 +1,7 @@
+singer\_sdk.SQLStream
+======================
+
+.. currentmodule:: singer_sdk
+
+.. autoclass:: SQLStream
+    :members:

docs/classes/singer_sdk.SQLTap.rst

@@ -0,0 +1,7 @@
+singer\_sdk.SQLTap
+===============
+
+.. currentmodule:: singer_sdk
+
+.. autoclass:: SQLTap
+    :members:

docs/reference.rst

@@ -11,6 +11,7 @@ Plugin Classes
     :template: class.rst
 
     Tap
+    SQLTap
     Target
 
 Stream Classes
@@ -23,6 +24,7 @@ Stream Classes
     Stream
     RESTStream
     GraphQLStream
+    SQLStream
 
 Sink Classes
 --------------
@@ -35,6 +37,15 @@ Sink Classes
     RecordSink
     BatchSink
 
+SQL Connector for SQL Taps and Targets
+--------------------------------------
+
+.. autosummary::
+    :toctree: _autosummary
+    :template: class.rst
+
+    SQLConnector
+
 Authenticator Classes
 ---------------------
 

mypy.ini

@@ -3,6 +3,7 @@ python_version = 3.8
 warn_unused_configs = True
 warn_return_any = True
 exclude = singer_sdk/tests
+plugins = sqlalchemy.ext.mypy.plugin
 
 [mypy-singer.*]
 # Library 'pipelinewise-singer-tools' does not have type hints:
@@ -34,3 +35,6 @@ ignore_missing_imports = True
 
 [mypy-jsonpath_ng.*]
 ignore_missing_imports = True
+
+[mypy-sqlalchemy.*]
+ignore_missing_imports = True