diff --git a/.rat-excludes b/.rat-excludes index 12f1f2fe543a9..4ed267f7ccb1c 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -70,6 +70,7 @@ google-sheets.svg ibm-db2.svg postgresql.svg snowflake.svg +ydb.svg # docs-related erd.puml diff --git a/README.md b/README.md index 62d5e11ee76a8..7928904a2f3b0 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ Here are some of the major database solutions that are supported: oceanbase oceanbase denodo + ydb

**A more comprehensive list of supported databases** along with the configuration instructions can be found [here](https://superset.apache.org/docs/configuration/databases). diff --git a/docs/docs/configuration/databases.mdx b/docs/docs/configuration/databases.mdx index 56c8897d1c21b..16fe59a434f66 100644 --- a/docs/docs/configuration/databases.mdx +++ b/docs/docs/configuration/databases.mdx @@ -81,6 +81,7 @@ are compatible with Superset. | [TimescaleDB](/docs/configuration/databases#timescaledb) | `pip install psycopg2` | `postgresql://:@:/` | | [Trino](/docs/configuration/databases#trino) | `pip install trino` | `trino://{username}:{password}@{hostname}:{port}/{catalog}` | | [Vertica](/docs/configuration/databases#vertica) | `pip install sqlalchemy-vertica-python` | `vertica+vertica_python://:@/` | +| [YDB](/docs/configuration/databases#ydb) | `pip install ydb-sqlalchemy` | `ydb://{host}:{port}/{database_name}` | | [YugabyteDB](/docs/configuration/databases#yugabytedb) | `pip install psycopg2` | `postgresql://:@/` | --- @@ -1537,6 +1538,78 @@ Other parameters: - Load Balancer - Backup Host + +#### YDB + +The recommended connector library for [YDB](https://ydb.tech/) is +[ydb-sqlalchemy](https://pypi.org/project/ydb-sqlalchemy/). + +##### Connection String + +The connection string for YDB looks like this: + +``` +ydb://{host}:{port}/{database_name} +``` + +##### Protocol +You can specify `protocol` in the `Secure Extra` field at `Advanced / Security`: + +``` +{ + "protocol": "grpcs" +} +``` + +Default is `grpc`. + + +##### Authentication Methods +###### Static Credentials +To use `Static Credentials` you should provide `username`/`password` in the `Secure Extra` field at `Advanced / Security`: + +``` +{ + "credentials": { + "username": "...", + "password": "..." + } +} +``` + + +###### Access Token Credentials +To use `Access Token Credentials` you should provide `token` in the `Secure Extra` field at `Advanced / Security`: + +``` +{ + "credentials": { + "token": "...", + } +} +``` + + +##### Service Account Credentials +To use Service Account Credentials, you should provide `service_account_json` in the `Secure Extra` field at `Advanced / Security`: + +``` +{ + "credentials": { + "service_account_json": { + "id": "...", + "service_account_id": "...", + "created_at": "...", + "key_algorithm": "...", + "public_key": "...", + "private_key": "..." + } + } +} +``` + + + #### YugabyteDB [YugabyteDB](https://www.yugabyte.com/) is a distributed SQL database built on top of PostgreSQL. diff --git a/docs/static/img/databases/ydb.svg b/docs/static/img/databases/ydb.svg new file mode 100644 index 0000000000000..6b70d0cf9c7d1 --- /dev/null +++ b/docs/static/img/databases/ydb.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/pyproject.toml b/pyproject.toml index c54d1a7f31c93..db836eab3203f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -179,6 +179,7 @@ netezza = ["nzalchemy>=11.0.2"] starrocks = ["starrocks>=1.0.0"] doris = ["pydoris>=1.0.0, <2.0.0"] oceanbase = ["oceanbase_py>=0.0.1"] +ydb = ["ydb-sqlalchemy>=0.1.2"] development = [ "docker", "flask-testing", diff --git a/superset/db_engine_specs/ydb.py b/superset/db_engine_specs/ydb.py new file mode 100755 index 0000000000000..50a1972a6b1b9 --- /dev/null +++ b/superset/db_engine_specs/ydb.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +from datetime import datetime +from typing import Any, TYPE_CHECKING + +from sqlalchemy import types + +from superset.constants import TimeGrain +from superset.db_engine_specs.base import BaseEngineSpec +from superset.utils import json + +if TYPE_CHECKING: + from superset.models.core import Database + + +logger = logging.getLogger(__name__) + + +class YDBEngineSpec(BaseEngineSpec): + engine = "yql" + engine_aliases = {"ydb", "yql+ydb"} + engine_name = "YDB" + + default_driver = "ydb" + + sqlalchemy_uri_placeholder = "ydb://{host}:{port}/{database_name}" + + # pylint: disable=invalid-name + encrypted_extra_sensitive_fields = {"$.connect_args.credentials", "$.credentials"} + + disable_ssh_tunneling = False + + supports_file_upload = False + + allows_alias_in_orderby = True + + _time_grain_expressions = { + None: "{col}", + TimeGrain.SECOND: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1S')))", + TimeGrain.THIRTY_SECONDS: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT30S')))", + TimeGrain.MINUTE: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1M')))", + TimeGrain.FIVE_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT5M')))", + TimeGrain.TEN_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT10M')))", + TimeGrain.FIFTEEN_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT15M')))", + TimeGrain.THIRTY_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT30M')))", + TimeGrain.HOUR: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1H')))", + TimeGrain.DAY: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('P1D')))", + TimeGrain.WEEK: "DateTime::MakeDatetime(DateTime::StartOfWeek({col}))", + TimeGrain.MONTH: "DateTime::MakeDatetime(DateTime::StartOfMonth({col}))", + TimeGrain.QUARTER: "DateTime::MakeDatetime(DateTime::StartOfQuarter({col}))", + TimeGrain.YEAR: "DateTime::MakeDatetime(DateTime::StartOfYear({col}))", + } + + @classmethod + def epoch_to_dttm(cls) -> str: + return "DateTime::MakeDatetime({col})" + + @classmethod + def convert_dttm( + cls, target_type: str, dttm: datetime, db_extra: dict[str, Any] | None = None + ) -> str | None: + sqla_type = cls.get_sqla_column_type(target_type) + + if isinstance(sqla_type, types.Date): + return f"DateTime::MakeDate(DateTime::ParseIso8601('{dttm.date().isoformat()}'))" + if isinstance(sqla_type, types.DateTime): + return f"""DateTime::MakeDatetime(DateTime::ParseIso8601('{dttm.isoformat(sep="T", timespec="seconds")}'))""" + return None + + @staticmethod + def update_params_from_encrypted_extra( + database: Database, + params: dict[str, Any], + ) -> None: + if not database.encrypted_extra: + return + + try: + encrypted_extra = json.loads(database.encrypted_extra) + connect_args = params.setdefault("connect_args", {}) + + if "protocol" in encrypted_extra: + connect_args["protocol"] = encrypted_extra["protocol"] + + if "credentials" in encrypted_extra: + credentials_info = encrypted_extra["credentials"] + connect_args["credentials"] = credentials_info + + except json.JSONDecodeError as ex: + logger.error(ex, exc_info=True) + raise diff --git a/tests/unit_tests/db_engine_specs/test_ydb.py b/tests/unit_tests/db_engine_specs/test_ydb.py new file mode 100644 index 0000000000000..c4e1586299541 --- /dev/null +++ b/tests/unit_tests/db_engine_specs/test_ydb.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=unused-argument, import-outside-toplevel, protected-access +from __future__ import annotations + +from datetime import datetime +from typing import Any, Optional +from unittest.mock import Mock + +import pytest + +from superset.utils import json +from tests.unit_tests.db_engine_specs.utils import assert_convert_dttm +from tests.unit_tests.fixtures.common import dttm # noqa: F401 + + +def test_epoch_to_dttm() -> None: + from superset.db_engine_specs.ydb import YDBEngineSpec + + assert YDBEngineSpec.epoch_to_dttm() == "DateTime::MakeDatetime({col})" + + +@pytest.mark.parametrize( + "target_type,expected_result", + [ + ("Date", "DateTime::MakeDate(DateTime::ParseIso8601('2019-01-02'))"), + ( + "DateTime", + "DateTime::MakeDatetime(DateTime::ParseIso8601('2019-01-02T03:04:05'))", + ), + ("UnknownType", None), + ], +) +def test_convert_dttm( + target_type: str, + expected_result: Optional[str], + dttm: datetime, # noqa: F811 +) -> None: + from superset.db_engine_specs.ydb import YDBEngineSpec as spec + + assert_convert_dttm(spec, target_type, expected_result, dttm) + + +def test_specify_protocol() -> None: + from superset.db_engine_specs.ydb import YDBEngineSpec + + database = Mock() + + extra = {"protocol": "grpcs"} + database.encrypted_extra = json.dumps(extra) + + params: dict[str, Any] = {} + YDBEngineSpec.update_params_from_encrypted_extra(database, params) + connect_args = params.setdefault("connect_args", {}) + assert connect_args.get("protocol") == "grpcs" + + +def test_specify_credentials() -> None: + from superset.db_engine_specs.ydb import YDBEngineSpec + + database = Mock() + + auth_params = {"username": "username", "password": "password"} + database.encrypted_extra = json.dumps({"credentials": auth_params}) + + params: dict[str, Any] = {} + YDBEngineSpec.update_params_from_encrypted_extra(database, params) + connect_args = params.setdefault("connect_args", {}) + assert connect_args.get("credentials") == auth_params