diff --git a/README.md b/README.md index 873411f..f1d8b8a 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,10 @@ This code base is now being actively developed and maintained by Cloudera. ### Requirements -Current version of dbt-hive use dbt-core 1.7.*. We are actively working on supporting the next version of dbt-core 1.8 +Current version of dbt-hive uses dbt-core 1.8.*. We are actively working on supporting the next available version of dbt-core. Python >= 3.8 -dbt-core ~= 1.7.* +dbt-core ~= 1.8.* impyla >= 0.18 ### Install diff --git a/dbt/adapters/hive/__version__.py b/dbt/adapters/hive/__version__.py index 6b7c011..cdc5e27 100644 --- a/dbt/adapters/hive/__version__.py +++ b/dbt/adapters/hive/__version__.py @@ -11,4 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -version = "1.7.0" +version = "1.8.0" diff --git a/dbt/adapters/hive/cloudera_tracking.py b/dbt/adapters/hive/cloudera_tracking.py index a9bd69f..56d0b50 100644 --- a/dbt/adapters/hive/cloudera_tracking.py +++ b/dbt/adapters/hive/cloudera_tracking.py @@ -23,8 +23,8 @@ import threading from dbt.tracking import active_user -from dbt.adapters.base import Credentials -from dbt.events import AdapterLogger +from dbt.adapters.contracts.connection import Credentials +from dbt.adapters.events.logging import AdapterLogger from decouple import config diff --git a/dbt/adapters/hive/column.py b/dbt/adapters/hive/column.py index 947eeff..27d42a3 100644 --- a/dbt/adapters/hive/column.py +++ b/dbt/adapters/hive/column.py @@ -15,7 +15,7 @@ from typing import TypeVar, Optional, Dict, Any from dbt.adapters.base.column import Column -from dbt.dataclass_schema import dbtClassMixin +from dbt_common.dataclass_schema import dbtClassMixin Self = TypeVar("Self", bound="HiveColumn") diff --git a/dbt/adapters/hive/connections.py b/dbt/adapters/hive/connections.py index 5383575..6e2e567 100644 --- a/dbt/adapters/hive/connections.py +++ b/dbt/adapters/hive/connections.py @@ -17,20 +17,21 @@ from dataclasses import dataclass from datetime import datetime from typing import Any, Optional, Tuple +from multiprocessing.context import SpawnContext import dbt.exceptions import impala.dbapi -from dbt.adapters.base import Credentials +from dbt.adapters.contracts.connection import Credentials from dbt.adapters.sql import SQLConnectionManager -from dbt.contracts.connection import ( +from dbt.adapters.contracts.connection import ( AdapterResponse, AdapterRequiredConfig, Connection, ConnectionState, ) -from dbt.events import AdapterLogger -from dbt.events.functions import fire_event -from dbt.events.types import ConnectionUsed, SQLQuery, SQLQueryStatus +from dbt.adapters.events.logging import AdapterLogger +from dbt_common.events.functions import fire_event +from dbt.adapters.events.types import ConnectionUsed, SQLQuery, SQLQueryStatus from dbt.utils import DECIMALS import json @@ -181,8 +182,8 @@ class HiveConnectionManager(SQLConnectionManager): hive_version = None - def __init__(self, profile: AdapterRequiredConfig): - super().__init__(profile) + def __init__(self, profile: AdapterRequiredConfig, mp_context: SpawnContext): + super().__init__(profile, mp_context) # generate profile related object for instrumentation. tracker.generate_profile_info(self) diff --git a/dbt/adapters/hive/impl.py b/dbt/adapters/hive/impl.py index 198cb40..9bed01f 100644 --- a/dbt/adapters/hive/impl.py +++ b/dbt/adapters/hive/impl.py @@ -14,7 +14,7 @@ from collections import OrderedDict from concurrent.futures import Future from dataclasses import dataclass -from typing import Optional, List, Dict, Any, Union, Iterable +from typing import Optional, List, Dict, Any, Union, Iterable, FrozenSet, Tuple import agate import dbt @@ -23,17 +23,18 @@ from dbt.adapters.base import AdapterConfig from dbt.adapters.base.impl import catch_as_completed from dbt.adapters.sql import SQLAdapter +from dbt.adapters.contracts.relation import RelationConfig import dbt.adapters.hive.cloudera_tracking as tracker from dbt.adapters.hive import HiveConnectionManager from dbt.adapters.hive import HiveRelation from dbt.adapters.hive import HiveColumn from dbt.adapters.base import BaseRelation -from dbt.utils import executor +from dbt_common.utils import executor -from dbt.clients import agate_helper +from dbt_common.clients import agate_helper -from dbt.events import AdapterLogger +from dbt.adapters.events.logging import AdapterLogger logger = AdapterLogger("Hive") @@ -347,9 +348,11 @@ def get_properties(self, relation: Relation) -> Dict[str, str]: ) return dict(properties) - def get_catalog(self, manifest): + def get_catalog( + self, relation_configs: Iterable[RelationConfig], used_schemas: FrozenSet[Tuple[str, str]] + ): """Return a catalogs that contains information of all schemas""" - schema_map = self._get_catalog_schemas(manifest) + schema_map = self._get_catalog_schemas(relation_configs) if len(schema_map) > 1: dbt.exceptions.raise_compiler_error( f"Expected only one database in get_catalog, found " f"{list(schema_map)}" @@ -361,14 +364,7 @@ def get_catalog(self, manifest): for info, schemas in schema_map.items(): for schema in schemas: futures.append( - tpe.submit_connected( - self, - schema, - self._get_one_catalog, - info, - [schema], - manifest, - ) + tpe.submit_connected(self, schema, self._get_one_catalog, info, [schema]) ) # at this point all catalogs in futures list will be merged into one # insides`catch_on_completed` method of the parent class @@ -388,12 +384,7 @@ def get_catalog(self, manifest): return catalogs, exceptions - def _get_one_catalog( - self, - information_schema, - schemas, - manifest, - ) -> agate.Table: + def _get_one_catalog(self, information_schema, schemas) -> agate.Table: """Get ONE catalog. Used by get_catalog manifest is used to run the method in other context's diff --git a/dbt/adapters/hive/relation.py b/dbt/adapters/hive/relation.py index 7ed5b4f..3a12c13 100644 --- a/dbt/adapters/hive/relation.py +++ b/dbt/adapters/hive/relation.py @@ -16,7 +16,7 @@ from dataclasses import dataclass, field from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import DbtRuntimeError +from dbt_common.exceptions import DbtRuntimeError import dbt.adapters.hive.cloudera_tracking as tracker diff --git a/dev-requirements.txt b/dev-requirements.txt index 450f11f..f9138ec 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ -dbt-tests-adapter==1.7.* +dbt-tests-adapter==1.8.* pre-commit~=2.21;python_version=="3.7" pre-commit~=3.2;python_version>="3.8" pytest diff --git a/setup.py b/setup.py index 4812916..386ded5 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def _get_dbt_core_version(): package_name = "dbt-hive" # make sure this always matches dbt/adapters/hive/__version__.py -package_version = "1.7.0" +package_version = "1.8.0" description = """The Hive adapter plugin for dbt""" dbt_core_version = _get_dbt_core_version()