Merge branch 'devel' into d#/fix_import_schemas

dlt/__main__.py

@@ -0,0 +1,4 @@
+from dlt.cli._dlt import main
+
+if __name__ == "__main__":
+    main()

dlt/common/configuration/specs/connection_string_credentials.py

@@ -1,5 +1,5 @@
 from typing import Any, ClassVar, Dict, List, Optional
-from sqlalchemy.engine import URL, make_url
+from dlt.common.libs.sql_alchemy import URL, make_url
 from dlt.common.configuration.specs.exceptions import InvalidConnectionString
 
 from dlt.common.typing import TSecretValue
@@ -26,6 +26,7 @@ def parse_native_representation(self, native_value: Any) -> None:
             # update only values that are not None
             self.update({k: v for k, v in url._asdict().items() if v is not None})
             if self.query is not None:
+                # query may be immutable so make it mutable
                 self.query = dict(self.query)
         except Exception:
             raise InvalidConnectionString(self.__class__, native_value, self.drivername)

dlt/common/data_types/type_helpers.py

@@ -13,7 +13,6 @@
 from dlt.common.data_types.typing import TDataType
 from dlt.common.time import (
     ensure_pendulum_datetime,
-    parse_iso_like_datetime,
     ensure_pendulum_date,
     ensure_pendulum_time,
 )

dlt/common/libs/numpy.py

@@ -0,0 +1,6 @@
+from dlt.common.exceptions import MissingDependencyException
+
+try:
+    import numpy
+except ModuleNotFoundError:
+    raise MissingDependencyException("DLT Numpy Helpers", ["numpy"])

dlt/common/libs/pyarrow.py

@@ -1,4 +1,5 @@
 from datetime import datetime, date  # noqa: I251
+from pendulum.tz import UTC
 from typing import Any, Tuple, Optional, Union, Callable, Iterable, Iterator, Sequence, Tuple
 
 from dlt import version
@@ -14,6 +15,7 @@
 try:
     import pyarrow
     import pyarrow.parquet
+    import pyarrow.compute
 except ModuleNotFoundError:
     raise MissingDependencyException(
         "dlt parquet Helpers", [f"{version.DLT_PKG_NAME}[parquet]"], "dlt Helpers for for parquet."
@@ -314,21 +316,21 @@ def is_arrow_item(item: Any) -> bool:
     return isinstance(item, (pyarrow.Table, pyarrow.RecordBatch))
 
 
-def to_arrow_compute_input(value: Any, arrow_type: pyarrow.DataType) -> Any:
+def to_arrow_scalar(value: Any, arrow_type: pyarrow.DataType) -> Any:
     """Converts python value to an arrow compute friendly version"""
     return pyarrow.scalar(value, type=arrow_type)
 
 
-def from_arrow_compute_output(arrow_value: pyarrow.Scalar) -> Any:
-    """Converts arrow scalar into Python type. Currently adds "UTC" to naive date times."""
+def from_arrow_scalar(arrow_value: pyarrow.Scalar) -> Any:
+    """Converts arrow scalar into Python type. Currently adds "UTC" to naive date times and converts all others to UTC"""
     row_value = arrow_value.as_py()
     # dates are not represented as datetimes but I see connector-x represents
     # datetimes as dates and keeping the exact time inside. probably a bug
     # but can be corrected this way
     if isinstance(row_value, date) and not isinstance(row_value, datetime):
         row_value = pendulum.from_timestamp(arrow_value.cast(pyarrow.int64()).as_py() / 1000)
     elif isinstance(row_value, datetime):
-        row_value = pendulum.instance(row_value)
+        row_value = pendulum.instance(row_value).in_tz("UTC")
     return row_value