From c14dd0573cd7bcb26a734aa1243e47dc909893c1 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Fri, 18 Oct 2024 10:24:56 -0400 Subject: [PATCH] Move object-store integration out of this repo (#248) --- Cargo.lock | 32 +-- Cargo.toml | 9 +- arro3-io/Cargo.toml | 2 +- object-store-rs/Cargo.toml | 36 --- object-store-rs/README.md | 1 - object-store-rs/pyproject.toml | 20 -- .../python/object_store_rs/__init__.py | 4 - .../python/object_store_rs/_copy.pyi | 6 - .../python/object_store_rs/_delete.pyi | 4 - .../python/object_store_rs/_get.pyi | 114 ---------- .../python/object_store_rs/_head.pyi | 5 - .../python/object_store_rs/_list.pyi | 45 ---- .../object_store_rs/_object_store_rs.pyi | 32 --- .../python/object_store_rs/_put.pyi | 21 -- .../object_store_rs/_pyo3_object_store.pyi | 115 ---------- .../python/object_store_rs/_rename.pyi | 8 - .../python/object_store_rs/_sign.pyi | 16 -- object-store-rs/src/copy.rs | 68 ------ object-store-rs/src/delete.rs | 28 --- object-store-rs/src/get.rs | 211 ------------------ object-store-rs/src/head.rs | 33 --- object-store-rs/src/lib.rs | 55 ----- object-store-rs/src/list.rs | 136 ----------- object-store-rs/src/put.rs | 119 ---------- object-store-rs/src/rename.rs | 72 ------ object-store-rs/src/runtime.rs | 18 -- object-store-rs/src/signer.rs | 161 ------------- pyo3-object_store/Cargo.toml | 27 --- pyo3-object_store/README.md | 3 - pyo3-object_store/src/api.rs | 35 --- pyo3-object_store/src/aws.rs | 152 ------------- pyo3-object_store/src/azure.rs | 90 -------- pyo3-object_store/src/client.rs | 39 ---- pyo3-object_store/src/error.rs | 41 ---- pyo3-object_store/src/fsspec.rs | 198 ---------------- pyo3-object_store/src/gcp.rs | 90 -------- pyo3-object_store/src/http.rs | 45 ---- pyo3-object_store/src/lib.rs | 24 -- pyo3-object_store/src/local.rs | 35 --- pyo3-object_store/src/memory.rs | 27 --- pyo3-object_store/src/retry.rs | 38 ---- pyo3-object_store/src/store.rs | 71 ------ 42 files changed, 9 insertions(+), 2277 deletions(-) delete mode 100644 object-store-rs/Cargo.toml delete mode 100644 object-store-rs/README.md delete mode 100644 object-store-rs/pyproject.toml delete mode 100644 object-store-rs/python/object_store_rs/__init__.py delete mode 100644 object-store-rs/python/object_store_rs/_copy.pyi delete mode 100644 object-store-rs/python/object_store_rs/_delete.pyi delete mode 100644 object-store-rs/python/object_store_rs/_get.pyi delete mode 100644 object-store-rs/python/object_store_rs/_head.pyi delete mode 100644 object-store-rs/python/object_store_rs/_list.pyi delete mode 100644 object-store-rs/python/object_store_rs/_object_store_rs.pyi delete mode 100644 object-store-rs/python/object_store_rs/_put.pyi delete mode 100644 object-store-rs/python/object_store_rs/_pyo3_object_store.pyi delete mode 100644 object-store-rs/python/object_store_rs/_rename.pyi delete mode 100644 object-store-rs/python/object_store_rs/_sign.pyi delete mode 100644 object-store-rs/src/copy.rs delete mode 100644 object-store-rs/src/delete.rs delete mode 100644 object-store-rs/src/get.rs delete mode 100644 object-store-rs/src/head.rs delete mode 100644 object-store-rs/src/lib.rs delete mode 100644 object-store-rs/src/list.rs delete mode 100644 object-store-rs/src/put.rs delete mode 100644 object-store-rs/src/rename.rs delete mode 100644 object-store-rs/src/runtime.rs delete mode 100644 object-store-rs/src/signer.rs delete mode 100644 pyo3-object_store/Cargo.toml delete mode 100644 pyo3-object_store/README.md delete mode 100644 pyo3-object_store/src/api.rs delete mode 100644 pyo3-object_store/src/aws.rs delete mode 100644 pyo3-object_store/src/azure.rs delete mode 100644 pyo3-object_store/src/client.rs delete mode 100644 pyo3-object_store/src/error.rs delete mode 100644 pyo3-object_store/src/fsspec.rs delete mode 100644 pyo3-object_store/src/gcp.rs delete mode 100644 pyo3-object_store/src/http.rs delete mode 100644 pyo3-object_store/src/lib.rs delete mode 100644 pyo3-object_store/src/local.rs delete mode 100644 pyo3-object_store/src/memory.rs delete mode 100644 pyo3-object_store/src/retry.rs delete mode 100644 pyo3-object_store/src/store.rs diff --git a/Cargo.lock b/Cargo.lock index dd01026..b2d0f6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -476,10 +476,8 @@ checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", - "js-sys", "num-traits", "serde", - "wasm-bindgen", "windows-targets", ] @@ -861,9 +859,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -1293,22 +1291,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "object-store-rs" -version = "0.4.2" -dependencies = [ - "bytes", - "chrono", - "futures", - "http", - "object_store", - "pyo3", - "pyo3-async-runtimes", - "pyo3-object_store", - "tokio", - "url", -] - [[package]] name = "object_store" version = "0.11.0" @@ -1635,8 +1617,8 @@ dependencies = [ [[package]] name = "pyo3-object_store" version = "0.1.0" +source = "git+https://github.com/developmentseed/object-store-rs?rev=922b58ff784271345ce80342cf4cd6cddce61adf#922b58ff784271345ce80342cf4cd6cddce61adf" dependencies = [ - "arrow-select", "futures", "object_store", "pyo3", @@ -1874,9 +1856,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" dependencies = [ "once_cell", "ring", @@ -1910,9 +1892,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" diff --git a/Cargo.toml b/Cargo.toml index 3c4ca1e..5bec80c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,5 @@ [workspace] -members = [ - "arro3-compute", - "arro3-core", - "arro3-io", - "object-store-rs", - "pyo3-arrow", - "pyo3-object_store", -] +members = ["arro3-compute", "arro3-core", "arro3-io", "pyo3-arrow"] resolver = "2" [workspace.package] diff --git a/arro3-io/Cargo.toml b/arro3-io/Cargo.toml index 9fbbf53..4de3aaf 100644 --- a/arro3-io/Cargo.toml +++ b/arro3-io/Cargo.toml @@ -45,5 +45,5 @@ pyo3-async-runtimes = { workspace = true, features = [ "tokio-runtime", ], optional = true } pyo3-file = { workspace = true } -pyo3-object_store = { path = "../pyo3-object_store", optional = true } +pyo3-object_store = { git = "https://github.com/developmentseed/object-store-rs", rev = "922b58ff784271345ce80342cf4cd6cddce61adf", optional = true } thiserror = { workspace = true } diff --git a/object-store-rs/Cargo.toml b/object-store-rs/Cargo.toml deleted file mode 100644 index 0b3beff..0000000 --- a/object-store-rs/Cargo.toml +++ /dev/null @@ -1,36 +0,0 @@ -[package] -name = "object-store-rs" -version = { workspace = true } -authors = { workspace = true } -edition = { workspace = true } -description = "Core library for representing Arrow data in Python." -readme = "README.md" -repository = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -keywords = { workspace = true } -categories = { workspace = true } -rust-version = { workspace = true } - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[lib] -name = "_object_store_rs" -crate-type = ["cdylib"] - -[dependencies] -bytes = { workspace = true } -chrono = "0.4.38" -futures = "0.3.31" -http = "1.1" -object_store = { workspace = true } -pyo3 = { workspace = true, features = ["chrono"] } -pyo3-async-runtimes = { workspace = true, features = ["tokio-runtime"] } -pyo3-file = { workspace = true } -pyo3-object_store = { path = "../pyo3-object_store" } -tokio = { version = "1.40", features = [ - "macros", - "rt", - "rt-multi-thread", - "sync", -] } -url = "2" diff --git a/object-store-rs/README.md b/object-store-rs/README.md deleted file mode 100644 index 0adcf9d..0000000 --- a/object-store-rs/README.md +++ /dev/null @@ -1 +0,0 @@ -# object-store-rs diff --git a/object-store-rs/pyproject.toml b/object-store-rs/pyproject.toml deleted file mode 100644 index 5a8f859..0000000 --- a/object-store-rs/pyproject.toml +++ /dev/null @@ -1,20 +0,0 @@ -[build-system] -requires = ["maturin>=1.4.0,<2.0"] -build-backend = "maturin" - -[project] -name = "object-store-rs" -requires-python = ">=3.9" -dependencies = [] -classifiers = [ - "Programming Language :: Rust", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", -] -dynamic = ["version"] - -[tool.maturin] -features = ["pyo3/extension-module"] -module-name = "object_store_rs._object_store_rs" -python-source = "python" -strip = true diff --git a/object-store-rs/python/object_store_rs/__init__.py b/object-store-rs/python/object_store_rs/__init__.py deleted file mode 100644 index e542bee..0000000 --- a/object-store-rs/python/object_store_rs/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from ._object_store_rs import * -from ._object_store_rs import ___version - -__version__: str = ___version() diff --git a/object-store-rs/python/object_store_rs/_copy.pyi b/object-store-rs/python/object_store_rs/_copy.pyi deleted file mode 100644 index c818902..0000000 --- a/object-store-rs/python/object_store_rs/_copy.pyi +++ /dev/null @@ -1,6 +0,0 @@ -from ._pyo3_object_store import ObjectStore - -def copy(store: ObjectStore, from_: str, to: str) -> None: ... -async def copy_async(store: ObjectStore, from_: str, to: str) -> None: ... -def copy_if_not_exists(store: ObjectStore, from_: str, to: str) -> None: ... -async def copy_if_not_exists_async(store: ObjectStore, from_: str, to: str) -> None: ... diff --git a/object-store-rs/python/object_store_rs/_delete.pyi b/object-store-rs/python/object_store_rs/_delete.pyi deleted file mode 100644 index 400d645..0000000 --- a/object-store-rs/python/object_store_rs/_delete.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from ._pyo3_object_store import ObjectStore - -def delete(store: ObjectStore, location: str) -> None: ... -async def delete_async(store: ObjectStore, location: str) -> None: ... diff --git a/object-store-rs/python/object_store_rs/_get.pyi b/object-store-rs/python/object_store_rs/_get.pyi deleted file mode 100644 index ce3d969..0000000 --- a/object-store-rs/python/object_store_rs/_get.pyi +++ /dev/null @@ -1,114 +0,0 @@ -from datetime import datetime -from typing import Sequence, TypedDict - -from ._list import ObjectMeta -from ._pyo3_object_store import ObjectStore -from ._sign import HTTP_METHOD as HTTP_METHOD -from ._sign import SignCapableStore as SignCapableStore -from ._sign import sign_url as sign_url -from ._sign import sign_url_async as sign_url_async - -class GetOptions(TypedDict): - """Options for a get request, such as range""" - - if_match: str | None - """ - Request will succeed if the `ObjectMeta::e_tag` matches - otherwise returning [`Error::Precondition`] - - See - - Examples: - - ```text - If-Match: "xyzzy" - If-Match: "xyzzy", "r2d2xxxx", "c3piozzzz" - If-Match: * - ``` - """ - - if_none_match: str | None - """ - Request will succeed if the `ObjectMeta::e_tag` does not match - otherwise returning [`Error::NotModified`] - - See - - Examples: - - ```text - If-None-Match: "xyzzy" - If-None-Match: "xyzzy", "r2d2xxxx", "c3piozzzz" - If-None-Match: * - ``` - """ - - if_unmodified_since: datetime | None - """ - Request will succeed if the object has been modified since - - - """ - - if_modified_since: datetime | None - """ - Request will succeed if the object has not been modified since - otherwise returning [`Error::Precondition`] - - Some stores, such as S3, will only return `NotModified` for exact - timestamp matches, instead of for any timestamp greater than or equal. - - - """ - - # range: - """ - Request transfer of only the specified range of bytes - otherwise returning [`Error::NotModified`] - - - """ - - version: str | None - """ - Request a particular object version - """ - - head: bool - """ - Request transfer of no content - - - """ - -class GetResult: - def bytes(self) -> bytes: - """ - Collects the data into bytes - """ - - async def bytes_async(self) -> bytes: - """ - Collects the data into bytes - """ - - @property - def meta(self) -> ObjectMeta: - """The ObjectMeta for this object""" - -def get( - store: ObjectStore, location: str, *, options: GetOptions | None = None -) -> GetResult: ... -async def get_async( - store: ObjectStore, location: str, *, options: GetOptions | None = None -) -> GetResult: ... -def get_range(store: ObjectStore, location: str, offset: int, length: int) -> bytes: ... -async def get_range_async( - store: ObjectStore, location: str, offset: int, length: int -) -> bytes: ... -def get_ranges( - store: ObjectStore, location: str, offset: Sequence[int], length: Sequence[int] -) -> bytes: ... -async def get_ranges_async( - store: ObjectStore, location: str, offset: Sequence[int], length: Sequence[int] -) -> bytes: ... diff --git a/object-store-rs/python/object_store_rs/_head.pyi b/object-store-rs/python/object_store_rs/_head.pyi deleted file mode 100644 index cc77152..0000000 --- a/object-store-rs/python/object_store_rs/_head.pyi +++ /dev/null @@ -1,5 +0,0 @@ -from ._list import ObjectMeta -from ._pyo3_object_store import ObjectStore - -def head(store: ObjectStore, location: str) -> ObjectMeta: ... -async def head_async(store: ObjectStore, location: str) -> ObjectMeta: ... diff --git a/object-store-rs/python/object_store_rs/_list.pyi b/object-store-rs/python/object_store_rs/_list.pyi deleted file mode 100644 index 4013c9c..0000000 --- a/object-store-rs/python/object_store_rs/_list.pyi +++ /dev/null @@ -1,45 +0,0 @@ -from datetime import datetime -from typing import List, TypedDict - -from ._pyo3_object_store import ObjectStore -from ._sign import HTTP_METHOD as HTTP_METHOD -from ._sign import SignCapableStore as SignCapableStore -from ._sign import sign_url as sign_url -from ._sign import sign_url_async as sign_url_async - -class ObjectMeta(TypedDict): - location: str - """The full path to the object""" - - last_modified: datetime - """The last modified time""" - - size: int - """The size in bytes of the object""" - - e_tag: str | None - """The unique identifier for the object - - - """ - - version: str | None - """A version indicator for this object""" - -class ListResult(TypedDict): - common_prefixes: List[str] - """Prefixes that are common (like directories)""" - - objects: List[ObjectMeta] - """Object metadata for the listing""" - -def list(store: ObjectStore, prefix: str | None = None) -> List[ObjectMeta]: ... -async def list_async( - store: ObjectStore, prefix: str | None = None -) -> List[ObjectMeta]: ... -def list_with_delimiter( - store: ObjectStore, prefix: str | None = None -) -> ListResult: ... -async def list_with_delimiter_async( - store: ObjectStore, prefix: str | None = None -) -> ListResult: ... diff --git a/object-store-rs/python/object_store_rs/_object_store_rs.pyi b/object-store-rs/python/object_store_rs/_object_store_rs.pyi deleted file mode 100644 index 443f718..0000000 --- a/object-store-rs/python/object_store_rs/_object_store_rs.pyi +++ /dev/null @@ -1,32 +0,0 @@ -from ._copy import copy as copy -from ._copy import copy_async as copy_async -from ._copy import copy_if_not_exists as copy_if_not_exists -from ._copy import copy_if_not_exists_async as copy_if_not_exists_async -from ._delete import delete as delete -from ._delete import delete_async as delete_async -from ._get import GetOptions as GetOptions -from ._get import GetResult as GetResult -from ._get import get as get -from ._get import get_async as get_async -from ._get import get_range as get_range -from ._get import get_range_async as get_range_async -from ._get import get_ranges as get_ranges -from ._get import get_ranges_async as get_ranges_async -from ._head import head as head -from ._head import head_async as head_async -from ._list import ListResult as ListResult -from ._list import ObjectMeta as ObjectMeta -from ._list import list as list -from ._list import list_async as list_async -from ._list import list_with_delimiter as list_with_delimiter -from ._list import list_with_delimiter_async as list_with_delimiter_async -from ._put import put_file as put_file -from ._put import put_file_async as put_file_async -from ._rename import rename as rename -from ._rename import rename_async as rename_async -from ._rename import rename_if_not_exists as rename_if_not_exists -from ._rename import rename_if_not_exists_async as rename_if_not_exists_async -from ._sign import HTTP_METHOD as HTTP_METHOD -from ._sign import SignCapableStore as SignCapableStore -from ._sign import sign_url as sign_url -from ._sign import sign_url_async as sign_url_async diff --git a/object-store-rs/python/object_store_rs/_put.pyi b/object-store-rs/python/object_store_rs/_put.pyi deleted file mode 100644 index fb5ee10..0000000 --- a/object-store-rs/python/object_store_rs/_put.pyi +++ /dev/null @@ -1,21 +0,0 @@ -from pathlib import Path -from typing import IO - -from ._pyo3_object_store import ObjectStore - -def put_file( - store: ObjectStore, - location: str, - file: IO[bytes] | Path | bytes, - *, - chunk_size: int = 5 * 1024, - max_concurrency: int = 12, -) -> None: ... -async def put_file_async( - store: ObjectStore, - location: str, - file: IO[bytes] | Path | bytes, - *, - chunk_size: int = 5 * 1024, - max_concurrency: int = 12, -) -> None: ... diff --git a/object-store-rs/python/object_store_rs/_pyo3_object_store.pyi b/object-store-rs/python/object_store_rs/_pyo3_object_store.pyi deleted file mode 100644 index d73d56c..0000000 --- a/object-store-rs/python/object_store_rs/_pyo3_object_store.pyi +++ /dev/null @@ -1,115 +0,0 @@ -# TODO: move this to a standalone package/docs website that can be shared across -# multiple python packages. - -from __future__ import annotations - -from datetime import timedelta -from typing import Dict, TypedDict - -import boto3 -import botocore -import botocore.session - -class BackoffConfig(TypedDict): - init_backoff: timedelta - max_backoff: timedelta - base: int | float - -class RetryConfig(TypedDict): - backoff: BackoffConfig - max_retries: int - retry_timeout: timedelta - -class AzureStore: - @classmethod - def from_env( - cls, - container: str, - *, - config: Dict[str, str] | None = None, - client_options: Dict[str, str] | None = None, - retry_config: RetryConfig | None = None, - ) -> S3Store: ... - @classmethod - def from_url( - cls, - url: str, - *, - config: Dict[str, str] | None = None, - client_options: Dict[str, str] | None = None, - retry_config: RetryConfig | None = None, - ) -> S3Store: ... - -class GCSStore: - @classmethod - def from_env( - cls, - bucket: str, - *, - config: Dict[str, str] | None = None, - client_options: Dict[str, str] | None = None, - retry_config: RetryConfig | None = None, - ) -> S3Store: ... - @classmethod - def from_url( - cls, - url: str, - *, - config: Dict[str, str] | None = None, - client_options: Dict[str, str] | None = None, - retry_config: RetryConfig | None = None, - ) -> S3Store: ... - -class HTTPStore: - @classmethod - def from_url( - cls, - url: str, - *, - client_options: Dict[str, str] | None = None, - retry_config: RetryConfig | None = None, - ) -> S3Store: ... - -class S3Store: - @classmethod - def from_env( - cls, - bucket: str, - *, - config: Dict[str, str] | None = None, - client_options: Dict[str, str] | None = None, - retry_config: RetryConfig | None = None, - ) -> S3Store: ... - @classmethod - def from_session( - cls, - session: boto3.Session | botocore.session.Session, - bucket: str, - *, - config: Dict[str, str] | None = None, - client_options: Dict[str, str] | None = None, - retry_config: RetryConfig | None = None, - ) -> S3Store: ... - @classmethod - def from_url( - cls, - url: str, - *, - config: Dict[str, str] | None = None, - client_options: Dict[str, str] | None = None, - retry_config: RetryConfig | None = None, - ) -> S3Store: ... - -class LocalStore: - """ - Local filesystem storage providing an ObjectStore interface to files on local disk. - Can optionally be created with a directory prefix. - - """ - def __init__(self, prefix: str | None = None) -> None: ... - -class MemoryStore: - """A fully in-memory implementation of ObjectStore.""" - def __init__(self) -> None: ... - -ObjectStore = AzureStore | GCSStore | HTTPStore | S3Store | LocalStore | MemoryStore diff --git a/object-store-rs/python/object_store_rs/_rename.pyi b/object-store-rs/python/object_store_rs/_rename.pyi deleted file mode 100644 index b3ce3ad..0000000 --- a/object-store-rs/python/object_store_rs/_rename.pyi +++ /dev/null @@ -1,8 +0,0 @@ -from ._pyo3_object_store import ObjectStore - -def rename(store: ObjectStore, from_: str, to: str) -> None: ... -async def rename_async(store: ObjectStore, from_: str, to: str) -> None: ... -def rename_if_not_exists(store: ObjectStore, from_: str, to: str) -> None: ... -async def rename_if_not_exists_async( - store: ObjectStore, from_: str, to: str -) -> None: ... diff --git a/object-store-rs/python/object_store_rs/_sign.pyi b/object-store-rs/python/object_store_rs/_sign.pyi deleted file mode 100644 index 41404a1..0000000 --- a/object-store-rs/python/object_store_rs/_sign.pyi +++ /dev/null @@ -1,16 +0,0 @@ -from datetime import timedelta -from typing import Literal - -from ._pyo3_object_store import AzureStore, GCSStore, S3Store - -HTTP_METHOD = Literal[ - "GET", "PUT", "POST", "HEAD", "PATCH", "TRACE", "DELETE", "OPTIONS", "CONNECT" -] -SignCapableStore = AzureStore | GCSStore | S3Store - -def sign_url( - store: SignCapableStore, method: HTTP_METHOD, path: str, expires_in: timedelta -) -> str: ... -async def sign_url_async( - store: SignCapableStore, method: HTTP_METHOD, path: str, expires_in: timedelta -) -> str: ... diff --git a/object-store-rs/src/copy.rs b/object-store-rs/src/copy.rs deleted file mode 100644 index 6976c89..0000000 --- a/object-store-rs/src/copy.rs +++ /dev/null @@ -1,68 +0,0 @@ -use object_store::ObjectStore; -use pyo3::prelude::*; -use pyo3_object_store::error::{PyObjectStoreError, PyObjectStoreResult}; -use pyo3_object_store::PyObjectStore; - -use crate::runtime::get_runtime; - -#[pyfunction] -pub(crate) fn copy( - py: Python, - store: PyObjectStore, - from_: String, - to: String, -) -> PyObjectStoreResult<()> { - let runtime = get_runtime(py)?; - py.allow_threads(|| { - runtime.block_on(store.as_ref().copy(&from_.into(), &to.into()))?; - Ok::<_, PyObjectStoreError>(()) - }) -} - -#[pyfunction] -pub(crate) fn copy_async( - py: Python, - store: PyObjectStore, - from_: String, - to: String, -) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - store - .as_ref() - .copy(&from_.into(), &to.into()) - .await - .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(()) - }) -} - -#[pyfunction] -pub(crate) fn copy_if_not_exists( - py: Python, - store: PyObjectStore, - from_: String, - to: String, -) -> PyObjectStoreResult<()> { - let runtime = get_runtime(py)?; - py.allow_threads(|| { - runtime.block_on(store.as_ref().copy_if_not_exists(&from_.into(), &to.into()))?; - Ok::<_, PyObjectStoreError>(()) - }) -} - -#[pyfunction] -pub(crate) fn copy_if_not_exists_async( - py: Python, - store: PyObjectStore, - from_: String, - to: String, -) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - store - .as_ref() - .copy_if_not_exists(&from_.into(), &to.into()) - .await - .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(()) - }) -} diff --git a/object-store-rs/src/delete.rs b/object-store-rs/src/delete.rs deleted file mode 100644 index aaf47b5..0000000 --- a/object-store-rs/src/delete.rs +++ /dev/null @@ -1,28 +0,0 @@ -use pyo3::prelude::*; -use pyo3_object_store::error::{PyObjectStoreError, PyObjectStoreResult}; -use pyo3_object_store::PyObjectStore; - -use crate::runtime::get_runtime; - -#[pyfunction] -pub fn delete(py: Python, store: PyObjectStore, location: String) -> PyObjectStoreResult<()> { - let runtime = get_runtime(py)?; - let store = store.into_inner(); - - py.allow_threads(|| { - runtime.block_on(store.delete(&location.into()))?; - Ok::<_, PyObjectStoreError>(()) - }) -} - -#[pyfunction] -pub fn delete_async(py: Python, store: PyObjectStore, location: String) -> PyResult> { - let store = store.into_inner().clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - store - .delete(&location.into()) - .await - .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(()) - }) -} diff --git a/object-store-rs/src/get.rs b/object-store-rs/src/get.rs deleted file mode 100644 index f417659..0000000 --- a/object-store-rs/src/get.rs +++ /dev/null @@ -1,211 +0,0 @@ -use chrono::{DateTime, Utc}; -use object_store::{GetOptions, GetResult, ObjectStore}; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyBytes; -use pyo3_object_store::error::{PyObjectStoreError, PyObjectStoreResult}; -use pyo3_object_store::PyObjectStore; - -use crate::list::PyObjectMeta; -use crate::runtime::get_runtime; - -#[derive(FromPyObject)] -pub(crate) struct PyGetOptions { - if_match: Option, - if_none_match: Option, - if_modified_since: Option>, - if_unmodified_since: Option>, - // TODO: - // range: Option>, - version: Option, - head: bool, -} - -impl From for GetOptions { - fn from(value: PyGetOptions) -> Self { - Self { - if_match: value.if_match, - if_none_match: value.if_none_match, - if_modified_since: value.if_modified_since, - if_unmodified_since: value.if_unmodified_since, - range: None, - version: value.version, - head: value.head, - } - } -} - -#[pyclass(name = "GetResult")] -pub(crate) struct PyGetResult(Option); - -impl PyGetResult { - fn new(result: GetResult) -> Self { - Self(Some(result)) - } -} - -#[pymethods] -impl PyGetResult { - fn bytes(&mut self, py: Python) -> PyObjectStoreResult { - let get_result = self - .0 - .take() - .ok_or(PyValueError::new_err("Result has already been disposed."))?; - let runtime = get_runtime(py)?; - py.allow_threads(|| { - let bytes = runtime.block_on(get_result.bytes())?; - Ok::<_, PyObjectStoreError>(PyBytesWrapper(bytes)) - }) - } - - fn bytes_async<'py>(&'py mut self, py: Python<'py>) -> PyResult> { - let get_result = self - .0 - .take() - .ok_or(PyValueError::new_err("Result has already been disposed."))?; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let bytes = get_result - .bytes() - .await - .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(PyBytesWrapper(bytes)) - }) - } - - #[getter] - fn meta(&self) -> PyResult { - let inner = self - .0 - .as_ref() - .ok_or(PyValueError::new_err("Result has already been disposed."))?; - Ok(PyObjectMeta::new(inner.meta.clone())) - } -} - -pub(crate) struct PyBytesWrapper(bytes::Bytes); - -// TODO: return buffer protocol object -impl IntoPy for PyBytesWrapper { - fn into_py(self, py: Python<'_>) -> PyObject { - PyBytes::new_bound(py, &self.0).into_py(py) - } -} - -#[pyfunction] -#[pyo3(signature = (store, location, *, options = None))] -pub(crate) fn get( - py: Python, - store: PyObjectStore, - location: String, - options: Option, -) -> PyObjectStoreResult { - let runtime = get_runtime(py)?; - py.allow_threads(|| { - let path = &location.into(); - let fut = if let Some(options) = options { - store.as_ref().get_opts(path, options.into()) - } else { - store.as_ref().get(path) - }; - let out = runtime.block_on(fut)?; - Ok::<_, PyObjectStoreError>(PyGetResult::new(out)) - }) -} - -#[pyfunction] -#[pyo3(signature = (store, location, *, options = None))] -pub(crate) fn get_async( - py: Python, - store: PyObjectStore, - location: String, - options: Option, -) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let path = &location.into(); - let fut = if let Some(options) = options { - store.as_ref().get_opts(path, options.into()) - } else { - store.as_ref().get(path) - }; - let out = fut.await.map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(PyGetResult::new(out)) - }) -} - -#[pyfunction] -pub(crate) fn get_range( - py: Python, - store: PyObjectStore, - location: String, - offset: usize, - length: usize, -) -> PyObjectStoreResult { - let runtime = get_runtime(py)?; - let range = offset..offset + length; - py.allow_threads(|| { - let out = runtime.block_on(store.as_ref().get_range(&location.into(), range))?; - Ok::<_, PyObjectStoreError>(PyBytesWrapper(out)) - }) -} - -#[pyfunction] -pub(crate) fn get_range_async( - py: Python, - store: PyObjectStore, - location: String, - offset: usize, - length: usize, -) -> PyResult> { - let range = offset..offset + length; - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let out = store - .as_ref() - .get_range(&location.into(), range) - .await - .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(PyBytesWrapper(out)) - }) -} - -#[pyfunction] -pub(crate) fn get_ranges( - py: Python, - store: PyObjectStore, - location: String, - offsets: Vec, - lengths: Vec, -) -> PyObjectStoreResult> { - let runtime = get_runtime(py)?; - let ranges = offsets - .into_iter() - .zip(lengths) - .map(|(offset, length)| offset..offset + length) - .collect::>(); - py.allow_threads(|| { - let out = runtime.block_on(store.as_ref().get_ranges(&location.into(), &ranges))?; - Ok::<_, PyObjectStoreError>(out.into_iter().map(PyBytesWrapper).collect()) - }) -} - -#[pyfunction] -pub(crate) fn get_ranges_async( - py: Python, - store: PyObjectStore, - location: String, - offsets: Vec, - lengths: Vec, -) -> PyResult> { - let ranges = offsets - .into_iter() - .zip(lengths) - .map(|(offset, length)| offset..offset + length) - .collect::>(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let out = store - .as_ref() - .get_ranges(&location.into(), &ranges) - .await - .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(out.into_iter().map(PyBytesWrapper).collect::>()) - }) -} diff --git a/object-store-rs/src/head.rs b/object-store-rs/src/head.rs deleted file mode 100644 index 5caf140..0000000 --- a/object-store-rs/src/head.rs +++ /dev/null @@ -1,33 +0,0 @@ -use pyo3::prelude::*; -use pyo3_object_store::error::{PyObjectStoreError, PyObjectStoreResult}; -use pyo3_object_store::PyObjectStore; - -use crate::list::PyObjectMeta; -use crate::runtime::get_runtime; - -#[pyfunction] -pub fn head( - py: Python, - store: PyObjectStore, - location: String, -) -> PyObjectStoreResult { - let runtime = get_runtime(py)?; - let store = store.into_inner(); - - py.allow_threads(|| { - let meta = runtime.block_on(store.head(&location.into()))?; - Ok::<_, PyObjectStoreError>(PyObjectMeta::new(meta)) - }) -} - -#[pyfunction] -pub fn head_async(py: Python, store: PyObjectStore, location: String) -> PyResult> { - let store = store.into_inner().clone(); - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let meta = store - .head(&location.into()) - .await - .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(PyObjectMeta::new(meta)) - }) -} diff --git a/object-store-rs/src/lib.rs b/object-store-rs/src/lib.rs deleted file mode 100644 index 33c942b..0000000 --- a/object-store-rs/src/lib.rs +++ /dev/null @@ -1,55 +0,0 @@ -use pyo3::prelude::*; - -mod copy; -mod delete; -mod get; -mod head; -mod list; -mod put; -mod rename; -mod runtime; -mod signer; - -const VERSION: &str = env!("CARGO_PKG_VERSION"); - -#[pyfunction] -fn ___version() -> &'static str { - VERSION -} - -/// A Python module implemented in Rust. -#[pymodule] -fn _object_store_rs(py: Python, m: &Bound) -> PyResult<()> { - m.add_wrapped(wrap_pyfunction!(___version))?; - - pyo3_object_store::register_store_module(py, m, "object_store_rs")?; - - m.add_wrapped(wrap_pyfunction!(copy::copy_async))?; - m.add_wrapped(wrap_pyfunction!(copy::copy_if_not_exists_async))?; - m.add_wrapped(wrap_pyfunction!(copy::copy_if_not_exists))?; - m.add_wrapped(wrap_pyfunction!(copy::copy))?; - m.add_wrapped(wrap_pyfunction!(delete::delete_async))?; - m.add_wrapped(wrap_pyfunction!(delete::delete))?; - m.add_wrapped(wrap_pyfunction!(get::get_async))?; - m.add_wrapped(wrap_pyfunction!(get::get_range_async))?; - m.add_wrapped(wrap_pyfunction!(get::get_range))?; - m.add_wrapped(wrap_pyfunction!(get::get_ranges_async))?; - m.add_wrapped(wrap_pyfunction!(get::get_ranges))?; - m.add_wrapped(wrap_pyfunction!(get::get))?; - m.add_wrapped(wrap_pyfunction!(head::head_async))?; - m.add_wrapped(wrap_pyfunction!(head::head))?; - m.add_wrapped(wrap_pyfunction!(list::list_async))?; - m.add_wrapped(wrap_pyfunction!(list::list_with_delimiter_async))?; - m.add_wrapped(wrap_pyfunction!(list::list_with_delimiter))?; - m.add_wrapped(wrap_pyfunction!(list::list))?; - m.add_wrapped(wrap_pyfunction!(put::put_file_async))?; - m.add_wrapped(wrap_pyfunction!(put::put_file))?; - m.add_wrapped(wrap_pyfunction!(rename::rename_async))?; - m.add_wrapped(wrap_pyfunction!(rename::rename_if_not_exists_async))?; - m.add_wrapped(wrap_pyfunction!(rename::rename_if_not_exists))?; - m.add_wrapped(wrap_pyfunction!(rename::rename))?; - m.add_wrapped(wrap_pyfunction!(signer::sign_url_async))?; - m.add_wrapped(wrap_pyfunction!(signer::sign_url))?; - - Ok(()) -} diff --git a/object-store-rs/src/list.rs b/object-store-rs/src/list.rs deleted file mode 100644 index f84e0a8..0000000 --- a/object-store-rs/src/list.rs +++ /dev/null @@ -1,136 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; - -use futures::TryStreamExt; -use object_store::path::Path; -use object_store::{ListResult, ObjectMeta, ObjectStore}; -use pyo3::prelude::*; -use pyo3_object_store::error::{PyObjectStoreError, PyObjectStoreResult}; -use pyo3_object_store::PyObjectStore; - -use crate::runtime::get_runtime; - -pub(crate) struct PyObjectMeta(ObjectMeta); - -impl PyObjectMeta { - pub(crate) fn new(meta: ObjectMeta) -> Self { - Self(meta) - } -} - -impl IntoPy for PyObjectMeta { - fn into_py(self, py: Python<'_>) -> PyObject { - let mut dict = HashMap::new(); - dict.insert("location", self.0.location.as_ref().into_py(py)); - dict.insert("last_modified", self.0.last_modified.into_py(py)); - dict.insert("size", self.0.size.into_py(py)); - dict.insert("e_tag", self.0.e_tag.into_py(py)); - dict.insert("version", self.0.version.into_py(py)); - dict.into_py(py) - } -} - -pub(crate) struct PyListResult(ListResult); - -impl IntoPy for PyListResult { - fn into_py(self, py: Python<'_>) -> PyObject { - let mut dict = HashMap::new(); - dict.insert( - "common_prefixes", - self.0 - .common_prefixes - .into_iter() - .map(String::from) - .collect::>() - .into_py(py), - ); - dict.insert( - "objects", - self.0 - .objects - .into_iter() - .map(PyObjectMeta) - .collect::>() - .into_py(py), - ); - dict.into_py(py) - } -} - -#[pyfunction] -#[pyo3(signature = (store, prefix = None))] -pub(crate) fn list( - py: Python, - store: PyObjectStore, - prefix: Option, -) -> PyObjectStoreResult> { - let runtime = get_runtime(py)?; - py.allow_threads(|| { - let out = runtime.block_on(list_materialize( - store.into_inner(), - prefix.map(|s| s.into()).as_ref(), - ))?; - Ok::<_, PyObjectStoreError>(out) - }) -} - -#[pyfunction] -#[pyo3(signature = (store, prefix = None))] -pub(crate) fn list_async( - py: Python, - store: PyObjectStore, - prefix: Option, -) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let out = list_materialize(store.into_inner(), prefix.map(|s| s.into()).as_ref()).await?; - Ok(out) - }) -} - -async fn list_materialize( - store: Arc, - prefix: Option<&Path>, -) -> PyObjectStoreResult> { - let list_result = store.list(prefix).try_collect::>().await?; - Ok(list_result.into_iter().map(PyObjectMeta).collect()) -} - -#[pyfunction] -#[pyo3(signature = (store, prefix = None))] -pub(crate) fn list_with_delimiter( - py: Python, - store: PyObjectStore, - prefix: Option, -) -> PyObjectStoreResult { - let runtime = get_runtime(py)?; - py.allow_threads(|| { - let out = runtime.block_on(list_with_delimiter_materialize( - store.into_inner(), - prefix.map(|s| s.into()).as_ref(), - ))?; - Ok::<_, PyObjectStoreError>(out) - }) -} - -#[pyfunction] -#[pyo3(signature = (store, prefix = None))] -pub(crate) fn list_with_delimiter_async( - py: Python, - store: PyObjectStore, - prefix: Option, -) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - let out = - list_with_delimiter_materialize(store.into_inner(), prefix.map(|s| s.into()).as_ref()) - .await?; - Ok(out) - }) -} - -async fn list_with_delimiter_materialize( - store: Arc, - prefix: Option<&Path>, -) -> PyObjectStoreResult { - let list_result = store.list_with_delimiter(prefix).await?; - Ok(PyListResult(list_result)) -} diff --git a/object-store-rs/src/put.rs b/object-store-rs/src/put.rs deleted file mode 100644 index fcdf83f..0000000 --- a/object-store-rs/src/put.rs +++ /dev/null @@ -1,119 +0,0 @@ -use std::fs::File; -use std::io::{BufReader, Cursor, Read}; -use std::path::PathBuf; -use std::sync::Arc; - -use object_store::path::Path; -use object_store::{ObjectStore, WriteMultipart}; -use pyo3::exceptions::PyIOError; -use pyo3::prelude::*; -use pyo3::pybacked::PyBackedBytes; -use pyo3_file::PyFileLikeObject; -use pyo3_object_store::error::PyObjectStoreResult; -use pyo3_object_store::PyObjectStore; - -use crate::runtime::get_runtime; - -/// Input types supported by multipart upload -#[derive(Debug)] -pub(crate) enum MultipartPutInput { - File(BufReader), - FileLike(PyFileLikeObject), - Buffer(Cursor), -} - -impl<'py> FromPyObject<'py> for MultipartPutInput { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - let py = ob.py(); - if let Ok(path) = ob.extract::() { - Ok(Self::File(BufReader::new(File::open(path)?))) - } else if let Ok(buffer) = ob.extract::() { - Ok(Self::Buffer(Cursor::new(buffer))) - } else { - Ok(Self::FileLike(PyFileLikeObject::with_requirements( - ob.into_py(py), - true, - false, - true, - false, - )?)) - } - } -} - -impl Read for MultipartPutInput { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - match self { - Self::File(f) => f.read(buf), - Self::FileLike(f) => f.read(buf), - Self::Buffer(f) => f.read(buf), - } - } -} - -#[pyfunction] -#[pyo3(signature = (store, location, file, *, chunk_size = 5120, max_concurrency = 12))] -pub(crate) fn put_file( - py: Python, - store: PyObjectStore, - location: String, - file: MultipartPutInput, - chunk_size: usize, - max_concurrency: usize, -) -> PyObjectStoreResult<()> { - let runtime = get_runtime(py)?; - runtime.block_on(put_multipart_inner( - store.into_inner(), - &location.into(), - file, - chunk_size, - max_concurrency, - )) -} - -#[pyfunction] -#[pyo3(signature = (store, location, file, *, chunk_size = 5120, max_concurrency = 12))] -pub(crate) fn put_file_async( - py: Python, - store: PyObjectStore, - location: String, - file: MultipartPutInput, - chunk_size: usize, - max_concurrency: usize, -) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - Ok(put_multipart_inner( - store.into_inner(), - &location.into(), - file, - chunk_size, - max_concurrency, - ) - .await?) - }) -} - -async fn put_multipart_inner( - store: Arc, - location: &Path, - mut reader: R, - chunk_size: usize, - max_concurrency: usize, -) -> PyObjectStoreResult<()> { - let upload = store.put_multipart(location).await?; - let mut write = WriteMultipart::new(upload); - let mut scratch_buffer = vec![0; chunk_size]; - loop { - let read_size = reader - .read(&mut scratch_buffer) - .map_err(|err| PyIOError::new_err(err.to_string()))?; - if read_size == 0 { - break; - } else { - write.wait_for_capacity(max_concurrency).await?; - write.write(&scratch_buffer[0..read_size]); - } - } - write.finish().await?; - Ok(()) -} diff --git a/object-store-rs/src/rename.rs b/object-store-rs/src/rename.rs deleted file mode 100644 index 08a4440..0000000 --- a/object-store-rs/src/rename.rs +++ /dev/null @@ -1,72 +0,0 @@ -use object_store::ObjectStore; -use pyo3::prelude::*; -use pyo3_object_store::error::{PyObjectStoreError, PyObjectStoreResult}; -use pyo3_object_store::PyObjectStore; - -use crate::runtime::get_runtime; - -#[pyfunction] -pub(crate) fn rename( - py: Python, - store: PyObjectStore, - from_: String, - to: String, -) -> PyObjectStoreResult<()> { - let runtime = get_runtime(py)?; - py.allow_threads(|| { - runtime.block_on(store.as_ref().rename(&from_.into(), &to.into()))?; - Ok::<_, PyObjectStoreError>(()) - }) -} - -#[pyfunction] -pub(crate) fn rename_async( - py: Python, - store: PyObjectStore, - from_: String, - to: String, -) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - store - .as_ref() - .rename(&from_.into(), &to.into()) - .await - .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(()) - }) -} - -#[pyfunction] -pub(crate) fn rename_if_not_exists( - py: Python, - store: PyObjectStore, - from_: String, - to: String, -) -> PyObjectStoreResult<()> { - let runtime = get_runtime(py)?; - py.allow_threads(|| { - runtime.block_on( - store - .as_ref() - .rename_if_not_exists(&from_.into(), &to.into()), - )?; - Ok::<_, PyObjectStoreError>(()) - }) -} - -#[pyfunction] -pub(crate) fn rename_if_not_exists_async( - py: Python, - store: PyObjectStore, - from_: String, - to: String, -) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - store - .as_ref() - .rename_if_not_exists(&from_.into(), &to.into()) - .await - .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(()) - }) -} diff --git a/object-store-rs/src/runtime.rs b/object-store-rs/src/runtime.rs deleted file mode 100644 index 9659387..0000000 --- a/object-store-rs/src/runtime.rs +++ /dev/null @@ -1,18 +0,0 @@ -use std::sync::Arc; - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::sync::GILOnceCell; -use tokio::runtime::Runtime; - -static RUNTIME: GILOnceCell> = GILOnceCell::new(); - -/// Get the tokio runtime for sync requests -pub(crate) fn get_runtime(py: Python<'_>) -> PyResult> { - let runtime = RUNTIME.get_or_try_init(py, || { - Ok::<_, PyErr>(Arc::new(Runtime::new().map_err(|err| { - PyValueError::new_err(format!("Could not create tokio runtime. {}", err)) - })?)) - })?; - Ok(runtime.clone()) -} diff --git a/object-store-rs/src/signer.rs b/object-store-rs/src/signer.rs deleted file mode 100644 index 339eb3d..0000000 --- a/object-store-rs/src/signer.rs +++ /dev/null @@ -1,161 +0,0 @@ -use core::time::Duration; -use std::sync::Arc; - -use object_store::aws::AmazonS3; -use object_store::azure::MicrosoftAzure; -use object_store::gcp::GoogleCloudStorage; -use object_store::signer::Signer; -use pyo3::exceptions::PyValueError; -use pyo3::intern; -use pyo3::prelude::*; -use pyo3::pybacked::PyBackedStr; -use pyo3_object_store::error::{PyObjectStoreError, PyObjectStoreResult}; -use pyo3_object_store::{PyAzureStore, PyGCSStore, PyS3Store}; -use url::Url; - -use crate::runtime::get_runtime; - -#[derive(Debug)] -pub(crate) enum SignCapableStore { - S3(Arc), - Gcs(Arc), - Azure(Arc), -} - -impl<'py> FromPyObject<'py> for SignCapableStore { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - if let Ok(store) = ob.downcast::() { - Ok(Self::S3(store.borrow().as_ref().clone())) - } else if let Ok(store) = ob.downcast::() { - Ok(Self::Gcs(store.borrow().as_ref().clone())) - } else if let Ok(store) = ob.downcast::() { - Ok(Self::Azure(store.borrow().as_ref().clone())) - } else { - let py = ob.py(); - // Check for object-store instance from other library - let cls_name = ob - .getattr(intern!(py, "__class__"))? - .getattr(intern!(py, "__name__"))? - .extract::()?; - if [ - "AzureStore", - "GCSStore", - "HTTPStore", - "LocalStore", - "MemoryStore", - "S3Store", - ] - .contains(&cls_name.as_ref()) - { - return Err(PyValueError::new_err("You must use an object store instance exported from **the same library** as this function. They cannot be used across libraries.\nThis is because object store instances are compiled with a specific version of Rust and Python." )); - } - - Err(PyValueError::new_err(format!( - "Expected an S3Store, GCSStore, or AzureStore instance, got {}", - ob.repr()? - ))) - } - } -} - -impl Signer for SignCapableStore { - fn signed_url<'life0, 'life1, 'async_trait>( - &'life0 self, - method: http::Method, - path: &'life1 object_store::path::Path, - expires_in: Duration, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, - { - match self { - Self::S3(inner) => inner.signed_url(method, path, expires_in), - Self::Gcs(inner) => inner.signed_url(method, path, expires_in), - Self::Azure(inner) => inner.signed_url(method, path, expires_in), - } - } -} - -pub(crate) struct PyMethod(http::Method); - -impl<'py> FromPyObject<'py> for PyMethod { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - let s = ob.extract::()?; - let method = match s.as_ref() { - "GET" => http::Method::GET, - "PUT" => http::Method::PUT, - "POST" => http::Method::POST, - "HEAD" => http::Method::HEAD, - "PATCH" => http::Method::PATCH, - "TRACE" => http::Method::TRACE, - "DELETE" => http::Method::DELETE, - "OPTIONS" => http::Method::OPTIONS, - "CONNECT" => http::Method::CONNECT, - other => { - return Err(PyValueError::new_err(format!( - "Unsupported HTTP method {}", - other - ))) - } - }; - Ok(Self(method)) - } -} - -pub(crate) struct PyUrl(url::Url); - -impl IntoPy for PyUrl { - fn into_py(self, _py: Python<'_>) -> String { - self.0.into() - } -} - -impl IntoPy for PyUrl { - fn into_py(self, py: Python<'_>) -> PyObject { - String::from(self.0).into_py(py) - } -} - -#[pyfunction] -pub(crate) fn sign_url( - py: Python, - store: SignCapableStore, - method: PyMethod, - path: String, - expires_in: Duration, -) -> PyObjectStoreResult { - let runtime = get_runtime(py)?; - let method = method.0; - - let signed_url = py.allow_threads(|| { - let url = runtime.block_on(store.signed_url(method, &path.into(), expires_in))?; - Ok::<_, object_store::Error>(url) - })?; - Ok(signed_url.into()) -} - -#[pyfunction] -pub(crate) fn sign_url_async( - py: Python, - store: SignCapableStore, - method: PyMethod, - path: String, - expires_in: Duration, -) -> PyResult { - let fut = pyo3_async_runtimes::tokio::future_into_py(py, async move { - let url = store - .signed_url(method.0, &path.into(), expires_in) - .await - .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(PyUrl(url)) - })?; - Ok(fut.into()) -} diff --git a/pyo3-object_store/Cargo.toml b/pyo3-object_store/Cargo.toml deleted file mode 100644 index 3b4dfd9..0000000 --- a/pyo3-object_store/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "pyo3-object_store" -version = "0.1.0" -authors = { workspace = true } -edition = { workspace = true } -description = "object_store integration for pyo3." -readme = "README.md" -repository = { workspace = true } -license = { workspace = true } -keywords = { workspace = true } -categories = { workspace = true } -rust-version = { workspace = true } - -[dependencies] -futures = "0.3.30" -object_store = { workspace = true, features = ["aws", "azure", "gcp", "http"] } -pyo3 = { workspace = true, features = ["chrono", "indexmap"] } -pyo3-async-runtimes = { git = "https://github.com/PyO3/pyo3-async-runtimes", features = [ - "tokio-runtime", -] } -thiserror = { workspace = true } - -[dev-dependencies] -arrow-select = { workspace = true } - -[lib] -crate-type = ["rlib"] diff --git a/pyo3-object_store/README.md b/pyo3-object_store/README.md deleted file mode 100644 index c8f0b66..0000000 --- a/pyo3-object_store/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# pyo3-object_store - -Use `object-store` in your pyo3-based libraries. diff --git a/pyo3-object_store/src/api.rs b/pyo3-object_store/src/api.rs deleted file mode 100644 index bcd7619..0000000 --- a/pyo3-object_store/src/api.rs +++ /dev/null @@ -1,35 +0,0 @@ -use pyo3::intern; -use pyo3::prelude::*; - -use crate::{PyAzureStore, PyGCSStore, PyHttpStore, PyLocalStore, PyMemoryStore, PyS3Store}; - -/// Export the default Python API as a submodule named "store" within the given parent module -// https://github.com/PyO3/pyo3/issues/1517#issuecomment-808664021 -// https://github.com/PyO3/pyo3/issues/759#issuecomment-977835119 -pub fn register_store_module( - py: Python<'_>, - parent_module: &Bound<'_, PyModule>, - parent_module_str: &str, -) -> PyResult<()> { - let full_module_string = format!("{}.store", parent_module_str); - - let child_module = PyModule::new_bound(parent_module.py(), "store")?; - - child_module.add_class::()?; - child_module.add_class::()?; - child_module.add_class::()?; - child_module.add_class::()?; - child_module.add_class::()?; - child_module.add_class::()?; - - parent_module.add_submodule(&child_module)?; - - py.import_bound(intern!(py, "sys"))? - .getattr(intern!(py, "modules"))? - .set_item(full_module_string.as_str(), child_module.to_object(py))?; - - // needs to be set *after* `add_submodule()` - child_module.setattr("__name__", full_module_string)?; - - Ok(()) -} diff --git a/pyo3-object_store/src/aws.rs b/pyo3-object_store/src/aws.rs deleted file mode 100644 index de22295..0000000 --- a/pyo3-object_store/src/aws.rs +++ /dev/null @@ -1,152 +0,0 @@ -use std::collections::HashMap; -use std::str::FromStr; -use std::sync::Arc; - -use object_store::aws::{AmazonS3, AmazonS3Builder, AmazonS3ConfigKey}; -use pyo3::intern; -use pyo3::prelude::*; -use pyo3::pybacked::PyBackedStr; -use pyo3::types::PyType; - -use crate::client::PyClientOptions; -use crate::error::{PyObjectStoreError, PyObjectStoreResult}; -use crate::retry::PyRetryConfig; - -#[pyclass(name = "S3Store")] -pub struct PyS3Store(Arc); - -impl AsRef> for PyS3Store { - fn as_ref(&self) -> &Arc { - &self.0 - } -} - -impl PyS3Store { - pub fn into_inner(self) -> Arc { - self.0 - } -} - -#[pymethods] -impl PyS3Store { - // Create from env variables - #[classmethod] - #[pyo3(signature = (bucket, *, config=None, client_options=None, retry_config=None))] - fn from_env( - _cls: &Bound, - bucket: String, - config: Option>, - client_options: Option, - retry_config: Option, - ) -> PyObjectStoreResult { - let mut builder = AmazonS3Builder::from_env().with_bucket_name(bucket); - if let Some(config) = config { - for (key, value) in config.into_iter() { - builder = builder.with_config(key.0, value); - } - } - if let Some(client_options) = client_options { - builder = builder.with_client_options(client_options.into()) - } - if let Some(retry_config) = retry_config { - builder = builder.with_retry(retry_config.into()) - } - Ok(Self(Arc::new(builder.build()?))) - } - - // Create from an existing boto3.Session or botocore.session.Session object - // https://stackoverflow.com/a/36291428 - #[classmethod] - #[pyo3(signature = (session, bucket, *, config=None, client_options=None, retry_config=None))] - fn from_session( - _cls: &Bound, - py: Python, - session: &Bound, - bucket: String, - config: Option>, - client_options: Option, - retry_config: Option, - ) -> PyObjectStoreResult { - // boto3.Session has a region_name attribute, but botocore.session.Session does not. - let region = if let Ok(region) = session.getattr(intern!(py, "region_name")) { - Some(region.extract::()?) - } else { - None - }; - - let creds = session.call_method0(intern!(py, "get_credentials"))?; - let frozen_creds = creds.call_method0(intern!(py, "get_frozen_credentials"))?; - - let access_key = frozen_creds - .getattr(intern!(py, "access_key"))? - .extract::>()?; - let secret_key = frozen_creds - .getattr(intern!(py, "secret_key"))? - .extract::>()?; - let token = frozen_creds - .getattr(intern!(py, "token"))? - .extract::>()?; - - let mut builder = AmazonS3Builder::new().with_bucket_name(bucket); - if let Some(region) = region { - builder = builder.with_region(region); - } - if let Some(access_key) = access_key { - builder = builder.with_access_key_id(access_key); - } - if let Some(secret_key) = secret_key { - builder = builder.with_secret_access_key(secret_key); - } - if let Some(token) = token { - builder = builder.with_token(token); - } - if let Some(config) = config { - for (key, value) in config.into_iter() { - builder = builder.with_config(key.0, value); - } - } - if let Some(client_options) = client_options { - builder = builder.with_client_options(client_options.into()) - } - if let Some(retry_config) = retry_config { - builder = builder.with_retry(retry_config.into()) - } - - Ok(Self(Arc::new(builder.build()?))) - } - - #[classmethod] - #[pyo3(signature = (url, *, config=None, client_options=None, retry_config=None))] - fn from_url( - _cls: &Bound, - url: &str, - config: Option>, - client_options: Option, - retry_config: Option, - ) -> PyObjectStoreResult { - let mut builder = AmazonS3Builder::from_env().with_url(url); - if let Some(config) = config { - for (key, value) in config.into_iter() { - builder = builder.with_config(key.0, value); - } - } - if let Some(client_options) = client_options { - builder = builder.with_client_options(client_options.into()) - } - if let Some(retry_config) = retry_config { - builder = builder.with_retry(retry_config.into()) - } - Ok(Self(Arc::new(builder.build()?))) - } -} - -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct PyAmazonS3ConfigKey(AmazonS3ConfigKey); - -impl<'py> FromPyObject<'py> for PyAmazonS3ConfigKey { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - let s = ob.extract::()?.to_lowercase(); - let key = AmazonS3ConfigKey::from_str(&s).map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(Self(key)) - } -} diff --git a/pyo3-object_store/src/azure.rs b/pyo3-object_store/src/azure.rs deleted file mode 100644 index 2cee960..0000000 --- a/pyo3-object_store/src/azure.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::collections::HashMap; -use std::str::FromStr; -use std::sync::Arc; - -use object_store::azure::{AzureConfigKey, MicrosoftAzure, MicrosoftAzureBuilder}; -use pyo3::prelude::*; -use pyo3::pybacked::PyBackedStr; -use pyo3::types::PyType; - -use crate::client::PyClientOptions; -use crate::error::{PyObjectStoreError, PyObjectStoreResult}; -use crate::retry::PyRetryConfig; - -#[pyclass(name = "AzureStore")] -pub struct PyAzureStore(Arc); - -impl AsRef> for PyAzureStore { - fn as_ref(&self) -> &Arc { - &self.0 - } -} - -impl PyAzureStore { - pub fn into_inner(self) -> Arc { - self.0 - } -} - -#[pymethods] -impl PyAzureStore { - // Create from env variables - #[classmethod] - #[pyo3(signature = (container, *, config=None, client_options=None, retry_config=None))] - fn from_env( - _cls: &Bound, - container: String, - config: Option>, - client_options: Option, - retry_config: Option, - ) -> PyObjectStoreResult { - let mut builder = MicrosoftAzureBuilder::from_env().with_container_name(container); - if let Some(config) = config { - for (key, value) in config.into_iter() { - builder = builder.with_config(key.0, value); - } - } - if let Some(client_options) = client_options { - builder = builder.with_client_options(client_options.into()) - } - if let Some(retry_config) = retry_config { - builder = builder.with_retry(retry_config.into()) - } - Ok(Self(Arc::new(builder.build()?))) - } - - #[classmethod] - #[pyo3(signature = (url, *, config=None, client_options=None, retry_config=None))] - fn from_url( - _cls: &Bound, - url: &str, - config: Option>, - client_options: Option, - retry_config: Option, - ) -> PyObjectStoreResult { - let mut builder = MicrosoftAzureBuilder::from_env().with_url(url); - if let Some(config) = config { - for (key, value) in config.into_iter() { - builder = builder.with_config(key.0, value); - } - } - if let Some(client_options) = client_options { - builder = builder.with_client_options(client_options.into()) - } - if let Some(retry_config) = retry_config { - builder = builder.with_retry(retry_config.into()) - } - Ok(Self(Arc::new(builder.build()?))) - } -} - -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct PyAzureConfigKey(AzureConfigKey); - -impl<'py> FromPyObject<'py> for PyAzureConfigKey { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - let s = ob.extract::()?.to_lowercase(); - let key = AzureConfigKey::from_str(&s).map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(Self(key)) - } -} diff --git a/pyo3-object_store/src/client.rs b/pyo3-object_store/src/client.rs deleted file mode 100644 index 21ecba4..0000000 --- a/pyo3-object_store/src/client.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::collections::HashMap; -use std::str::FromStr; - -use object_store::{ClientConfigKey, ClientOptions}; -use pyo3::prelude::*; -use pyo3::pybacked::PyBackedStr; - -use crate::error::PyObjectStoreError; - -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct PyClientConfigKey(ClientConfigKey); - -impl<'py> FromPyObject<'py> for PyClientConfigKey { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - let s = ob.extract::()?.to_lowercase(); - let key = ClientConfigKey::from_str(&s).map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(Self(key)) - } -} - -#[derive(Debug)] -pub struct PyClientOptions(ClientOptions); - -impl<'py> FromPyObject<'py> for PyClientOptions { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - let py_input = ob.extract::>()?; - let mut options = ClientOptions::new(); - for (key, value) in py_input.into_iter() { - options = options.with_config(key.0, value); - } - Ok(Self(options)) - } -} - -impl From for ClientOptions { - fn from(value: PyClientOptions) -> Self { - value.0 - } -} diff --git a/pyo3-object_store/src/error.rs b/pyo3-object_store/src/error.rs deleted file mode 100644 index 3136d1a..0000000 --- a/pyo3-object_store/src/error.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! Contains the [`PyObjectStoreError`], the Error returned by most fallible functions in this -//! crate. - -use pyo3::exceptions::{PyException, PyValueError}; -use pyo3::prelude::*; -use pyo3::DowncastError; -use thiserror::Error; - -/// The Error variants returned by this crate. -#[derive(Error, Debug)] -#[non_exhaustive] -pub enum PyObjectStoreError { - /// A wrapped [object_store::Error] - #[error(transparent)] - ObjectStoreError(#[from] object_store::Error), - - /// A wrapped [PyErr] - #[error(transparent)] - PyErr(#[from] PyErr), -} - -impl From for PyErr { - fn from(error: PyObjectStoreError) -> Self { - match error { - PyObjectStoreError::PyErr(err) => err, - PyObjectStoreError::ObjectStoreError(err) => PyException::new_err(err.to_string()), - } - } -} - -impl<'a, 'py> From> for PyObjectStoreError { - fn from(other: DowncastError<'a, 'py>) -> Self { - Self::PyErr(PyValueError::new_err(format!( - "Could not downcast: {}", - other - ))) - } -} - -/// A type wrapper around `Result`. -pub type PyObjectStoreResult = Result; diff --git a/pyo3-object_store/src/fsspec.rs b/pyo3-object_store/src/fsspec.rs deleted file mode 100644 index 5575378..0000000 --- a/pyo3-object_store/src/fsspec.rs +++ /dev/null @@ -1,198 +0,0 @@ -use std::fmt::Display; -use std::sync::Arc; - -// use bytes::{Buf, Bytes}; -use futures::future::{BoxFuture, FutureExt}; -use object_store::path::Path; -use object_store::ObjectStore; -use pyo3::intern; -use pyo3::prelude::*; -use pyo3::types::{PyList, PyTuple}; - -/// A wrapper around an Async fsspec filesystem instance -#[derive(Debug)] -pub struct AsyncFsspec { - fs: PyObject, - path: String, - file_length: usize, -} - -impl Display for AsyncFsspec { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // TODO: get the __repr__ of the underlying fsspec object, and print like - // "RustWrapper(fsspec_repr)" - write!(f, "AsyncFsspec") - } -} - -impl AsyncFsspec { - pub fn new(fs: PyObject, path: String, file_length: usize) -> Self { - // TODO: verify isinstance of fsspec base class - // TODO: verify is async - Self { - fs, - path, - file_length, - } - } -} - -impl ObjectStore for AsyncFsspec { - fn put_opts<'life0, 'life1, 'async_trait>( - &'life0 self, - location: &'life1 object_store::path::Path, - payload: object_store::PutPayload, - opts: object_store::PutOptions, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, - { - // put_file - todo!() - } - - fn put_multipart_opts<'life0, 'life1, 'async_trait>( - &'life0 self, - location: &'life1 Path, - opts: object_store::PutMultipartOpts, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future< - Output = object_store::Result>, - > + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, - { - todo!() - } - - fn get_opts<'life0, 'life1, 'async_trait>( - &'life0 self, - location: &'life1 Path, - options: object_store::GetOptions, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, - { - todo!() - } - - fn delete<'life0, 'life1, 'async_trait>( - &'life0 self, - location: &'life1 Path, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, - { - async move { - Python::with_gil(|py| -> PyResult<_> { - let path = location.to_string(); - let args = PyTuple::new_bound(py, vec![path]); - - let coroutine = self.fs.call_method1(py, intern!(py, "_rm"), args)?; - pyo3_async_runtimes::tokio::into_future(coroutine.bind(py).clone()) - }) - .unwrap() - .await - .unwrap(); - - Ok(()) - } - .boxed() - } - - fn list( - &self, - prefix: Option<&Path>, - ) -> futures::stream::BoxStream<'_, object_store::Result> { - todo!() - } - - fn list_with_delimiter<'life0, 'life1, 'async_trait>( - &'life0 self, - prefix: Option<&'life1 Path>, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - Self: 'async_trait, - { - todo!() - } - - fn copy<'life0, 'life1, 'life2, 'async_trait>( - &'life0 self, - from: &'life1 Path, - to: &'life2 Path, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - 'life2: 'async_trait, - Self: 'async_trait, - { - // .copy - todo!() - } - - fn copy_if_not_exists<'life0, 'life1, 'life2, 'async_trait>( - &'life0 self, - from: &'life1 Path, - to: &'life2 Path, - ) -> ::core::pin::Pin< - Box< - dyn ::core::future::Future> - + ::core::marker::Send - + 'async_trait, - >, - > - where - 'life0: 'async_trait, - 'life1: 'async_trait, - 'life2: 'async_trait, - Self: 'async_trait, - { - todo!() - } -} diff --git a/pyo3-object_store/src/gcp.rs b/pyo3-object_store/src/gcp.rs deleted file mode 100644 index 5780776..0000000 --- a/pyo3-object_store/src/gcp.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::collections::HashMap; -use std::str::FromStr; -use std::sync::Arc; - -use object_store::gcp::{GoogleCloudStorage, GoogleCloudStorageBuilder, GoogleConfigKey}; -use pyo3::prelude::*; -use pyo3::pybacked::PyBackedStr; -use pyo3::types::PyType; - -use crate::client::PyClientOptions; -use crate::error::{PyObjectStoreError, PyObjectStoreResult}; -use crate::retry::PyRetryConfig; - -#[pyclass(name = "GCSStore")] -pub struct PyGCSStore(Arc); - -impl AsRef> for PyGCSStore { - fn as_ref(&self) -> &Arc { - &self.0 - } -} - -impl PyGCSStore { - pub fn into_inner(self) -> Arc { - self.0 - } -} - -#[pymethods] -impl PyGCSStore { - // Create from env variables - #[classmethod] - #[pyo3(signature = (bucket, *, config=None, client_options=None, retry_config=None))] - fn from_env( - _cls: &Bound, - bucket: String, - config: Option>, - client_options: Option, - retry_config: Option, - ) -> PyObjectStoreResult { - let mut builder = GoogleCloudStorageBuilder::from_env().with_bucket_name(bucket); - if let Some(config) = config { - for (key, value) in config.into_iter() { - builder = builder.with_config(key.0, value); - } - } - if let Some(client_options) = client_options { - builder = builder.with_client_options(client_options.into()) - } - if let Some(retry_config) = retry_config { - builder = builder.with_retry(retry_config.into()) - } - Ok(Self(Arc::new(builder.build()?))) - } - - #[classmethod] - #[pyo3(signature = (url, *, config=None, client_options=None, retry_config=None))] - fn from_url( - _cls: &Bound, - url: &str, - config: Option>, - client_options: Option, - retry_config: Option, - ) -> PyObjectStoreResult { - let mut builder = GoogleCloudStorageBuilder::from_env().with_url(url); - if let Some(config) = config { - for (key, value) in config.into_iter() { - builder = builder.with_config(key.0, value); - } - } - if let Some(client_options) = client_options { - builder = builder.with_client_options(client_options.into()) - } - if let Some(retry_config) = retry_config { - builder = builder.with_retry(retry_config.into()) - } - Ok(Self(Arc::new(builder.build()?))) - } -} - -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct PyGoogleConfigKey(GoogleConfigKey); - -impl<'py> FromPyObject<'py> for PyGoogleConfigKey { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - let s = ob.extract::()?.to_lowercase(); - let key = GoogleConfigKey::from_str(&s).map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(Self(key)) - } -} diff --git a/pyo3-object_store/src/http.rs b/pyo3-object_store/src/http.rs deleted file mode 100644 index b3813a1..0000000 --- a/pyo3-object_store/src/http.rs +++ /dev/null @@ -1,45 +0,0 @@ -use std::sync::Arc; - -use object_store::http::{HttpBuilder, HttpStore}; -use pyo3::prelude::*; -use pyo3::types::PyType; - -use crate::error::PyObjectStoreResult; -use crate::retry::PyRetryConfig; -use crate::PyClientOptions; - -#[pyclass(name = "HTTPStore")] -pub struct PyHttpStore(Arc); - -impl AsRef> for PyHttpStore { - fn as_ref(&self) -> &Arc { - &self.0 - } -} - -impl PyHttpStore { - pub fn into_inner(self) -> Arc { - self.0 - } -} - -#[pymethods] -impl PyHttpStore { - #[classmethod] - #[pyo3(signature = (url, *, client_options=None, retry_config=None))] - fn from_url( - _cls: &Bound, - url: &str, - client_options: Option, - retry_config: Option, - ) -> PyObjectStoreResult { - let mut builder = HttpBuilder::new().with_url(url); - if let Some(client_options) = client_options { - builder = builder.with_client_options(client_options.into()) - } - if let Some(retry_config) = retry_config { - builder = builder.with_retry(retry_config.into()) - } - Ok(Self(Arc::new(builder.build()?))) - } -} diff --git a/pyo3-object_store/src/lib.rs b/pyo3-object_store/src/lib.rs deleted file mode 100644 index 9e9ddb6..0000000 --- a/pyo3-object_store/src/lib.rs +++ /dev/null @@ -1,24 +0,0 @@ -#![doc = include_str!("../README.md")] -// #![deny(missing_docs)] - -mod api; -mod aws; -mod azure; -mod client; -pub mod error; -mod gcp; -mod http; -mod local; -mod memory; -mod retry; -mod store; - -pub use api::register_store_module; -pub use aws::PyS3Store; -pub use azure::PyAzureStore; -pub use client::{PyClientConfigKey, PyClientOptions}; -pub use gcp::PyGCSStore; -pub use http::PyHttpStore; -pub use local::PyLocalStore; -pub use memory::PyMemoryStore; -pub use store::PyObjectStore; diff --git a/pyo3-object_store/src/local.rs b/pyo3-object_store/src/local.rs deleted file mode 100644 index b980636..0000000 --- a/pyo3-object_store/src/local.rs +++ /dev/null @@ -1,35 +0,0 @@ -use std::sync::Arc; - -use object_store::local::LocalFileSystem; -use pyo3::prelude::*; - -use crate::error::PyObjectStoreResult; - -#[pyclass(name = "LocalStore")] -pub struct PyLocalStore(Arc); - -impl AsRef> for PyLocalStore { - fn as_ref(&self) -> &Arc { - &self.0 - } -} - -impl PyLocalStore { - pub fn into_inner(self) -> Arc { - self.0 - } -} - -#[pymethods] -impl PyLocalStore { - #[new] - #[pyo3(signature = (prefix = None))] - fn py_new(prefix: Option) -> PyObjectStoreResult { - let fs = if let Some(prefix) = prefix { - LocalFileSystem::new_with_prefix(prefix)? - } else { - LocalFileSystem::new() - }; - Ok(Self(Arc::new(fs))) - } -} diff --git a/pyo3-object_store/src/memory.rs b/pyo3-object_store/src/memory.rs deleted file mode 100644 index fb45bca..0000000 --- a/pyo3-object_store/src/memory.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::sync::Arc; - -use object_store::memory::InMemory; -use pyo3::prelude::*; - -#[pyclass(name = "MemoryStore")] -pub struct PyMemoryStore(Arc); - -impl AsRef> for PyMemoryStore { - fn as_ref(&self) -> &Arc { - &self.0 - } -} - -impl PyMemoryStore { - pub fn into_inner(self) -> Arc { - self.0 - } -} - -#[pymethods] -impl PyMemoryStore { - #[new] - fn py_new() -> Self { - Self(Arc::new(InMemory::new())) - } -} diff --git a/pyo3-object_store/src/retry.rs b/pyo3-object_store/src/retry.rs deleted file mode 100644 index 71b2c80..0000000 --- a/pyo3-object_store/src/retry.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::time::Duration; - -use object_store::{BackoffConfig, RetryConfig}; -use pyo3::prelude::*; - -#[derive(Debug, FromPyObject)] -pub struct PyBackoffConfig { - init_backoff: Duration, - max_backoff: Duration, - base: f64, -} - -impl From for BackoffConfig { - fn from(value: PyBackoffConfig) -> Self { - BackoffConfig { - init_backoff: value.init_backoff, - max_backoff: value.max_backoff, - base: value.base, - } - } -} - -#[derive(Debug, FromPyObject)] -pub struct PyRetryConfig { - backoff: PyBackoffConfig, - max_retries: usize, - retry_timeout: Duration, -} - -impl From for RetryConfig { - fn from(value: PyRetryConfig) -> Self { - RetryConfig { - backoff: value.backoff.into(), - max_retries: value.max_retries, - retry_timeout: value.retry_timeout, - } - } -} diff --git a/pyo3-object_store/src/store.rs b/pyo3-object_store/src/store.rs deleted file mode 100644 index 3e5cb55..0000000 --- a/pyo3-object_store/src/store.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::sync::Arc; - -use object_store::ObjectStore; -use pyo3::exceptions::PyValueError; -use pyo3::intern; -use pyo3::prelude::*; -use pyo3::pybacked::PyBackedStr; - -use crate::http::PyHttpStore; -use crate::{PyAzureStore, PyGCSStore, PyLocalStore, PyMemoryStore, PyS3Store}; - -/// A wrapper around a Rust ObjectStore instance that allows any rust-native implementation of -/// ObjectStore. -// (In the future we'll have a separate AnyObjectStore that allows either an fsspec-based -// implementation or a rust-based implementation.) -pub struct PyObjectStore(Arc); - -impl<'py> FromPyObject<'py> for PyObjectStore { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - if let Ok(store) = ob.downcast::() { - Ok(Self(store.borrow().as_ref().clone())) - } else if let Ok(store) = ob.downcast::() { - Ok(Self(store.borrow().as_ref().clone())) - } else if let Ok(store) = ob.downcast::() { - Ok(Self(store.borrow().as_ref().clone())) - } else if let Ok(store) = ob.downcast::() { - Ok(Self(store.borrow().as_ref().clone())) - } else if let Ok(store) = ob.downcast::() { - Ok(Self(store.borrow().as_ref().clone())) - } else if let Ok(store) = ob.downcast::() { - Ok(Self(store.borrow().as_ref().clone())) - } else { - let py = ob.py(); - // Check for object-store instance from other library - let cls_name = ob - .getattr(intern!(py, "__class__"))? - .getattr(intern!(py, "__name__"))? - .extract::()?; - if [ - "AzureStore", - "GCSStore", - "HTTPStore", - "LocalStore", - "MemoryStore", - "S3Store", - ] - .contains(&cls_name.as_ref()) - { - return Err(PyValueError::new_err("You must use an object store instance exported from **the same library** as this function. They cannot be used across libraries.\nThis is because object store instances are compiled with a specific version of Rust and Python." )); - } - - // TODO: Check for fsspec - Err(PyValueError::new_err(format!( - "Expected an object store instance, got {}", - ob.repr()? - ))) - } - } -} - -impl AsRef> for PyObjectStore { - fn as_ref(&self) -> &Arc { - &self.0 - } -} - -impl PyObjectStore { - pub fn into_inner(self) -> Arc { - self.0 - } -}