diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index 69d448cd7ca..00000000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Coverage - -on: [pull_request, push] - -jobs: - coverage: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true # needed to test IPC, which are located in a submodule - - name: Install Rust - run: rustup toolchain install stable --component llvm-tools-preview - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - name: Setup parquet files - run: | - apt update && apt install python3-pip python3-venv -y -q - python3 -m venv venv - source venv/bin/activate - pip install pip --upgrade - pip install pyarrow==6 pyorc - python parquet_integration/write_parquet.py - python tests/it/io/orc/write.py - deactivate - - uses: Swatinem/rust-cache@v1 - - name: Generate code coverage - run: cargo llvm-cov --features full --lcov --output-path lcov.info - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - files: lcov.info - fail_ci_if_error: true diff --git a/.github/workflows/integration-ffi.yml b/.github/workflows/integration-ffi.yml index f09f0955880..ccb7c0cba67 100644 --- a/.github/workflows/integration-ffi.yml +++ b/.github/workflows/integration-ffi.yml @@ -10,8 +10,8 @@ jobs: - uses: actions/checkout@v2 - name: Setup Rust toolchain run: | - rustup toolchain install stable - rustup default stable + rustup toolchain install 1.72.1 + rustup default 1.72.1 rustup component add rustfmt clippy - name: Cache Cargo uses: actions/cache@v2 diff --git a/.github/workflows/integration-odbc.yml b/.github/workflows/integration-odbc.yml deleted file mode 100644 index 2ee3bf096f2..00000000000 --- a/.github/workflows/integration-odbc.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Integration ODBC - -on: [push, pull_request] - -env: - CARGO_TERM_COLOR: always - -jobs: - linux: - name: Test - runs-on: ubuntu-latest - - services: - sqlserver: - image: mcr.microsoft.com/mssql/server:2017-latest-ubuntu - ports: - - 1433:1433 - env: - ACCEPT_EULA: Y - SA_PASSWORD: My@Test@Password1 - - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install ODBC Drivers - run: | - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list > /etc/apt/sources.list.d/mssql-release.list - apt-get update - ACCEPT_EULA=Y apt-get install -y msodbcsql17 - ln -s /opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.*.so.* /opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.so - shell: sudo bash {0} - - name: Setup Rust toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt clippy - - uses: Swatinem/rust-cache@v1 - - name: Test - run: cd arrow-odbc-integration-testing && cargo test diff --git a/.github/workflows/integration-parquet.yml b/.github/workflows/integration-parquet.yml deleted file mode 100644 index 53a76d5b52f..00000000000 --- a/.github/workflows/integration-parquet.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Integration Parquet - -on: [push, pull_request] - -jobs: - docker: - name: Test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Setup Rust toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt clippy - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /home/runner/.cargo - key: cargo-parquet-cache- - - name: Cache Rust dependencies - uses: actions/cache@v2 - with: - path: /home/runner/target - key: ${{ runner.os }}-amd64-target-parquet-cache - - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - name: Build - run: | - export CARGO_HOME="/home/runner/.cargo" - export CARGO_TARGET_DIR="/home/runner/target" - - cd arrow-parquet-integration-testing - - cargo build - - name: Run - run: | - export CARGO_HOME="/home/runner/.cargo" - export CARGO_TARGET_DIR="/home/runner/target" - - cd arrow-parquet-integration-testing - - python -m venv venv - source venv/bin/activate - pip install --upgrade pip - pip install pyarrow==8 pyspark==3 - python main.py - # test against spark - python main_spark.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a9e91c2435..a2580f5dc18 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,10 +9,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - with: - submodules: true # needed to test IPC, which are located in a submodule - name: Install Rust - run: rustup update stable + run: rustup update 1.72.1 - name: Setup parquet files run: | apt update && apt install python3-pip python3-venv -y -q @@ -35,10 +33,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - with: - submodules: true # needed to test IPC, which are located in a submodule - name: Install Rust - run: rustup update stable + run: rustup update 1.72.1 - uses: Swatinem/rust-cache@v1 - name: Run shell: bash @@ -52,7 +48,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install Rust - run: rustup update stable + run: rustup update 1.72.1 - uses: Swatinem/rust-cache@v1 - name: Install clippy run: rustup component add clippy @@ -65,7 +61,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install Rust - run: rustup update stable + run: rustup update 1.72.1 - uses: Swatinem/rust-cache@v1 - name: Install rustfmt run: rustup component add rustfmt @@ -79,7 +75,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly-2022-12-05 + toolchain: nightly override: true - uses: Swatinem/rust-cache@v1 with: @@ -98,11 +94,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - with: - submodules: true # needed to test IPC, which are located in a submodule - uses: actions-rs/toolchain@v1 with: - toolchain: nightly-2022-12-05 + toolchain: nightly override: true - uses: Swatinem/rust-cache@v1 with: @@ -121,7 +115,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly-2022-12-05 + toolchain: nightly override: true - uses: Swatinem/rust-cache@v1 with: @@ -139,7 +133,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install Rust - run: rustup update stable + run: rustup update 1.72.1 - name: Setup all features run: cargo install cargo-all-features - uses: Swatinem/rust-cache@v1 @@ -158,15 +152,13 @@ jobs: - i686-unknown-linux-gnu - powerpc-unknown-linux-gnu - powerpc64-unknown-linux-gnu - - mips-unknown-linux-gnu + # - mips-unknown-linux-gnu - arm-linux-androideabi steps: - uses: actions/checkout@v2 - with: - submodules: true - uses: actions-rs/toolchain@v1 with: - toolchain: nightly-2022-12-05 + toolchain: nightly target: ${{ matrix.target }} override: true - uses: Swatinem/rust-cache@v1 @@ -174,7 +166,7 @@ jobs: with: use-cross: true command: check - args: --features=compute_merge_sort,io_ipc,io_csv,io_print,io_json,io_parquet --target ${{ matrix.target }} + args: --features=compute_merge_sort,io_ipc,io_csv,io_print,io_json --target ${{ matrix.target }} linux-simd-test: name: SIMD @@ -183,7 +175,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly-2022-12-05 + toolchain: nightly override: true - uses: Swatinem/rust-cache@v1 - name: Run diff --git a/.gitignore b/.gitignore index c10f9e51df0..a5a306f937a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ target +target_ra target-tarpaulin venv lcov.info -Cargo.lock example.arrow fixtures settings.json diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 61a00f837c1..00000000000 --- a/.gitmodules +++ /dev/null @@ -1,6 +0,0 @@ -[submodule "testing/arrow-testing"] - path = testing/arrow-testing - url = https://github.com/apache/arrow-testing -[submodule "testing/parquet-testing"] - path = testing/parquet-testing - url = https://github.com/apache/parquet-testing diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000000..cc9510604a2 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3317 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + +[[package]] +name = "ahash" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +dependencies = [ + "cfg-if 1.0.0", + "const-random", + "getrandom 0.2.8", + "once_cell", + "version_check", + "zerocopy 0.7.32", +] + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anyhow" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" + +[[package]] +name = "array-init-cursor" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" + +[[package]] +name = "arrow-array" +version = "49.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bda9acea48b25123c08340f3a8ac361aa0f74469bb36f5ee9acf923fce23e9d" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half 2.2.1", + "hashbrown 0.14.3", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "49.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a0fc21915b00fc6c2667b069c1b64bdd920982f426079bc4a7cab86822886c" +dependencies = [ + "bytes", + "half 2.2.1", + "num", +] + +[[package]] +name = "arrow-data" +version = "49.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907fafe280a3874474678c1858b9ca4cb7fd83fb8034ff5b6d6376205a08c634" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half 2.2.1", + "num", +] + +[[package]] +name = "arrow-format" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07884ea216994cdc32a2d5f8274a8bee979cfe90274b83f86f440866ee3132c7" +dependencies = [ + "planus", + "prost 0.11.8", + "prost-derive 0.11.8", + "serde", +] + +[[package]] +name = "arrow-schema" +version = "49.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e28a5e781bf1b0f981333684ad13f5901f4cd2f20589eab7cf1797da8fc167" + +[[package]] +name = "arrow2" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59c468daea140b747d781a1da9f7db5f0a8e6636d4af20cc539e43d05b0604fa" +dependencies = [ + "ahash", + "bytemuck", + "chrono", + "dyn-clone", + "either", + "ethnum", + "foreign_vec", + "getrandom 0.2.8", + "hash_hasher", + "num-traits", + "rustc_version", + "simdutf8", +] + +[[package]] +name = "async-stream" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "async-trait" +version = "0.1.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ea188f25f0255d8f92797797c97ebf5631fa88178beb1a46fdf5622c9a00e4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "avro-rs" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece550dd6710221de9bcdc1697424d8eee4fc4ca7e017479ea9d50c348465e37" +dependencies = [ + "byteorder", + "crc 1.8.1", + "digest", + "lazy_static", + "libflate", + "num-bigint 0.2.6", + "rand 0.7.3", + "serde", + "serde_json", + "snap 0.2.5", + "strum 0.18.0", + "strum_macros 0.18.0", + "thiserror", + "typed-builder", + "uuid", + "zerocopy 0.3.0", +] + +[[package]] +name = "avro-schema" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5281855b39aba9684d2f47bf96983fbfd8f1725f12fabb0513a8ab879647bbd" +dependencies = [ + "async-stream", + "crc 2.1.0", + "fallible-streaming-iterator", + "futures", + "libflate", + "serde", + "serde_json", + "snap 1.1.0", +] + +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bstr" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" +dependencies = [ + "memchr", + "once_cell", + "regex-automata 0.1.10", + "serde", +] + +[[package]] +name = "build_const" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + +[[package]] +name = "bytemuck" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aca418a974d83d40a0c1f0c5cba6ff4bc28d8df099109ca459a2118d40b6322" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "calloop" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf2eec61efe56aa1e813f5126959296933cf0700030e4314786c48779a66ab82" +dependencies = [ + "log", + "nix 0.22.3", +] + +[[package]] +name = "casey" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614586263949597dcc18675da12ef9b429135e13628d92eb8b8c6fa50ca5656b" +dependencies = [ + "syn 1.0.109", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-targets 0.48.0", +] + +[[package]] +name = "chrono-tz" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9cc2b23599e6d7479755f3594285efb3f74a1bdca7a7374948bc831e23a552" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9998fb9f7e9b2111641485bf8beb32f92945f97f92a3d061f744cfef335f751" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + +[[package]] +name = "ciborium" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" + +[[package]] +name = "ciborium-ll" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +dependencies = [ + "ciborium-io", + "half 1.8.2", +] + +[[package]] +name = "clap" +version = "3.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +dependencies = [ + "bitflags", + "clap_lex", + "indexmap", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "cocoa" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f425db7937052c684daec3bd6375c8abe2d146dca4b8b143d6db777c39138f3a" +dependencies = [ + "bitflags", + "block", + "cocoa-foundation", + "core-foundation 0.9.3", + "core-graphics 0.22.3", + "foreign-types", + "libc", + "objc", +] + +[[package]] +name = "cocoa-foundation" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "931d3837c286f56e3c58423ce4eba12d08db2374461a785c86f672b08b5650d6" +dependencies = [ + "bitflags", + "block", + "core-foundation 0.9.3", + "core-graphics-types", + "foreign-types", + "libc", + "objc", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "comfy-table" +version = "6.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7b787b0dc42e8111badfdbe4c3059158ccb2db8780352fa1b01e8ccf45cc4d" +dependencies = [ + "strum 0.24.1", + "strum_macros 0.24.3", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.8", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys 0.7.0", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys 0.8.3", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "core-graphics" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3889374e6ea6ab25dba90bb5d96202f61108058361f6dc72e8b03e6f8bbe923" +dependencies = [ + "bitflags", + "core-foundation 0.7.0", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2581bbab3b8ffc6fcbd550bf46c355135d16e9ff2a6ea032ad6b9bf1d7efe4fb" +dependencies = [ + "bitflags", + "core-foundation 0.9.3", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a68b68b3446082644c91ac778bf50cd4104bfb002b5a6a7c44cca5a2c70788b" +dependencies = [ + "bitflags", + "core-foundation 0.9.3", + "foreign-types", + "libc", +] + +[[package]] +name = "core-video-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ecad23610ad9757664d644e369246edde1803fcb43ed72876565098a5d3828" +dependencies = [ + "cfg-if 0.1.10", + "core-foundation-sys 0.7.0", + "core-graphics 0.19.2", + "libc", + "objc", +] + +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +dependencies = [ + "build_const", +] + +[[package]] +name = "crc" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "crossbeam-utils", + "memoffset 0.8.0", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "csv" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-async" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71933d3f2d0481d5111cb2817b15b6961961458ec58adf8008194e6c850046f4" +dependencies = [ + "bstr", + "cfg-if 1.0.0", + "csv-core", + "futures", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + +[[package]] +name = "cxx" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.48", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + +[[package]] +name = "dlib" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1b7517328c04c2aa68422fc60a41b92208182142ed04a25879c26c8f878794" +dependencies = [ + "libloading", +] + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + +[[package]] +name = "dyn-clone" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "env_logger" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "ethnum" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0198b9d0078e0f30dedc7acbb21c974e838fc8fae3ee170128658a98cb2c1c04" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "force-send-sync" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d9188d4e883c054455b2c7950be30c54b11d7400f0a8562b74a4acd79c495b6" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "foreign_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1b05cbd864bcaecbd3455d6d967862d446e4ebfc3c2e5e5b9841e53cba6673" + +[[package]] +name = "futures" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" + +[[package]] +name = "futures-executor" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" + +[[package]] +name = "futures-macro" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "futures-sink" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" + +[[package]] +name = "futures-task" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" + +[[package]] +name = "futures-util" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "half" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +dependencies = [ + "crunchy", + "num-traits", +] + +[[package]] +name = "hash_hasher" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "iana-time-zone" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +dependencies = [ + "android_system_properties", + "core-foundation-sys 0.8.3", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-deserializer" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f63b421e16eb4100beb677af56f0b4f3a4f08bab74ef2af079ce5bb92c2683f" +dependencies = [ + "indexmap", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" + +[[package]] +name = "libflate" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97822bf791bd4d5b403713886a5fbe8bf49520fe78e323b0dc480ca1a03e50b0" +dependencies = [ + "adler32", + "crc32fast", + "libflate_lz77", +] + +[[package]] +name = "libflate_lz77" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" +dependencies = [ + "rle-decode-fast", +] + +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if 1.0.0", + "winapi", +] + +[[package]] +name = "libm" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" + +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "lz4" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "lz4_flex" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8cbbb2831780bc3b9c15a41f5b49222ef756b6730a95f3decfdd15903eb5a3" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "memmap2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b6c2ebff6180198788f5db08d7ce3bc1d0b617176678831a7510825973e357" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "multiversion" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2c7b9d7fe61760ce5ea19532ead98541f6b4c495d87247aff9826445cf6872a" +dependencies = [ + "multiversion-macros", + "target-features", +] + +[[package]] +name = "multiversion-macros" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26a83d8500ed06d68877e9de1dde76c1dbb83885dcdbda4ef44ccbc3fbda2ac8" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "target-features", +] + +[[package]] +name = "ndk" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d868f654c72e75f8687572699cdabe755f03effbb62542768e995d5b8d699d" +dependencies = [ + "bitflags", + "jni-sys", + "ndk-sys", + "num_enum", + "thiserror", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-glue" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71bee8ea72d685477e28bd004cfe1bf99c754d688cd78cad139eae4089484d4" +dependencies = [ + "lazy_static", + "libc", + "log", + "ndk", + "ndk-context", + "ndk-macro", + "ndk-sys", +] + +[[package]] +name = "ndk-macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0df7ac00c4672f9d5aece54ee3347520b7e20f158656c7db2e6de01902eb7a6c" +dependencies = [ + "darling", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ndk-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1bcdd74c20ad5d95aacd60ef9ba40fdf77f767051040541df557b7a9b2a2121" + +[[package]] +name = "nix" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4916f159ed8e5de0082076562152a76b7a1f64a01fd9d1e0fea002c37624faf" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", + "memoffset 0.6.5", +] + +[[package]] +name = "nix" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "libc", + "memoffset 0.6.5", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +dependencies = [ + "num-bigint 0.4.3", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint 0.4.3", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", +] + +[[package]] +name = "odbc-api" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d9ca44aa6e1cd7c7a64dc9eeff0fb918993e118b6c235f5844f05321c6ac13" +dependencies = [ + "force-send-sync", + "log", + "odbc-sys", + "thiserror", + "widestring", + "winit", +] + +[[package]] +name = "odbc-sys" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "592c5c4ce58f47dde428c52b39904252191d25436b410cc232fae0813ff58f08" + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "orc-format" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a2dcf15f0a73c4fe33c622dec93adf95e05cb72d5b9a9af2bf51f3cc41f0b" +dependencies = [ + "fallible-streaming-iterator", + "flate2", + "prost 0.9.0", +] + +[[package]] +name = "os_str_bytes" +version = "6.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "parquet-format-safe" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" +dependencies = [ + "async-trait", + "futures", +] + +[[package]] +name = "parquet2" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aefc53bedbf9bbe0ff8912befafaafe30ced83851fb0aebe86696a9289ebb29e" +dependencies = [ + "async-stream", + "brotli", + "flate2", + "futures", + "lz4", + "lz4_flex", + "parquet-format-safe", + "seq-macro", + "snap 1.1.0", + "streaming-decompression", + "xxhash-rust", + "zstd 0.11.2+zstd.1.5.2", +] + +[[package]] +name = "parse-zoneinfo" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +dependencies = [ + "regex", +] + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "phf" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_shared" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "planus" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1691dd09e82f428ce8d6310bd6d5da2557c82ff17694d2a32cad7242aea89f" +dependencies = [ + "array-init-cursor", +] + +[[package]] +name = "plotters" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" + +[[package]] +name = "plotters-svg" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29f1b898011ce9595050a68e60f90bad083ff2987a695a42357134c8381fba70" +dependencies = [ + "bitflags", + "byteorder", + "lazy_static", + "num-traits", + "quick-error", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_xorshift", + "regex-syntax 0.6.28", + "unarray", +] + +[[package]] +name = "prost" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +dependencies = [ + "bytes", + "prost-derive 0.9.0", +] + +[[package]] +name = "prost" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537" +dependencies = [ + "bytes", + "prost-derive 0.11.8", +] + +[[package]] +name = "prost-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-derive" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea9b0f8cbe5e15a8a042d030bd96668db28ecb567ec37d691971ff5731d2b1b" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quickcheck" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +dependencies = [ + "env_logger", + "log", + "rand 0.8.5", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.8", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_regex" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a9fe2d7d9eeaf3279d1780452a5bbd26b31b27938787ef1c3e930d1e9cfbd" +dependencies = [ + "rand 0.8.5", + "regex-syntax 0.6.28", +] + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "raw-window-handle" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b800beb9b6e7d2df1fe337c9e3d04e3af22a124460fb4c30fcc22c9117cefb41" +dependencies = [ + "cty", +] + +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "re_arrow2" +version = "0.17.4" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-format", + "arrow-schema", + "async-stream", + "avro-rs", + "avro-schema", + "base64", + "bytemuck", + "chrono", + "chrono-tz", + "comfy-table", + "criterion", + "crossbeam-channel", + "csv", + "csv-async", + "csv-core", + "doc-comment", + "dyn-clone", + "either", + "ethnum", + "fallible-streaming-iterator", + "flate2", + "foreign_vec", + "futures", + "getrandom 0.2.8", + "hash_hasher", + "hashbrown 0.14.3", + "hex", + "indexmap", + "itertools", + "json-deserializer", + "lexical-core", + "lz4", + "memchr", + "multiversion", + "num-traits", + "odbc-api", + "orc-format", + "parquet2", + "proptest", + "rand 0.8.5", + "regex", + "regex-syntax 0.7.5", + "rustc_version", + "sample-arrow2", + "sample-std", + "sample-test", + "serde", + "serde_derive", + "serde_json", + "simdutf8", + "streaming-iterator", + "strength_reduce", + "tokio", + "tokio-util", + "zstd 0.12.3+zstd.1.5.2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-syntax" +version = "0.6.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "sample-arrow2" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b557c2d798c04f6ce0ae365117c6dcc48554bd71149dd4d78129badbbc717ac" +dependencies = [ + "arrow2", + "sample-std", +] + +[[package]] +name = "sample-std" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567a153dc3302ce838920fb095c025a6d0529fff0290d25deeec2136e41a57c8" +dependencies = [ + "casey", + "quickcheck", + "rand 0.8.5", + "rand_regex", + "regex", +] + +[[package]] +name = "sample-test" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713e500947ff19fc1ae2805afa33ef45f3bb2ec656c77d92252d24cf9e3091b2" +dependencies = [ + "quickcheck", + "sample-std", + "sample-test-macros", +] + +[[package]] +name = "sample-test-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1a2c832a259aae95b6ed1da3aa377111ffde38d4282fa734faa3fff356534e" +dependencies = [ + "proc-macro2", + "quote", + "sample-std", + "syn 1.0.109", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "scratch" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" + +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" + +[[package]] +name = "seq-macro" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" + +[[package]] +name = "serde" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "serde_json" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "smithay-client-toolkit" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a28f16a97fa0e8ce563b2774d1e732dd5d4025d2772c5dba0a41a0f90a29da3" +dependencies = [ + "bitflags", + "calloop", + "dlib", + "lazy_static", + "log", + "memmap2", + "nix 0.22.3", + "pkg-config", + "wayland-client", + "wayland-cursor", + "wayland-protocols", +] + +[[package]] +name = "snap" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95d697d63d44ad8b78b8d235bf85b34022a78af292c8918527c5f0cffdde7f43" +dependencies = [ + "byteorder", + "lazy_static", +] + +[[package]] +name = "snap" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "streaming-decompression" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3" +dependencies = [ + "fallible-streaming-iterator", +] + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57bd81eb48f4c437cadc685403cad539345bf703d78e63707418431cecd4522b" + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + +[[package]] +name = "strum_macros" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87c85aa3f8ea653bfd3ddf25f7ee357ee4d204731f6aa9ad04002306f6e2774c" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + +[[package]] +name = "target-features" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24840de800c1707d75c800893dbd727a5e1501ce921944e602f0698167491e36" + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tokio" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" +dependencies = [ + "autocfg", + "bytes", + "memchr", + "pin-project-lite", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tokio-util" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" + +[[package]] +name = "toml_edit" +version = "0.19.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc18466501acd8ac6a3f615dd29a3438f8ca6bb3b19537138b3106e575621274" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if 1.0.0", + "static_assertions", +] + +[[package]] +name = "typed-builder" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78cea224ddd4282dfc40d1edabbd0c020a12e946e3a48e2c2b8f6ff167ad29fe" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.8", + "serde", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "walkdir" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "wayland-client" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3b068c05a039c9f755f881dc50f01732214f5685e379829759088967c46715" +dependencies = [ + "bitflags", + "downcast-rs", + "libc", + "nix 0.24.3", + "scoped-tls", + "wayland-commons", + "wayland-scanner", + "wayland-sys", +] + +[[package]] +name = "wayland-commons" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8691f134d584a33a6606d9d717b95c4fa20065605f798a3f350d78dced02a902" +dependencies = [ + "nix 0.24.3", + "once_cell", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-cursor" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6865c6b66f13d6257bef1cd40cbfe8ef2f150fb8ebbdb1e8e873455931377661" +dependencies = [ + "nix 0.24.3", + "wayland-client", + "xcursor", +] + +[[package]] +name = "wayland-protocols" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b950621f9354b322ee817a23474e479b34be96c2e909c14f7bc0100e9a970bc6" +dependencies = [ + "bitflags", + "wayland-client", + "wayland-commons", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f4303d8fa22ab852f789e75a967f0a2cdc430a607751c0499bada3e451cbd53" +dependencies = [ + "proc-macro2", + "quote", + "xml-rs", +] + +[[package]] +name = "wayland-sys" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be12ce1a3c39ec7dba25594b97b42cb3195d54953ddb9d3d95a7c3902bc6e9d4" +dependencies = [ + "dlib", + "lazy_static", + "pkg-config", +] + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "widestring" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "winit" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b43cc931d58b99461188607efd7acb2a093e65fc621f54cad78517a6063e73a" +dependencies = [ + "bitflags", + "cocoa", + "core-foundation 0.9.3", + "core-graphics 0.22.3", + "core-video-sys", + "dispatch", + "instant", + "lazy_static", + "libc", + "log", + "mio", + "ndk", + "ndk-glue", + "ndk-sys", + "objc", + "parking_lot", + "percent-encoding", + "raw-window-handle", + "smithay-client-toolkit", + "wasm-bindgen", + "wayland-client", + "wayland-protocols", + "web-sys", + "winapi", + "x11-dl", +] + +[[package]] +name = "winnow" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d020b441f92996c80d94ae9166e8501e59c7bb56121189dc9eab3bd8216966" +dependencies = [ + "memchr", +] + +[[package]] +name = "x11-dl" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38735924fedd5314a6e548792904ed8c6de6636285cb9fec04d5b1db85c1516f" +dependencies = [ + "libc", + "once_cell", + "pkg-config", +] + +[[package]] +name = "xcursor" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "463705a63313cd4301184381c5e8042f0a7e9b4bb63653f216311d4ae74690b7" +dependencies = [ + "nom", +] + +[[package]] +name = "xml-rs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" + +[[package]] +name = "xxhash-rust" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70" + +[[package]] +name = "zerocopy" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6580539ad917b7c026220c4b3f2c08d52ce54d6ce0dc491e66002e35388fab46" +dependencies = [ + "byteorder", + "zerocopy-derive 0.2.0", +] + +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive 0.7.32", +] + +[[package]] +name = "zerocopy-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" +dependencies = [ + "proc-macro2", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.12.3+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +dependencies = [ + "zstd-safe 6.0.5+zstd.1.5.4", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "6.0.5+zstd.1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.7+zstd.1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" +dependencies = [ + "cc", + "libc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index a8e5933d2fe..a246f702222 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,17 +1,22 @@ [package] -name = "arrow2" +name = "re_arrow2" version = "0.17.4" license = "Apache-2.0" description = "Unofficial implementation of Apache Arrow spec in safe Rust" -homepage = "https://github.com/jorgecarleitao/arrow2" -repository = "https://github.com/jorgecarleitao/arrow2" -authors = ["Jorge C. Leitao ", "Apache Arrow "] +homepage = "https://github.com/rerun-io/re_arrow2" +repository = "https://github.com/rerun-io/re_arrow2" +authors = [ + "Rerun.io ", + "Jorge C. Leitao ", + "Apache Arrow ", +] keywords = ["arrow", "analytics"] edition = "2021" +rust-version = "1.72" exclude = ["testing/"] [lib] -name = "arrow2" +name = "re_arrow2" bench = false [dependencies] @@ -51,7 +56,9 @@ regex-syntax = { version = "0.7", optional = true } streaming-iterator = { version = "0.1", optional = true } fallible-streaming-iterator = { version = "0.1", optional = true } -json-deserializer = { version = "0.4.4", optional = true, features = ["preserve_order"] } +json-deserializer = { version = "0.4.4", optional = true, features = [ + "preserve_order", +] } indexmap = { version = "^1.6", optional = true } # used to print columns in a nice columnar format @@ -86,7 +93,9 @@ orc-format = { version = "0.3.0", optional = true } # Arrow integration tests support serde = { version = "^1.0", features = ["rc"], optional = true } serde_derive = { version = "^1.0", optional = true } -serde_json = { version = "^1.0", features = ["preserve_order"], optional = true } +serde_json = { version = "^1.0", features = [ + "preserve_order", +], optional = true } # for division/remainder optimization at runtime strength_reduce = { version = "0.2", optional = true } @@ -138,7 +147,7 @@ sample-test = "0.1" # ugly hack needed to match this library in sample_arrow2 [patch.crates-io] -arrow2 = { path = "." } +re_arrow2 = { path = "." } [package.metadata.docs.rs] features = ["full"] @@ -148,7 +157,7 @@ rustdoc-args = ["--cfg", "docsrs"] default = [] full = [ "arrow", - "io_odbc", + # "io_odbc", "io_csv", "io_csv_async", "io_json", @@ -159,8 +168,8 @@ full = [ "io_ipc_compression", "io_json_integration", "io_print", - "io_parquet_async", - "io_parquet_compression", + # "io_parquet_async", + # "io_parquet_compression", "io_avro", "io_orc", "io_avro_compression", @@ -180,7 +189,11 @@ io_csv_read_async = ["csv-async", "lexical-core", "futures"] io_csv_write = ["csv-core", "streaming-iterator", "lexical-core"] io_json = ["io_json_read", "io_json_write"] io_json_read = ["json-deserializer", "indexmap", "lexical-core"] -io_json_write = ["streaming-iterator", "fallible-streaming-iterator", "lexical-core"] +io_json_write = [ + "streaming-iterator", + "fallible-streaming-iterator", + "lexical-core", +] io_ipc = ["arrow-format"] io_ipc_write_async = ["io_ipc", "futures"] io_ipc_read_async = ["io_ipc", "futures", "async-stream"] @@ -188,35 +201,39 @@ io_ipc_compression = ["lz4", "zstd"] io_flight = ["io_ipc", "arrow-format/flight-data"] # base64 + io_ipc because arrow schemas are stored as base64-encoded ipc format. -io_parquet = ["parquet2", "io_ipc", "base64", "streaming-iterator", "fallible-streaming-iterator"] -io_parquet_async = ["futures", "io_parquet", "parquet2/async"] +io_parquet = [ + "parquet2", + "io_ipc", + "base64", + "streaming-iterator", + "fallible-streaming-iterator", +] +io_parquet_async = ["futures", "io_parquet", "parquet2?/async"] io_parquet_compression = [ "io_parquet_zstd", "io_parquet_gzip", "io_parquet_snappy", "io_parquet_lz4", - "io_parquet_brotli" + "io_parquet_brotli", ] # sample testing of generated arrow data io_parquet_sample_test = ["io_parquet_async"] # compression backends -io_parquet_zstd = ["parquet2/zstd"] -io_parquet_snappy = ["parquet2/snappy"] -io_parquet_gzip = ["parquet2/gzip"] -io_parquet_lz4_flex = ["parquet2/lz4_flex"] -io_parquet_lz4 = ["parquet2/lz4"] -io_parquet_brotli = ["parquet2/brotli"] +io_parquet_zstd = ["parquet2?/zstd"] +io_parquet_snappy = ["parquet2?/snappy"] +io_parquet_gzip = ["parquet2?/gzip"] +io_parquet_lz4_flex = ["parquet2?/lz4_flex"] +io_parquet_lz4 = ["parquet2?/lz4"] +io_parquet_brotli = ["parquet2?/brotli"] # parquet bloom filter functions -io_parquet_bloom_filter = ["parquet2/bloom_filter"] +io_parquet_bloom_filter = ["parquet2?/bloom_filter"] io_avro = ["avro-schema", "streaming-iterator"] -io_avro_compression = [ - "avro-schema/compression", -] +io_avro_compression = ["avro-schema/compression"] io_avro_async = ["avro-schema/async"] io_orc = ["orc-format"] @@ -277,7 +294,7 @@ compute = [ "compute_take", "compute_temporal", "compute_utf8", - "compute_window" + "compute_window", ] benchmarks = ["rand"] serde_types = ["serde", "serde_derive"] @@ -322,14 +339,6 @@ name = "comparison_kernels" harness = false -[[bench]] -name = "read_parquet" -harness = false - -[[bench]] -name = "write_parquet" -harness = false - [[bench]] name = "aggregate" harness = false @@ -401,4 +410,3 @@ harness = false [[bench]] name = "like_kernels" harness = false - diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 23ed2664e01..1afaff592b4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -4,13 +4,6 @@ This crate follows the standard for developing a Rust library via `cargo`. The CI is our "ground truth" over the state of the library. Check out the different parts of the CI to understand how to test the different parts of this library locally. -## Git clone with submodules -The crate comes with additional submodules to aid with testing, to ensure you have them if you plan on testing, using `--recurse-submodules` will clone the submodules alongside the repository. - -```bash -git clone --recurse-submodules https://github.com/jorgecarleitao/arrow2 -``` - ## Checks PRs will run the following checks: @@ -22,7 +15,7 @@ cargo clippy --all --features=full --tests -- -D warnings ## Testing -The simplest way to test the crate is to run +The simplest way to test the crate is to run ```bash cargo test --tests @@ -62,14 +55,6 @@ python tests/it/io/orc/write.py deactivate ``` -If you receive warnings about other files not found (IPC), ensure you have all submodules: -```bash -# If you didn't clone with `git clone --recurse-submodules https://github.com/jorgecarleitao/arrow2` -git submodule update --init --recursive - -# Update to the latest submodules -git submodule update --recursive --remote -``` during development of particular parts of the crate, it is usually faster to reduce the feature set - the tests are gated to only the relevant tests @@ -91,7 +76,7 @@ an issue, we favor having the PR on the changelog, since it includes a reference the author (credits). Summary: -* pull requests with both backward-incompatible changes and new +* pull requests with both backward-incompatible changes and new features/enhancements MUST close at least one issue (the one documenting the backward-incompatible change) * Every other pull request MAY close one issue diff --git a/README.md b/README.md index 0c4e428b57b..55cfe4d2c60 100644 --- a/README.md +++ b/README.md @@ -1,131 +1,6 @@ -# Arrow2: Transmute-free Arrow - -[![test](https://github.com/jorgecarleitao/arrow2/actions/workflows/test.yml/badge.svg)](https://github.com/jorgecarleitao/arrow2/actions/workflows/Build.yml) -[![codecov](https://codecov.io/gh/jorgecarleitao/arrow2/branch/main/graph/badge.svg?token=AgyTF60R3D)](https://codecov.io/gh/jorgecarleitao/arrow2) -[![](https://img.shields.io/crates/d/arrow2.svg)](https://crates.io/crates/arrow2) -[![](https://img.shields.io/crates/dv/arrow2.svg)](https://crates.io/crates/arrow2) -[![](https://docs.rs/arrow2/badge.svg)](https://docs.rs/arrow2/) - -A Rust crate to work with [Apache Arrow](https://arrow.apache.org/). -The most feature-complete implementation of the Arrow format after the C++ -implementation. - -Check out [the guide](https://jorgecarleitao.github.io/arrow2/main/guide) -for a general introduction on how to use this crate, and -[API docs](https://jorgecarleitao.github.io/arrow2/main/docs/arrow2) -for a detailed documentation of each of its APIs. - -## Features - -* Most feature-complete implementation of Apache Arrow after the reference implementation (C++) - * Decimal 256 unsupported (not a Rust native type) -* C data interface supported for all Arrow types (read and write) -* C stream interface supported for all Arrow types (read and write) -* Full interoperability with Rust's `Vec` -* MutableArray API to work with bitmaps and arrays in-place -* Full support for timestamps with timezones, including arithmetics that take - timezones into account -* Support to read from, and write to: - * CSV - * Apache Arrow IPC (all types) - * Apache Arrow Flight (all types) - * Apache Parquet (except deep nested types) - * Apache Avro (all types) - * NJSON - * ODBC (some types) -* Extensive suite of compute operations - * aggregations - * arithmetics - * cast - * comparison - * sort and merge-sort - * boolean (AND, OR, etc) and boolean kleene - * filter, take - * hash - * if-then-else - * nullif - * temporal (day, month, week day, hour, etc.) - * window - * ... and more ... -* Extensive set of cargo feature flags to reduce compilation time and binary size -* Fully-decoupled IO between CPU-bounded and IO-bounded tasks, allowing - this crate to both be used in `async` contexts without blocking and leverage parallelism -* Fastest known implementation of Avro and Parquet (e.g. faster than the official - C++ implementations) - -## Safety and Security - -This crate uses `unsafe` when strictly necessary: -* when the compiler can't prove certain invariants and -* FFI - -We have extensive tests over these, all of which run and pass under MIRI. -Most uses of `unsafe` fall into 3 categories: - -* The Arrow format has invariants over UTF-8 that can't be written in safe Rust -* `TrustedLen` and trait specialization are still nightly features -* FFI - -We actively monitor for vulnerabilities in Rust's advisory and either patch or mitigate -them (see e.g. `.cargo/audit.yaml` and `.github/workflows/security.yaml`). - -Reading from untrusted data currently _may_ `panic!` on the following formats: - -* Apache Parquet -* Apache Avro - -We are actively addressing this. - -## Integration tests - -Our tests include roundtrip against: -* Apache Arrow IPC (both little and big endian) generated by C++, Java, Go, C# and JS - implementations. -* Apache Parquet format (in its different configurations) generated by Arrow's C++ and - Spark's implementation -* Apache Avro generated by the official Rust Avro implementation - -Check [DEVELOPMENT.md](DEVELOPMENT.md) for our development practices. - -## Versioning - -We use the SemVer 2.0 used by Cargo and the remaining of the Rust ecosystem, -we also use the `0.x.y` versioning, since we are iterating over the API. - -## Design - -This repo and crate's primary goal is to offer a safe Rust implementation of the Arrow specification. -As such, it - -* MUST NOT implement any logical type other than the ones defined on the arrow specification, [schema.fbs](https://github.com/apache/arrow/blob/master/format/Schema.fbs). -* MUST lay out memory according to the [arrow specification](https://arrow.apache.org/docs/format/Columnar.html) -* MUST support reading from and writing to the [C data interface](https://arrow.apache.org/docs/format/CDataInterface.html) at zero-copy. -* MUST support reading from, and writing to, the [IPC specification](https://arrow.apache.org/docs/python/ipc.html), which it MUST verify against golden files available [here](https://github.com/apache/arrow-testing). - -Design documents about each of the parts of this repo are available on their respective READMEs. - -## FAQ - -### Any plans to merge with the Apache Arrow project? - -Maybe. The primary reason to have this repo and crate is to be able to prototype -and mature using a fundamentally different design based on a transmute-free -implementation. This requires breaking backward compatibility and loss of -features that is impossible to achieve on the Arrow repo. - -Furthermore, the arrow project currently has a release mechanism that is -unsuitable for this type of work: - -* A release of the Apache consists of a release of all implementations of the - arrow format at once, with the same version. It is currently at `5.0.0`. - -This implies that the crate version is independent of the changelog or its API stability, -which violates SemVer. This procedure makes the crate incompatible with -Rust's (and many others') ecosystem that heavily relies on SemVer to constraint -software versions. - -Secondly, this implies the arrow crate is versioned as `>0.x`. This places -expectations about API stability that are incompatible with this effort. +# `re_arrow2` - a fork of +The eco-system is moving to , so we suggest you use that instead. +We are planning on switching to `arrow` in the future as well. ## License @@ -135,7 +10,3 @@ Licensed under either of * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) at your option. - -### Contribution - -Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. diff --git a/arrow-odbc-integration-testing/Cargo.toml b/arrow-odbc-integration-testing/Cargo.toml index add16ba5d2d..5fb2f8ec10b 100644 --- a/arrow-odbc-integration-testing/Cargo.toml +++ b/arrow-odbc-integration-testing/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Jorge C. Leitao "] edition = "2021" [dependencies] -arrow2 = { path = "../", default-features = false, features = ["io_odbc"] } +re_arrow2 = { path = "../", default-features = false, features = ["io_odbc"] } lazy_static = "1.4.0" # Function name macro is used to ensure unique table names in test stdext = "0.3.1" diff --git a/arrow-odbc-integration-testing/src/lib.rs b/arrow-odbc-integration-testing/src/lib.rs index bfc24d65dc3..981523fbd71 100644 --- a/arrow-odbc-integration-testing/src/lib.rs +++ b/arrow-odbc-integration-testing/src/lib.rs @@ -3,8 +3,8 @@ mod read; mod write; -use arrow2::io::odbc::api::{Connection, Environment, Error as OdbcError}; use lazy_static::lazy_static; +use re_arrow2::io::odbc::api::{Connection, Environment, Error as OdbcError}; lazy_static! { /// This is an example for using doc comment attributes diff --git a/arrow-odbc-integration-testing/src/read.rs b/arrow-odbc-integration-testing/src/read.rs index a41b1388738..faa9ecb84bb 100644 --- a/arrow-odbc-integration-testing/src/read.rs +++ b/arrow-odbc-integration-testing/src/read.rs @@ -1,11 +1,11 @@ use stdext::function_name; -use arrow2::array::{Array, BinaryArray, BooleanArray, Int32Array, Int64Array, Utf8Array}; -use arrow2::chunk::Chunk; -use arrow2::datatypes::{DataType, Field, TimeUnit}; -use arrow2::error::Result; -use arrow2::io::odbc::api::{Connection, Cursor}; -use arrow2::io::odbc::read::{buffer_from_metadata, deserialize, infer_schema}; +use re_arrow2::array::{Array, BinaryArray, BooleanArray, Int32Array, Int64Array, Utf8Array}; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::{DataType, Field, TimeUnit}; +use re_arrow2::error::Result; +use re_arrow2::io::odbc::api::{Connection, Cursor}; +use re_arrow2::io::odbc::read::{buffer_from_metadata, deserialize, infer_schema}; use super::{setup_empty_table, ENV, MSSQL}; diff --git a/arrow-odbc-integration-testing/src/write.rs b/arrow-odbc-integration-testing/src/write.rs index bcf12761abd..9ca07396080 100644 --- a/arrow-odbc-integration-testing/src/write.rs +++ b/arrow-odbc-integration-testing/src/write.rs @@ -1,10 +1,10 @@ use stdext::function_name; -use arrow2::array::{Array, BinaryArray, BooleanArray, Int32Array, Utf8Array}; -use arrow2::chunk::Chunk; -use arrow2::datatypes::{DataType, Field}; -use arrow2::error::Result; -use arrow2::io::odbc::write::{buffer_from_description, infer_descriptions, serialize}; +use re_arrow2::array::{Array, BinaryArray, BooleanArray, Int32Array, Utf8Array}; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::{DataType, Field}; +use re_arrow2::error::Result; +use re_arrow2::io::odbc::write::{buffer_from_description, infer_descriptions, serialize}; use super::read::read; use super::{setup_empty_table, ENV, MSSQL}; diff --git a/arrow-parquet-integration-testing/.gitignore b/arrow-parquet-integration-testing/.gitignore deleted file mode 100644 index 47693803088..00000000000 --- a/arrow-parquet-integration-testing/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -target -target-tarpaulin -venv -lcov.info -Cargo.lock -fixtures -settings.json -*.parquet diff --git a/arrow-parquet-integration-testing/Cargo.toml b/arrow-parquet-integration-testing/Cargo.toml deleted file mode 100644 index 570bd8fa6f9..00000000000 --- a/arrow-parquet-integration-testing/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "arrow-parquet-integration-testing" -version = "0.1.0" -authors = ["Jorge C. Leitao "] -edition = "2021" - -[dependencies] -clap = { version = "^3", features = ["derive"] } -arrow2 = { path = "../", default-features = false, features = ["io_parquet", "io_json_integration", "io_parquet_compression"] } -flate2 = "^1" -serde = { version = "^1.0", features = ["rc"] } -serde_derive = { version = "^1.0" } -serde_json = { version = "^1.0", features = ["preserve_order"] } diff --git a/arrow-parquet-integration-testing/main.py b/arrow-parquet-integration-testing/main.py deleted file mode 100644 index a880af617d8..00000000000 --- a/arrow-parquet-integration-testing/main.py +++ /dev/null @@ -1,108 +0,0 @@ -import subprocess -import os - -import pyarrow.ipc -import pyarrow.parquet as pq - - -def get_file_path(file: str): - return f"../testing/arrow-testing/data/arrow-ipc-stream/integration/1.0.0-littleendian/{file}.arrow_file" - - -def _prepare( - file: str, - version: str, - compression: str, - encoding_utf8: str, - encoding_int: str, - projection=None, -): - write = f"{file}.parquet" - - args = [ - "cargo", - "run", - "--", - "--json", - file, - "--output", - write, - "--version", - version, - "--encoding-utf8", - encoding_utf8, - "--encoding-int", - encoding_int, - "--compression", - compression, - ] - - if projection: - projection = list(map(str, projection)) - args += ["--projection", ",".join(projection)] - - subprocess.call(args) - return write - - -def _expected(file: str) -> pyarrow.Table: - return pyarrow.ipc.RecordBatchFileReader(get_file_path(file)).read_all() - - -# types without a native parquet logical representation -# There is currently no specification on how to represent these in parquet, -# and thus we ignore them in comparisons -non_native_types = [ - pyarrow.date64(), - pyarrow.time32("s"), - pyarrow.timestamp("s"), - # the issue here is the second, not the tz - pyarrow.timestamp("s", tz="UTC"), -] - - -def variations(): - for version in ["1", "2"]: - for file in [ - "generated_primitive", - "generated_primitive_no_batches", - "generated_primitive_zerolength", - "generated_null", - "generated_null_trivial", - "generated_primitive_large_offsets", - "generated_datetime", - "generated_decimal", - "generated_interval", - # see https://issues.apache.org/jira/browse/ARROW-13486 and - # https://issues.apache.org/jira/browse/ARROW-13487 - # "generated_dictionary", - # requires writing Struct - # "generated_duplicate_fieldnames", - # requires writing un-nested List - # "generated_custom_metadata", - ]: - # pyarrow does not support decoding "delta"-encoded values. - for encoding_int in ["plain", "delta"]: - if encoding_int == "delta" and file in {"generated_primitive", "generated_null"}: - # see https://issues.apache.org/jira/browse/ARROW-17465 - continue - - for compression in ["uncompressed", "zstd", "snappy"]: - yield (version, file, compression, "plain", encoding_int) - - -if __name__ == "__main__": - for (version, file, compression, encoding_utf8, encoding_int) in variations(): - expected = _expected(file) - path = _prepare(file, version, compression, encoding_utf8, encoding_int) - - table = pq.read_table(path) - os.remove(path) - - for c1, c2 in zip(expected, table): - if c1.type in non_native_types: - continue - if str(c1.type) in ["month_interval", "day_time_interval"]: - # pyarrow does not support interval types from parquet - continue - assert c1 == c2, (c1, c2) diff --git a/arrow-parquet-integration-testing/main_spark.py b/arrow-parquet-integration-testing/main_spark.py deleted file mode 100644 index e29655fb46b..00000000000 --- a/arrow-parquet-integration-testing/main_spark.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -Verifies that spark can correctly read a delta-encoded utf8 column written by arrow2. -""" -import os -import pyspark.sql - -from main import _prepare, _expected - - -def test( - file: str, - version: str, - column: str, - compression: str, - encoding: str, -): - """ - Tests that pyspark can read a parquet file written by arrow2. - - In arrow2: read IPC, write parquet - In pyarrow: read (same) IPC to Python - In pyspark: read (written) parquet to Python - assert that they are equal - """ - # read IPC to Python - expected = _expected(file) - column_index = next(i for i, c in enumerate(expected.column_names) if c == column) - expected = expected[column].combine_chunks().tolist() - - # write parquet - path = _prepare(file, version, compression, encoding, encoding, [column_index]) - - # read parquet to Python - spark = pyspark.sql.SparkSession.builder.config( - # see https://stackoverflow.com/a/62024670/931303 - "spark.sql.parquet.enableVectorizedReader", - "false", - ).getOrCreate() - - result = spark.read.parquet(path).select(column).collect() - result = [r[column] for r in result] - os.remove(path) - - # assert equality - assert expected == result - - -test("generated_null", "2", "f1", "uncompressed", "delta") - -test("generated_primitive", "2", "utf8_nullable", "uncompressed", "delta") -test("generated_primitive", "2", "utf8_nullable", "snappy", "delta") -test("generated_primitive", "2", "int32_nullable", "uncompressed", "delta") -test("generated_primitive", "2", "int32_nullable", "snappy", "delta") -test("generated_primitive", "2", "int16_nullable", "uncompressed", "delta") -test("generated_primitive", "2", "int16_nullable", "snappy", "delta") - -test("generated_dictionary", "1", "dict0", "uncompressed", "plain") -test("generated_dictionary", "1", "dict0", "snappy", "plain") -test("generated_dictionary", "2", "dict0", "uncompressed", "plain") -test("generated_dictionary", "2", "dict0", "snappy", "plain") - -test("generated_dictionary", "1", "dict1", "uncompressed", "plain") -test("generated_dictionary", "1", "dict1", "snappy", "plain") -test("generated_dictionary", "2", "dict1", "uncompressed", "plain") -test("generated_dictionary", "2", "dict1", "snappy", "plain") - -test("generated_dictionary", "1", "dict2", "uncompressed", "plain") -test("generated_dictionary", "1", "dict2", "snappy", "plain") -test("generated_dictionary", "2", "dict2", "uncompressed", "plain") -test("generated_dictionary", "2", "dict2", "snappy", "plain") diff --git a/arrow-parquet-integration-testing/src/main.rs b/arrow-parquet-integration-testing/src/main.rs index f9c95dc26b7..02f9feb4c8e 100644 --- a/arrow-parquet-integration-testing/src/main.rs +++ b/arrow-parquet-integration-testing/src/main.rs @@ -1,9 +1,11 @@ use std::fs::File; use std::io::Read; -use arrow2::array::Array; -use arrow2::io::ipc::IpcField; -use arrow2::{ +use clap::Parser; +use flate2::read::GzDecoder; +use re_arrow2::array::Array; +use re_arrow2::io::ipc::IpcField; +use re_arrow2::{ chunk::Chunk, datatypes::{DataType, Schema}, error::Result, @@ -17,8 +19,6 @@ use arrow2::{ }, AHashMap, }; -use clap::Parser; -use flate2::read::GzDecoder; /// Read gzipped JSON file pub fn read_gzip_json( diff --git a/arrow-pyarrow-integration-testing/Cargo.toml b/arrow-pyarrow-integration-testing/Cargo.toml index 009dc24d7e8..e686bd934e4 100644 --- a/arrow-pyarrow-integration-testing/Cargo.toml +++ b/arrow-pyarrow-integration-testing/Cargo.toml @@ -18,7 +18,10 @@ [package] name = "arrow-pyarrow-integration-testing" version = "0.0.0" -authors = ["Jorge C. Leitao ", "Apache Arrow "] +authors = [ + "Jorge C. Leitao ", + "Apache Arrow ", +] license = "Apache-2.0" edition = "2021" @@ -27,7 +30,7 @@ name = "arrow_pyarrow_integration_testing" crate-type = ["cdylib"] [dependencies] -arrow2 = { path = "../", default-features = false } +re_arrow2 = { path = "../", default-features = false } pyo3 = { version = "0.14", features = ["extension-module"] } [package.metadata.maturin] diff --git a/arrow-pyarrow-integration-testing/src/c_stream.rs b/arrow-pyarrow-integration-testing/src/c_stream.rs index 1c512040880..fad93404851 100644 --- a/arrow-pyarrow-integration-testing/src/c_stream.rs +++ b/arrow-pyarrow-integration-testing/src/c_stream.rs @@ -3,9 +3,9 @@ use pyo3::ffi::Py_uintptr_t; use pyo3::prelude::*; -use arrow2::array::{Int32Array, StructArray}; -use arrow2::datatypes::DataType; -use arrow2::ffi; +use re_arrow2::array::{Int32Array, StructArray}; +use re_arrow2::datatypes::DataType; +use re_arrow2::ffi; use super::*; diff --git a/arrow-pyarrow-integration-testing/src/lib.rs b/arrow-pyarrow-integration-testing/src/lib.rs index ddfe8005999..452263a42c8 100644 --- a/arrow-pyarrow-integration-testing/src/lib.rs +++ b/arrow-pyarrow-integration-testing/src/lib.rs @@ -10,7 +10,7 @@ use pyo3::ffi::Py_uintptr_t; use pyo3::prelude::*; use pyo3::wrap_pyfunction; -use arrow2::{array::Array, datatypes::Field, error::Error, ffi}; +use re_arrow2::{array::Array, datatypes::Field, error::Error, ffi}; /// an error that bridges Error with a Python error #[derive(Debug)] @@ -80,8 +80,8 @@ fn to_py_array(array: Box, py: Python) -> PyResult { ))); let array = Box::new(ffi::export_array_to_c(array)); - let schema_ptr: *const arrow2::ffi::ArrowSchema = &*schema; - let array_ptr: *const arrow2::ffi::ArrowArray = &*array; + let schema_ptr: *const re_arrow2::ffi::ArrowSchema = &*schema; + let array_ptr: *const re_arrow2::ffi::ArrowArray = &*array; let pa = py.import("pyarrow")?; @@ -110,7 +110,7 @@ fn to_rust_field(ob: PyObject, py: Python) -> PyResult { fn to_py_field(field: &Field, py: Python) -> PyResult { let schema = Box::new(ffi::export_field_to_c(field)); - let schema_ptr: *const arrow2::ffi::ArrowSchema = &*schema; + let schema_ptr: *const re_arrow2::ffi::ArrowSchema = &*schema; let pa = py.import("pyarrow")?; diff --git a/benches/aggregate.rs b/benches/aggregate.rs index ac5002d31e9..d29ff5cb17c 100644 --- a/benches/aggregate.rs +++ b/benches/aggregate.rs @@ -1,8 +1,8 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::array::*; -use arrow2::compute::aggregate::*; -use arrow2::util::bench_util::*; +use re_arrow2::array::*; +use re_arrow2::compute::aggregate::*; +use re_arrow2::util::bench_util::*; fn bench_sum(arr_a: &dyn Array) { sum(criterion::black_box(arr_a)).unwrap(); diff --git a/benches/arithmetic_kernels.rs b/benches/arithmetic_kernels.rs index 950a08b10fb..4c755c46014 100644 --- a/benches/arithmetic_kernels.rs +++ b/benches/arithmetic_kernels.rs @@ -1,10 +1,10 @@ -use arrow2::compute::arithmetics::basic::NativeArithmetics; use criterion::{criterion_group, criterion_main, Criterion}; +use re_arrow2::compute::arithmetics::basic::NativeArithmetics; -use arrow2::array::*; -use arrow2::util::bench_util::*; -use arrow2::{compute::arithmetics::basic::add, compute::arithmetics::basic::div_scalar}; use num_traits::NumCast; +use re_arrow2::array::*; +use re_arrow2::util::bench_util::*; +use re_arrow2::{compute::arithmetics::basic::add, compute::arithmetics::basic::div_scalar}; use std::ops::{Add, Div}; fn bench_div_scalar(lhs: &PrimitiveArray, rhs: &T) diff --git a/benches/assign_ops.rs b/benches/assign_ops.rs index 5c190e43c5d..ad3061b28a3 100644 --- a/benches/assign_ops.rs +++ b/benches/assign_ops.rs @@ -1,7 +1,7 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::compute::arity_assign::{binary, unary}; -use arrow2::{ +use re_arrow2::compute::arity_assign::{binary, unary}; +use re_arrow2::{ compute::arithmetics::basic::{mul, mul_scalar}, util::bench_util::*, }; diff --git a/benches/avro_read.rs b/benches/avro_read.rs index 37088492df8..c09e5ed84c7 100644 --- a/benches/avro_read.rs +++ b/benches/avro_read.rs @@ -3,11 +3,11 @@ use std::io::Cursor; use avro_rs::types::Record; use criterion::*; -use arrow2::error::Result; -use arrow2::io::avro::avro_schema::read::read_metadata; -use arrow2::io::avro::read; use avro_rs::*; use avro_rs::{Codec, Schema as AvroSchema}; +use re_arrow2::error::Result; +use re_arrow2::io::avro::avro_schema::read::read_metadata; +use re_arrow2::io::avro::read; fn schema() -> AvroSchema { let raw_schema = r#" diff --git a/benches/bitmap.rs b/benches/bitmap.rs index 5fde7fca77f..6adb55427a2 100644 --- a/benches/bitmap.rs +++ b/benches/bitmap.rs @@ -2,7 +2,7 @@ use std::iter::FromIterator; use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::bitmap::*; +use re_arrow2::bitmap::*; fn add_benchmark(c: &mut Criterion) { (10..=20).step_by(2).for_each(|log2_size| { diff --git a/benches/bitmap_assign_ops.rs b/benches/bitmap_assign_ops.rs index 926be0a60e9..a926f4da266 100644 --- a/benches/bitmap_assign_ops.rs +++ b/benches/bitmap_assign_ops.rs @@ -1,7 +1,7 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::bitmap::{binary_assign, unary_assign}; -use arrow2::bitmap::{Bitmap, MutableBitmap}; +use re_arrow2::bitmap::{binary_assign, unary_assign}; +use re_arrow2::bitmap::{Bitmap, MutableBitmap}; fn add_benchmark(c: &mut Criterion) { (10..=20).step_by(2).for_each(|log2_size| { diff --git a/benches/bitmap_ops.rs b/benches/bitmap_ops.rs index 85a6db602b9..985d1f5b5c4 100644 --- a/benches/bitmap_ops.rs +++ b/benches/bitmap_ops.rs @@ -1,6 +1,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::bitmap::Bitmap; +use re_arrow2::bitmap::Bitmap; fn bench_arrow2(lhs: &Bitmap, rhs: &Bitmap) { let r = lhs | rhs; diff --git a/benches/bitwise.rs b/benches/bitwise.rs index 704e32aac77..d45aa8661a9 100644 --- a/benches/bitwise.rs +++ b/benches/bitwise.rs @@ -3,7 +3,7 @@ use std::ops::{BitAnd, BitOr, BitXor, Not}; use criterion::{criterion_group, criterion_main, Criterion}; use num_traits::NumCast; -use arrow2::{ +use re_arrow2::{ array::PrimitiveArray, compute::bitwise::*, types::NativeType, util::bench_util::create_primitive_array_with_seed, }; diff --git a/benches/cast_kernels.rs b/benches/cast_kernels.rs index 2367ef03d03..036f7e7a2ac 100644 --- a/benches/cast_kernels.rs +++ b/benches/cast_kernels.rs @@ -19,10 +19,10 @@ use criterion::{criterion_group, criterion_main, Criterion}; use rand::distributions::Uniform; use rand::Rng; -use arrow2::array::*; -use arrow2::compute::cast; -use arrow2::datatypes::*; -use arrow2::util::bench_util::*; +use re_arrow2::array::*; +use re_arrow2::compute::cast; +use re_arrow2::datatypes::*; +use re_arrow2::util::bench_util::*; fn build_utf8_date_array(size: usize, with_nulls: bool) -> Utf8Array { use chrono::NaiveDate; diff --git a/benches/comparison_kernels.rs b/benches/comparison_kernels.rs index cbea8f8f74d..c609cfa53fd 100644 --- a/benches/comparison_kernels.rs +++ b/benches/comparison_kernels.rs @@ -1,8 +1,8 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::compute::comparison::{eq, eq_scalar}; -use arrow2::scalar::*; -use arrow2::util::bench_util::*; +use re_arrow2::compute::comparison::{eq, eq_scalar}; +use re_arrow2::scalar::*; +use re_arrow2::util::bench_util::*; fn add_benchmark(c: &mut Criterion) { (10..=20).step_by(2).for_each(|log2_size| { diff --git a/benches/concatenate.rs b/benches/concatenate.rs index b2e3713d447..6892173472f 100644 --- a/benches/concatenate.rs +++ b/benches/concatenate.rs @@ -1,6 +1,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::{ +use re_arrow2::{ compute::concatenate::concatenate, util::bench_util::{create_boolean_array, create_primitive_array}, }; diff --git a/benches/count_zeros.rs b/benches/count_zeros.rs index 38d1570d213..7d7ad6b5909 100644 --- a/benches/count_zeros.rs +++ b/benches/count_zeros.rs @@ -1,6 +1,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::bitmap::utils::count_zeros; +use re_arrow2::bitmap::utils::count_zeros; fn add_benchmark(c: &mut Criterion) { (10..=20).step_by(2).for_each(|log2_size| { diff --git a/benches/filter_kernels.rs b/benches/filter_kernels.rs index 97e8dc320cd..5f7386b83f1 100644 --- a/benches/filter_kernels.rs +++ b/benches/filter_kernels.rs @@ -16,11 +16,13 @@ // under the License. use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::compute::filter::{build_filter, filter, filter_chunk, Filter}; -use arrow2::datatypes::DataType; -use arrow2::util::bench_util::{create_boolean_array, create_primitive_array, create_string_array}; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::compute::filter::{build_filter, filter, filter_chunk, Filter}; +use re_arrow2::datatypes::DataType; +use re_arrow2::util::bench_util::{ + create_boolean_array, create_primitive_array, create_string_array, +}; fn bench_filter(data_array: &dyn Array, filter_array: &BooleanArray) { criterion::black_box(filter(data_array, filter_array).unwrap()); diff --git a/benches/growable.rs b/benches/growable.rs index ca7ac8a9045..ff7a7232c67 100644 --- a/benches/growable.rs +++ b/benches/growable.rs @@ -1,6 +1,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::{ +use re_arrow2::{ array::growable::{Growable, GrowablePrimitive}, util::bench_util::create_primitive_array, }; diff --git a/benches/hash_kernel.rs b/benches/hash_kernel.rs index 81119b3aaf9..67005b977ea 100644 --- a/benches/hash_kernel.rs +++ b/benches/hash_kernel.rs @@ -1,7 +1,7 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::compute::hash::hash; -use arrow2::util::bench_util::*; +use re_arrow2::compute::hash::hash; +use re_arrow2::util::bench_util::*; fn add_benchmark(c: &mut Criterion) { let log2_size = 10; diff --git a/benches/iter_list.rs b/benches/iter_list.rs index 43b7698f3e5..1cc9d47f0c4 100644 --- a/benches/iter_list.rs +++ b/benches/iter_list.rs @@ -2,7 +2,7 @@ use std::iter::FromIterator; use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::{ +use re_arrow2::{ array::{ListArray, PrimitiveArray}, bitmap::Bitmap, buffer::Buffer, diff --git a/benches/iter_utf8.rs b/benches/iter_utf8.rs index d80ba123890..402e27077f1 100644 --- a/benches/iter_utf8.rs +++ b/benches/iter_utf8.rs @@ -1,6 +1,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::array::Utf8Array; +use re_arrow2::array::Utf8Array; fn add_benchmark(c: &mut Criterion) { (10..=20).step_by(2).for_each(|log2_size| { diff --git a/benches/length_kernel.rs b/benches/length_kernel.rs index a5fc2ab08d4..3bccaec7088 100644 --- a/benches/length_kernel.rs +++ b/benches/length_kernel.rs @@ -17,8 +17,8 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::array::*; -use arrow2::compute::length::length; +use re_arrow2::array::*; +use re_arrow2::compute::length::length; fn bench_length(array: &Utf8Array) { criterion::black_box(length(array).unwrap()); diff --git a/benches/like_kernels.rs b/benches/like_kernels.rs index 24f700244cc..c37cfe12235 100644 --- a/benches/like_kernels.rs +++ b/benches/like_kernels.rs @@ -1,8 +1,8 @@ -use arrow2::util::bench_util::create_string_array; use criterion::{criterion_group, criterion_main, Criterion}; +use re_arrow2::util::bench_util::create_string_array; -use arrow2::array::*; -use arrow2::compute::like::like_utf8_scalar; +use re_arrow2::array::*; +use re_arrow2::compute::like::like_utf8_scalar; fn bench_like(array: &Utf8Array, pattern: &str) { criterion::black_box(like_utf8_scalar(array, pattern).unwrap()); diff --git a/benches/read_json.rs b/benches/read_json.rs index 0da79b9437a..4a9dbaedd85 100644 --- a/benches/read_json.rs +++ b/benches/read_json.rs @@ -1,9 +1,9 @@ -use arrow2::array::Array; -use arrow2::datatypes::DataType; use criterion::{criterion_group, criterion_main, Criterion}; +use re_arrow2::array::Array; +use re_arrow2::datatypes::DataType; -use arrow2::io::json::{read, write}; -use arrow2::util::bench_util::*; +use re_arrow2::io::json::{read, write}; +use re_arrow2::util::bench_util::*; fn prep(array: impl Array + 'static) -> (Vec, DataType) { let mut data = vec![]; diff --git a/benches/read_parquet.rs b/benches/read_parquet.rs index 5aa16963601..1ab57bb2a67 100644 --- a/benches/read_parquet.rs +++ b/benches/read_parquet.rs @@ -3,8 +3,8 @@ use std::{fs, io::Cursor, path::PathBuf}; use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::error::Result; -use arrow2::io::parquet::read; +use re_arrow2::error::Result; +use re_arrow2::io::parquet::read; fn to_buffer( size: usize, diff --git a/benches/slices_iterator.rs b/benches/slices_iterator.rs index cea662d16a9..353ce8ae81e 100644 --- a/benches/slices_iterator.rs +++ b/benches/slices_iterator.rs @@ -1,6 +1,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::bitmap::{utils::SlicesIterator, Bitmap}; +use re_arrow2::bitmap::{utils::SlicesIterator, Bitmap}; fn bench_slices(lhs: &Bitmap) { let set_count = lhs.len() - lhs.unset_bits(); diff --git a/benches/sort_kernel.rs b/benches/sort_kernel.rs index 562d7d7b444..61ff3fdd029 100644 --- a/benches/sort_kernel.rs +++ b/benches/sort_kernel.rs @@ -17,9 +17,9 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::array::*; -use arrow2::compute::sort::{lexsort, sort, sort_to_indices, SortColumn, SortOptions}; -use arrow2::util::bench_util::*; +use re_arrow2::array::*; +use re_arrow2::compute::sort::{lexsort, sort, sort_to_indices, SortColumn, SortOptions}; +use re_arrow2::util::bench_util::*; fn bench_lexsort(arr_a: &dyn Array, array_b: &dyn Array) { let columns = vec![ diff --git a/benches/take_kernels.rs b/benches/take_kernels.rs index f18e091d12a..40f3e05865b 100644 --- a/benches/take_kernels.rs +++ b/benches/take_kernels.rs @@ -2,9 +2,9 @@ use rand::{rngs::StdRng, Rng, SeedableRng}; use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::array::*; -use arrow2::compute::take; -use arrow2::util::bench_util::*; +use re_arrow2::array::*; +use re_arrow2::compute::take; +use re_arrow2::util::bench_util::*; fn create_random_index(size: usize, null_density: f32) -> PrimitiveArray { let mut rng = StdRng::seed_from_u64(42); diff --git a/benches/unset_count.rs b/benches/unset_count.rs index 38d1570d213..7d7ad6b5909 100644 --- a/benches/unset_count.rs +++ b/benches/unset_count.rs @@ -1,6 +1,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::bitmap::utils::count_zeros; +use re_arrow2::bitmap::utils::count_zeros; fn add_benchmark(c: &mut Criterion) { (10..=20).step_by(2).for_each(|log2_size| { diff --git a/benches/write_csv.rs b/benches/write_csv.rs index 7778b0781f4..494d66dd8d3 100644 --- a/benches/write_csv.rs +++ b/benches/write_csv.rs @@ -1,10 +1,10 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::error::Result; -use arrow2::io::csv::write; -use arrow2::util::bench_util::*; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::error::Result; +use re_arrow2::io::csv::write; +use re_arrow2::util::bench_util::*; type ChunkBox = Chunk>; diff --git a/benches/write_ipc.rs b/benches/write_ipc.rs index 346ad783468..7746baf5fcb 100644 --- a/benches/write_ipc.rs +++ b/benches/write_ipc.rs @@ -1,12 +1,14 @@ use criterion::{criterion_group, criterion_main, Criterion}; use std::io::Cursor; -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::datatypes::Field; -use arrow2::error::Result; -use arrow2::io::ipc::write::*; -use arrow2::util::bench_util::{create_boolean_array, create_primitive_array, create_string_array}; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::Field; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::write::*; +use re_arrow2::util::bench_util::{ + create_boolean_array, create_primitive_array, create_string_array, +}; fn write(array: &dyn Array) -> Result<()> { let field = Field::new("c1", array.data_type().clone(), true); diff --git a/benches/write_json.rs b/benches/write_json.rs index d4f464040eb..272d5b1e3d1 100644 --- a/benches/write_json.rs +++ b/benches/write_json.rs @@ -1,9 +1,9 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::array::*; -use arrow2::error::Error; -use arrow2::io::json::write; -use arrow2::util::bench_util::*; +use re_arrow2::array::*; +use re_arrow2::error::Error; +use re_arrow2::io::json::write; +use re_arrow2::util::bench_util::*; fn write_array(array: Box) -> Result<(), Error> { let mut writer = vec![]; diff --git a/benches/write_parquet.rs b/benches/write_parquet.rs index 7062ab919d9..955b4d3e4f8 100644 --- a/benches/write_parquet.rs +++ b/benches/write_parquet.rs @@ -1,11 +1,13 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use arrow2::array::{clone, Array}; -use arrow2::chunk::Chunk; -use arrow2::datatypes::{Field, Schema}; -use arrow2::error::Result; -use arrow2::io::parquet::write::*; -use arrow2::util::bench_util::{create_boolean_array, create_primitive_array, create_string_array}; +use re_arrow2::array::{clone, Array}; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::{Field, Schema}; +use re_arrow2::error::Result; +use re_arrow2::io::parquet::write::*; +use re_arrow2::util::bench_util::{ + create_boolean_array, create_primitive_array, create_string_array, +}; type ChunkBox = Chunk>; diff --git a/examples/arithmetics.rs b/examples/arithmetics.rs index fe931b07e3e..d129891107c 100644 --- a/examples/arithmetics.rs +++ b/examples/arithmetics.rs @@ -1,8 +1,8 @@ -use arrow2::array::{Array, PrimitiveArray}; -use arrow2::compute::arithmetics::basic::*; -use arrow2::compute::arithmetics::{add as dyn_add, can_add}; -use arrow2::compute::arity::{binary, unary}; -use arrow2::datatypes::DataType; +use re_arrow2::array::{Array, PrimitiveArray}; +use re_arrow2::compute::arithmetics::basic::*; +use re_arrow2::compute::arithmetics::{add as dyn_add, can_add}; +use re_arrow2::compute::arity::{binary, unary}; +use re_arrow2::datatypes::DataType; fn main() { // say we have two arrays diff --git a/examples/avro_kafka.rs b/examples/avro_kafka.rs index 7645024939e..3211d759058 100644 --- a/examples/avro_kafka.rs +++ b/examples/avro_kafka.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ datatypes::{DataType, Field}, error::Error, io::avro, diff --git a/examples/avro_read.rs b/examples/avro_read.rs index 6f45afae32f..e8ab2fd5e15 100644 --- a/examples/avro_read.rs +++ b/examples/avro_read.rs @@ -1,9 +1,9 @@ use std::fs::File; use std::io::BufReader; -use arrow2::error::Result; -use arrow2::io::avro::avro_schema; -use arrow2::io::avro::read; +use re_arrow2::error::Result; +use re_arrow2::io::avro::avro_schema; +use re_arrow2::io::avro::read; fn main() -> Result<()> { use std::env; diff --git a/examples/avro_read_async.rs b/examples/avro_read_async.rs index ac7ad0b6450..63b82a3bebe 100644 --- a/examples/avro_read_async.rs +++ b/examples/avro_read_async.rs @@ -5,10 +5,10 @@ use futures::StreamExt; use tokio::fs::File; use tokio_util::compat::*; -use arrow2::error::Result; -use arrow2::io::avro::avro_schema::file::Block; -use arrow2::io::avro::avro_schema::read_async::{block_stream, decompress_block, read_metadata}; -use arrow2::io::avro::read::{deserialize, infer_schema}; +use re_arrow2::error::Result; +use re_arrow2::io::avro::avro_schema::file::Block; +use re_arrow2::io::avro::avro_schema::read_async::{block_stream, decompress_block, read_metadata}; +use re_arrow2::io::avro::read::{deserialize, infer_schema}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<()> { diff --git a/examples/avro_write.rs b/examples/avro_write.rs index 6042172913a..8b6bda2253c 100644 --- a/examples/avro_write.rs +++ b/examples/avro_write.rs @@ -1,6 +1,6 @@ use std::fs::File; -use arrow2::{ +use re_arrow2::{ array::{Array, Int32Array}, datatypes::{Field, Schema}, error::Result, diff --git a/examples/cow.rs b/examples/cow.rs index 65e3920727c..0b23407e617 100644 --- a/examples/cow.rs +++ b/examples/cow.rs @@ -1,6 +1,6 @@ // This example demos how to operate on arrays in-place. -use arrow2::array::{Array, PrimitiveArray}; -use arrow2::compute::arity_assign; +use re_arrow2::array::{Array, PrimitiveArray}; +use re_arrow2::compute::arity_assign; fn main() { // say we have have received an `Array` diff --git a/examples/csv_read.rs b/examples/csv_read.rs index 21addf9d0fe..b3a11ea8c57 100644 --- a/examples/csv_read.rs +++ b/examples/csv_read.rs @@ -1,7 +1,7 @@ -use arrow2::array::Array; -use arrow2::chunk::Chunk; -use arrow2::error::Result; -use arrow2::io::csv::read; +use re_arrow2::array::Array; +use re_arrow2::chunk::Chunk; +use re_arrow2::error::Result; +use re_arrow2::io::csv::read; fn read_path(path: &str, projection: Option<&[usize]>) -> Result>> { // Create a CSV reader. This is typically created on the thread that reads the file and diff --git a/examples/csv_read_async.rs b/examples/csv_read_async.rs index 10d5377fe16..638f8e88e62 100644 --- a/examples/csv_read_async.rs +++ b/examples/csv_read_async.rs @@ -1,8 +1,8 @@ use tokio::fs::File; use tokio_util::compat::*; -use arrow2::error::Result; -use arrow2::io::csv::read_async::*; +use re_arrow2::error::Result; +use re_arrow2::io::csv::read_async::*; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<()> { diff --git a/examples/csv_read_parallel.rs b/examples/csv_read_parallel.rs index 0c3f6e2b106..0bfd625133e 100644 --- a/examples/csv_read_parallel.rs +++ b/examples/csv_read_parallel.rs @@ -3,9 +3,9 @@ use crossbeam_channel::unbounded; use std::thread; use std::time::SystemTime; -use arrow2::array::Array; -use arrow2::chunk::Chunk; -use arrow2::{error::Result, io::csv::read}; +use re_arrow2::array::Array; +use re_arrow2::chunk::Chunk; +use re_arrow2::{error::Result, io::csv::read}; fn parallel_read(path: &str) -> Result>>> { let batch_size = 100; diff --git a/examples/csv_write.rs b/examples/csv_write.rs index 6a40fb7b515..76549a31640 100644 --- a/examples/csv_write.rs +++ b/examples/csv_write.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::{Array, Int32Array}, chunk::Chunk, error::Result, diff --git a/examples/csv_write_parallel.rs b/examples/csv_write_parallel.rs index 65a7e74e5c3..3023b409717 100644 --- a/examples/csv_write_parallel.rs +++ b/examples/csv_write_parallel.rs @@ -3,7 +3,7 @@ use std::sync::mpsc; use std::sync::mpsc::{Receiver, Sender}; use std::thread; -use arrow2::{ +use re_arrow2::{ array::{Array, Int32Array}, chunk::Chunk, error::Result, diff --git a/examples/extension.rs b/examples/extension.rs index c0aea843166..81f0ab3f8d1 100644 --- a/examples/extension.rs +++ b/examples/extension.rs @@ -1,11 +1,11 @@ use std::io::{Cursor, Seek, Write}; -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::datatypes::*; -use arrow2::error::Result; -use arrow2::io::ipc::read; -use arrow2::io::ipc::write; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::*; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read; +use re_arrow2::io::ipc::write; fn main() -> Result<()> { // declare an extension. diff --git a/examples/ffi.rs b/examples/ffi.rs index ed5b9cd87e7..f1f88e62555 100644 --- a/examples/ffi.rs +++ b/examples/ffi.rs @@ -1,7 +1,7 @@ -use arrow2::array::{Array, PrimitiveArray}; -use arrow2::datatypes::Field; -use arrow2::error::Result; -use arrow2::ffi; +use re_arrow2::array::{Array, PrimitiveArray}; +use re_arrow2::datatypes::Field; +use re_arrow2::error::Result; +use re_arrow2::ffi; fn export(array: Box) -> (ffi::ArrowArray, ffi::ArrowSchema) { // importing an array requires an associated field so that the consumer knows its datatype. diff --git a/examples/growable.rs b/examples/growable.rs index cb1e20fcb7e..31ca322a644 100644 --- a/examples/growable.rs +++ b/examples/growable.rs @@ -1,5 +1,5 @@ -use arrow2::array::growable::{Growable, GrowablePrimitive}; -use arrow2::array::PrimitiveArray; +use re_arrow2::array::growable::{Growable, GrowablePrimitive}; +use re_arrow2::array::PrimitiveArray; fn main() { // say we have two sorted arrays diff --git a/examples/io_odbc.rs b/examples/io_odbc.rs index 9305fab6e24..d103b5ac4ca 100644 --- a/examples/io_odbc.rs +++ b/examples/io_odbc.rs @@ -5,14 +5,14 @@ //! sudo apt install libsqliteodbc sqlite3 unixodbc-dev //! sudo sed --in-place 's/libsqlite3odbc.so/\/usr\/lib\/x86_64-linux-gnu\/odbc\/libsqlite3odbc.so/' /etc/odbcinst.ini //! ``` -use arrow2::array::{Array, Int32Array, Utf8Array}; -use arrow2::chunk::Chunk; -use arrow2::datatypes::{DataType, Field}; -use arrow2::error::Result; -use arrow2::io::odbc::api; -use arrow2::io::odbc::api::Cursor; -use arrow2::io::odbc::read; -use arrow2::io::odbc::write; +use re_arrow2::array::{Array, Int32Array, Utf8Array}; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::{DataType, Field}; +use re_arrow2::error::Result; +use re_arrow2::io::odbc::api; +use re_arrow2::io::odbc::api::Cursor; +use re_arrow2::io::odbc::read; +use re_arrow2::io::odbc::write; fn main() -> Result<()> { let connector = "Driver={SQLite3};Database=sqlite-test.db"; diff --git a/examples/ipc_file_mmap.rs b/examples/ipc_file_mmap.rs index e51b49de5be..26081c210ba 100644 --- a/examples/ipc_file_mmap.rs +++ b/examples/ipc_file_mmap.rs @@ -1,10 +1,10 @@ //! Example showing how to memory map an Arrow IPC file into a [`Chunk`]. use std::sync::Arc; -use arrow2::array::{Array, BooleanArray}; -use arrow2::chunk::Chunk; -use arrow2::datatypes::{Field, Schema}; -use arrow2::error::Error; +use re_arrow2::array::{Array, BooleanArray}; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::{Field, Schema}; +use re_arrow2::error::Error; // Arrow2 requires something that implements `AsRef<[u8]>`, which // `Mmap` supports. Here we mock it @@ -22,12 +22,12 @@ impl AsRef<[u8]> for Mmap { fn write( chunks: &[Chunk>], schema: &Schema, - ipc_fields: Option>, - compression: Option, + ipc_fields: Option>, + compression: Option, ) -> Result, Error> { let result = vec![]; - let options = arrow2::io::ipc::write::WriteOptions { compression }; - let mut writer = arrow2::io::ipc::write::FileWriter::try_new( + let options = re_arrow2::io::ipc::write::WriteOptions { compression }; + let mut writer = re_arrow2::io::ipc::write::FileWriter::try_new( result, schema.clone(), ipc_fields.clone(), @@ -49,16 +49,16 @@ fn check_round_trip(array: Box) -> Result<(), Error> { // we first read the files' metadata let metadata = - arrow2::io::ipc::read::read_file_metadata(&mut std::io::Cursor::new(data.as_ref()))?; + re_arrow2::io::ipc::read::read_file_metadata(&mut std::io::Cursor::new(data.as_ref()))?; // next we mmap the dictionaries // Safety: `write` above guarantees that this is a valid Arrow IPC file let dictionaries = - unsafe { arrow2::mmap::mmap_dictionaries_unchecked(&metadata, data.clone())? }; + unsafe { re_arrow2::mmap::mmap_dictionaries_unchecked(&metadata, data.clone())? }; // and finally mmap a chunk (0 in this case). // Safety: `write` above guarantees that this is a valid Arrow IPC file - let new_array = unsafe { arrow2::mmap::mmap_unchecked(&metadata, &dictionaries, data, 0)? }; + let new_array = unsafe { re_arrow2::mmap::mmap_unchecked(&metadata, &dictionaries, data, 0)? }; assert_eq!(new_array.into_arrays()[0], array); Ok(()) } diff --git a/examples/ipc_file_read.rs b/examples/ipc_file_read.rs index c171a5d18db..93e1ff47af3 100644 --- a/examples/ipc_file_read.rs +++ b/examples/ipc_file_read.rs @@ -1,11 +1,11 @@ use std::fs::File; -use arrow2::array::Array; -use arrow2::chunk::Chunk; -use arrow2::datatypes::Schema; -use arrow2::error::Result; -use arrow2::io::ipc::read; -use arrow2::io::print; +use re_arrow2::array::Array; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::Schema; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read; +use re_arrow2::io::print; /// Simplest way: read all record batches from the file. This can be used e.g. for random access. #[allow(clippy::type_complexity)] diff --git a/examples/ipc_file_write.rs b/examples/ipc_file_write.rs index 0629faa80a0..624f5a2dd6f 100644 --- a/examples/ipc_file_write.rs +++ b/examples/ipc_file_write.rs @@ -1,10 +1,10 @@ use std::fs::File; -use arrow2::array::{Array, Int32Array, Utf8Array}; -use arrow2::chunk::Chunk; -use arrow2::datatypes::{DataType, Field, Schema}; -use arrow2::error::Result; -use arrow2::io::ipc::write; +use re_arrow2::array::{Array, Int32Array, Utf8Array}; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::{DataType, Field, Schema}; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::write; fn write_batches(path: &str, schema: Schema, chunks: &[Chunk>]) -> Result<()> { let file = File::create(path)?; diff --git a/examples/ipc_pyarrow/src/main.rs b/examples/ipc_pyarrow/src/main.rs index ce92e4e1b21..904f323ea0e 100644 --- a/examples/ipc_pyarrow/src/main.rs +++ b/examples/ipc_pyarrow/src/main.rs @@ -2,10 +2,10 @@ use std::net::TcpStream; use std::thread; use std::time::Duration; -use arrow2::array::{Array, Int64Array}; -use arrow2::datatypes::DataType; -use arrow2::error::Result; -use arrow2::io::ipc::read; +use re_arrow2::array::{Array, Int64Array}; +use re_arrow2::datatypes::DataType; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read; fn main() -> Result<()> { const ADDRESS: &str = "127.0.0.1:12989"; diff --git a/examples/json_read.rs b/examples/json_read.rs index edea82f2755..15c1f0964e3 100644 --- a/examples/json_read.rs +++ b/examples/json_read.rs @@ -1,9 +1,9 @@ /// Example of reading a JSON file. use std::fs; -use arrow2::array::Array; -use arrow2::error::Result; -use arrow2::io::json::read; +use re_arrow2::array::Array; +use re_arrow2::error::Result; +use re_arrow2::io::json::read; fn read_path(path: &str) -> Result> { // read the file into memory (IO-bounded) diff --git a/examples/json_write.rs b/examples/json_write.rs index 1e86f0560b6..6576ef35fb3 100644 --- a/examples/json_write.rs +++ b/examples/json_write.rs @@ -1,6 +1,6 @@ use std::fs::File; -use arrow2::{ +use re_arrow2::{ array::{Array, Int32Array}, error::Error, io::json::write, diff --git a/examples/metadata.rs b/examples/metadata.rs index c56849daead..2e9949d5784 100644 --- a/examples/metadata.rs +++ b/examples/metadata.rs @@ -1,4 +1,4 @@ -use arrow2::datatypes::{DataType, Field, Metadata, Schema}; +use re_arrow2::datatypes::{DataType, Field, Metadata, Schema}; fn main() { // two data types (logical types) diff --git a/examples/ndjson_read.rs b/examples/ndjson_read.rs index bd242d22df0..8edffdb2f27 100644 --- a/examples/ndjson_read.rs +++ b/examples/ndjson_read.rs @@ -1,10 +1,10 @@ use std::fs::File; use std::io::{BufReader, Seek}; -use arrow2::array::Array; -use arrow2::error::Result; -use arrow2::io::ndjson::read; -use arrow2::io::ndjson::read::FallibleStreamingIterator; +use re_arrow2::array::Array; +use re_arrow2::error::Result; +use re_arrow2::io::ndjson::read; +use re_arrow2::io::ndjson::read::FallibleStreamingIterator; fn read_path(path: &str) -> Result>> { let batch_size = 1024; // number of rows per array diff --git a/examples/ndjson_write.rs b/examples/ndjson_write.rs index 91a0e1a9ed7..ed5cd690fb2 100644 --- a/examples/ndjson_write.rs +++ b/examples/ndjson_write.rs @@ -1,8 +1,8 @@ use std::fs::File; -use arrow2::array::{Array, Int32Array}; -use arrow2::error::Result; -use arrow2::io::ndjson::write; +use re_arrow2::array::{Array, Int32Array}; +use re_arrow2::error::Result; +use re_arrow2::io::ndjson::write; fn write_path(path: &str, array: Box) -> Result<()> { let writer = File::create(path)?; diff --git a/examples/orc_read.rs b/examples/orc_read.rs index f1a5acee4dd..4149ba14842 100644 --- a/examples/orc_read.rs +++ b/examples/orc_read.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::error::Error; -use arrow2::io::orc::{format, read}; +use re_arrow2::array::*; +use re_arrow2::error::Error; +use re_arrow2::io::orc::{format, read}; fn deserialize_column(path: &str, column_name: &str) -> Result, Error> { // open the file diff --git a/examples/parquet_read.rs b/examples/parquet_read.rs index eefb2d23d6d..4b839bb20ad 100644 --- a/examples/parquet_read.rs +++ b/examples/parquet_read.rs @@ -1,8 +1,8 @@ use std::fs::File; use std::time::SystemTime; -use arrow2::error::Error; -use arrow2::io::parquet::read; +use re_arrow2::error::Error; +use re_arrow2::io::parquet::read; fn main() -> Result<(), Error> { // say we have a file diff --git a/examples/parquet_read_async.rs b/examples/parquet_read_async.rs index 8056b853cca..a431f6b5966 100644 --- a/examples/parquet_read_async.rs +++ b/examples/parquet_read_async.rs @@ -5,8 +5,8 @@ use tokio::fs::File; use tokio::io::BufReader; use tokio_util::compat::*; -use arrow2::error::Result; -use arrow2::io::parquet::read::{self, RowGroupDeserializer}; +use re_arrow2::error::Result; +use re_arrow2::io::parquet::read::{self, RowGroupDeserializer}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<()> { diff --git a/examples/parquet_read_parallel/src/main.rs b/examples/parquet_read_parallel/src/main.rs index 6ab26d61f7d..7e293b9efc1 100644 --- a/examples/parquet_read_parallel/src/main.rs +++ b/examples/parquet_read_parallel/src/main.rs @@ -6,7 +6,7 @@ use std::time::SystemTime; use log::trace; use rayon::prelude::*; -use arrow2::{ +use re_arrow2::{ array::Array, chunk::Chunk, error::Result, diff --git a/examples/parquet_write.rs b/examples/parquet_write.rs index 1387f615ebc..6b816a51b48 100644 --- a/examples/parquet_write.rs +++ b/examples/parquet_write.rs @@ -1,6 +1,6 @@ use std::fs::File; -use arrow2::{ +use re_arrow2::{ array::{Array, Int32Array}, chunk::Chunk, datatypes::{Field, Schema}, diff --git a/examples/parquet_write_async.rs b/examples/parquet_write_async.rs index db2ae9e08ca..772d486b88c 100644 --- a/examples/parquet_write_async.rs +++ b/examples/parquet_write_async.rs @@ -1,7 +1,7 @@ use futures::SinkExt; use tokio::fs::File; -use arrow2::{ +use re_arrow2::{ array::{Array, Int32Array}, chunk::Chunk, datatypes::{Field, Schema}, diff --git a/examples/parquet_write_parallel/src/main.rs b/examples/parquet_write_parallel/src/main.rs index 6c87be6143a..743c42ffabb 100644 --- a/examples/parquet_write_parallel/src/main.rs +++ b/examples/parquet_write_parallel/src/main.rs @@ -3,7 +3,7 @@ use std::collections::VecDeque; use rayon::prelude::*; -use arrow2::{ +use re_arrow2::{ array::*, chunk::Chunk as AChunk, datatypes::*, diff --git a/examples/s3/src/main.rs b/examples/s3/src/main.rs index d3f668d4421..fc708cc7308 100644 --- a/examples/s3/src/main.rs +++ b/examples/s3/src/main.rs @@ -1,8 +1,8 @@ -use arrow2::array::{Array, Int64Array}; -use arrow2::error::Result; -use arrow2::io::parquet::read; use futures::future::BoxFuture; use range_reader::{RangeOutput, RangedAsyncReader}; +use re_arrow2::array::{Array, Int64Array}; +use re_arrow2::error::Result; +use re_arrow2::io::parquet::read; use s3::Bucket; #[tokio::main] diff --git a/guide/src/high_level.md b/guide/src/high_level.md index 008bdd83dbe..d706be8d0be 100644 --- a/guide/src/high_level.md +++ b/guide/src/high_level.md @@ -9,7 +9,7 @@ Probably the simplest `Array` in this crate is the `PrimitiveArray`. It can b constructed from a slice of option values, ```rust -# use arrow2::array::{Array, PrimitiveArray}; +# use re_arrow2::array::{Array, PrimitiveArray}; # fn main() { let array = PrimitiveArray::::from([Some(1), None, Some(123)]); assert_eq!(array.len(), 3) @@ -19,7 +19,7 @@ assert_eq!(array.len(), 3) from a slice of values, ```rust -# use arrow2::array::{Array, PrimitiveArray}; +# use re_arrow2::array::{Array, PrimitiveArray}; # fn main() { let array = PrimitiveArray::::from_slice([1.0, 0.0, 123.0]); assert_eq!(array.len(), 3) @@ -29,7 +29,7 @@ assert_eq!(array.len(), 3) or from an iterator ```rust -# use arrow2::array::{Array, PrimitiveArray}; +# use re_arrow2::array::{Array, PrimitiveArray}; # fn main() { let array: PrimitiveArray = [Some(1), None, Some(123)].iter().collect(); assert_eq!(array.len(), 3) @@ -52,8 +52,8 @@ The first allows interoperability with Arrow's ecosystem and efficient SIMD oper In the example ```rust -# use arrow2::array::PrimitiveArray; -# use arrow2::datatypes::DataType; +# use re_arrow2::array::PrimitiveArray; +# use re_arrow2::datatypes::DataType; # fn main() { let ints = PrimitiveArray::::from([Some(1), None]); let dates = PrimitiveArray::::from([Some(1), None]).to(DataType::Date32); @@ -67,8 +67,8 @@ All physical types (e.g. `i32`) have a "natural" logical `DataType` (e.g. `DataT which is assigned when allocating arrays from iterators, slices, etc. ```rust -# use arrow2::array::{Array, Int32Array, PrimitiveArray}; -# use arrow2::datatypes::DataType; +# use re_arrow2::array::{Array, Int32Array, PrimitiveArray}; +# use re_arrow2::datatypes::DataType; # fn main() { let array = PrimitiveArray::::from_slice([1, 0, 123]); assert_eq!(array.data_type(), &DataType::Int32); @@ -96,7 +96,7 @@ The following arrays are supported: to `&dyn Array`, which enables dynamic casting and run-time nesting. ```rust -# use arrow2::array::{Array, PrimitiveArray}; +# use re_arrow2::array::{Array, PrimitiveArray}; # fn main() { let a = PrimitiveArray::::from(&[Some(1), None]); let a: &dyn Array = &a; @@ -110,8 +110,8 @@ Given a trait object `array: &dyn Array`, we know its physical type via to its concrete physical type: ```rust -# use arrow2::array::{Array, PrimitiveArray}; -# use arrow2::datatypes::PhysicalType; +# use re_arrow2::array::{Array, PrimitiveArray}; +# use re_arrow2::datatypes::PhysicalType; # fn main() { let array = PrimitiveArray::::from(&[Some(1), None]); let array = &array as &dyn Array; @@ -144,8 +144,8 @@ where `_` represents each of the variants (e.g. `PrimitiveType::Int32 <-> i32`). In this context, a common idiom in using `Array` as a trait object is as follows: ```rust -use arrow2::datatypes::{PhysicalType, PrimitiveType}; -use arrow2::array::{Array, PrimitiveArray}; +use re_arrow2::datatypes::{PhysicalType, PrimitiveType}; +use re_arrow2::array::{Array, PrimitiveArray}; fn float_operator(array: &dyn Array) -> Result, String> { match array.data_type().to_physical_type() { @@ -193,7 +193,7 @@ We've already seen how to create an array from an iterator. Most arrays also imp `IntoIterator`: ```rust -# use arrow2::array::{Array, Int32Array}; +# use re_arrow2::array::{Array, Int32Array}; # fn main() { let array = Int32Array::from(&[Some(1), None, Some(123)]); @@ -219,7 +219,7 @@ validity and values, while the latter is suitable for SIMD and copies, as they r contiguous memory regions (buffers and bitmaps). We will see below how to leverage these APIs. This idea holds more generally in this crate's arrays: `values()` returns something that has -a contiguous in-memory representation, while `iter()` returns items taking validity into account. +a contiguous in-memory representation, while `iter()` returns items taking validity into account. To get an iterator over contiguous values, use `array.values().iter()`. There is one last API that is worth mentioning, and that is `Bitmap::chunks`. When performing @@ -236,8 +236,8 @@ it often enables SIMD. For example, an unary operation `op` on a `PrimitiveArray likely emits SIMD instructions on the following code: ```rust -# use arrow2::buffer::Buffer; -# use arrow2::{ +# use re_arrow2::buffer::Buffer; +# use re_arrow2::{ # array::{Array, PrimitiveArray}, # types::NativeType, # datatypes::DataType, @@ -275,7 +275,7 @@ We support the mutation of arrays in-place via clone-on-write semantics. Essentially, all data is under an `Arc`, but it can be taken via `Arc::get_mut` and operated in place. -Below is a complete example of how to operate on a `Box` without +Below is a complete example of how to operate on a `Box` without extra allocations. ```rust,ignore diff --git a/guide/src/low_level.md b/guide/src/low_level.md index 83fa0f08bf1..6ca90923a02 100644 --- a/guide/src/low_level.md +++ b/guide/src/low_level.md @@ -28,7 +28,7 @@ Let's see how these structures are used. Create a new `Buffer`: ```rust -# use arrow2::buffer::Buffer; +# use re_arrow2::buffer::Buffer; # fn main() { let x = vec![1u32, 2, 3]; let x: Buffer = x.into(); @@ -45,8 +45,8 @@ the following physical types: * `i8-i128` * `u8-u64` * `f32` and `f64` -* `arrow2::types::days_ms` -* `arrow2::types::months_days_ns` +* `re_arrow2::types::days_ms` +* `re_arrow2::types::months_days_ns` This is because the arrow specification only supports the above Rust types; all other complex types supported by arrow are built on top of these types, which enables Arrow to be a highly @@ -57,7 +57,7 @@ interoperable in-memory format. Arrow's in-memory arrangement of boolean values is different from `Vec`. Specifically, arrow uses individual bits to represent a boolean, as opposed to the usual byte that `bool` holds. -Besides the 8x compression, this makes the validity particularly useful for +Besides the 8x compression, this makes the validity particularly useful for [AVX512](https://en.wikipedia.org/wiki/AVX-512) masks. One tradeoff is that an arrows' bitmap is not represented as a Rust slice, as Rust slices use pointer arithmetics, whose smallest unit is a byte. @@ -66,7 +66,7 @@ Arrow2 has two containers for bitmaps: `Bitmap` (immutable and sharable) and `MutableBitmap` (mutable): ```rust -use arrow2::bitmap::Bitmap; +use re_arrow2::bitmap::Bitmap; # fn main() { let x = Bitmap::from(&[true, false]); let iter = x.iter().map(|x| !x); @@ -77,7 +77,7 @@ assert_eq!(y.get_bit(1), true); ``` ```rust -use arrow2::bitmap::MutableBitmap; +use re_arrow2::bitmap::MutableBitmap; # fn main() { let mut x = MutableBitmap::new(); x.push(true); diff --git a/guide/src/metadata.md b/guide/src/metadata.md index 7a78d82edac..026ef1d5cc9 100644 --- a/guide/src/metadata.md +++ b/guide/src/metadata.md @@ -9,7 +9,7 @@ The Arrow specification contains a set of logical types, an enumeration of the different semantical types defined in Arrow. -In Arrow2, logical types are declared as variants of the `enum` `arrow2::datatypes::DataType`. +In Arrow2, logical types are declared as variants of the `enum` `re_arrow2::datatypes::DataType`. For example, `DataType::Int32` represents a signed integer of 32 bits. Each `DataType` has an associated `enum PhysicalType` (many-to-one) representing the @@ -29,7 +29,7 @@ nullable (`bool`), and optional metadata. ## `Schema` (table metadata) -The most common use of `Field` is to declare a `arrow2::datatypes::Schema`, a sequence of `Field`s +The most common use of `Field` is to declare a `re_arrow2::datatypes::Schema`, a sequence of `Field`s with optional metadata. `Schema` is essentially metadata of a "table": it has a sequence of named columns and their metadata (`Field`s) with optional metadata. diff --git a/integration-testing/src/bin/arrow-file-to-stream.rs b/integration-testing/src/bin/arrow-file-to-stream.rs index c63606e28c3..2645bb2d2f7 100644 --- a/integration-testing/src/bin/arrow-file-to-stream.rs +++ b/integration-testing/src/bin/arrow-file-to-stream.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -use arrow2::error::Result; -use arrow2::io::ipc::read; -use arrow2::io::ipc::write; use clap::Parser; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read; +use re_arrow2::io::ipc::write; use std::fs::File; #[derive(Debug, Parser)] diff --git a/integration-testing/src/bin/arrow-json-integration-test.rs b/integration-testing/src/bin/arrow-json-integration-test.rs index 6c1dea1e2b1..a8caf86ceb7 100644 --- a/integration-testing/src/bin/arrow-json-integration-test.rs +++ b/integration-testing/src/bin/arrow-json-integration-test.rs @@ -1,15 +1,15 @@ use std::fs::File; -use arrow2::io::json_integration::ArrowJson; use clap::Parser; +use re_arrow2::io::json_integration::ArrowJson; -use arrow2::io::ipc::read; -use arrow2::io::ipc::write; -use arrow2::{ +use arrow_integration_testing::read_json_file; +use re_arrow2::io::ipc::read; +use re_arrow2::io::ipc::write; +use re_arrow2::{ error::{Error, Result}, io::json_integration::write as json_write, }; -use arrow_integration_testing::read_json_file; #[derive(Debug, Clone, clap::ArgEnum)] #[clap(rename_all = "SCREAMING_SNAKE_CASE")] diff --git a/integration-testing/src/bin/arrow-stream-to-file.rs b/integration-testing/src/bin/arrow-stream-to-file.rs index bd431e73c07..a7c121b897c 100644 --- a/integration-testing/src/bin/arrow-stream-to-file.rs +++ b/integration-testing/src/bin/arrow-stream-to-file.rs @@ -17,9 +17,9 @@ use std::io; -use arrow2::error::Result; -use arrow2::io::ipc::read; -use arrow2::io::ipc::write; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read; +use re_arrow2::io::ipc::write; fn main() -> Result<()> { let mut reader = io::stdin(); diff --git a/integration-testing/src/flight_client_scenarios/integration_test.rs b/integration-testing/src/flight_client_scenarios/integration_test.rs index 8955a3e121d..94e389b28d2 100644 --- a/integration-testing/src/flight_client_scenarios/integration_test.rs +++ b/integration-testing/src/flight_client_scenarios/integration_test.rs @@ -17,7 +17,12 @@ use crate::{read_json_file, ArrowFile}; -use arrow2::{ +use arrow_format::flight::data::{ + flight_descriptor::DescriptorType, FlightData, FlightDescriptor, Location, Ticket, +}; +use arrow_format::flight::service::flight_service_client::FlightServiceClient; +use futures::{stream::BoxStream, StreamExt, TryStreamExt}; +use re_arrow2::{ array::Array, chunk::Chunk, datatypes::*, @@ -26,11 +31,6 @@ use arrow2::{ ipc::{read::Dictionaries, write, IpcField, IpcSchema}, }, }; -use arrow_format::flight::data::{ - flight_descriptor::DescriptorType, FlightData, FlightDescriptor, Location, Ticket, -}; -use arrow_format::flight::service::flight_service_client::FlightServiceClient; -use futures::{stream::BoxStream, StreamExt, TryStreamExt}; use tonic::{Request, Streaming}; type Error = Box; diff --git a/integration-testing/src/flight_server_scenarios/integration_test.rs b/integration-testing/src/flight_server_scenarios/integration_test.rs index 89df0a041b5..448a9363822 100644 --- a/integration-testing/src/flight_server_scenarios/integration_test.rs +++ b/integration-testing/src/flight_server_scenarios/integration_test.rs @@ -27,15 +27,15 @@ use arrow_format::flight::data::flight_descriptor::*; use arrow_format::flight::data::*; use arrow_format::flight::service::flight_service_server::*; -use arrow2::array::Array; -use arrow2::chunk::Chunk; -use arrow2::datatypes::Schema; -use arrow2::io::flight::{ +use re_arrow2::array::Array; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::Schema; +use re_arrow2::io::flight::{ deserialize_message, deserialize_schemas, serialize_batch, serialize_schema, serialize_schema_to_info, }; -use arrow2::io::ipc; -use arrow2::io::ipc::read::Dictionaries; +use re_arrow2::io::ipc; +use re_arrow2::io::ipc::read::Dictionaries; use super::{Result, TonicStream}; diff --git a/integration-testing/src/lib.rs b/integration-testing/src/lib.rs index e36ad7eef03..58099081c41 100644 --- a/integration-testing/src/lib.rs +++ b/integration-testing/src/lib.rs @@ -17,15 +17,15 @@ //! Common code used in the integration test binaries -use arrow2::array::Array; -use arrow2::io::ipc::IpcField; +use re_arrow2::array::Array; +use re_arrow2::io::ipc::IpcField; use serde_json::Value; -use arrow2::chunk::Chunk; -use arrow2::AHashMap; -use arrow2::datatypes::*; -use arrow2::error::Result; -use arrow2::io::json_integration::{read, ArrowJsonBatch, ArrowJsonDictionaryBatch}; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::*; +use re_arrow2::error::Result; +use re_arrow2::io::json_integration::{read, ArrowJsonBatch, ArrowJsonDictionaryBatch}; +use re_arrow2::AHashMap; use std::fs::File; use std::io::BufReader; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 904c6cc5fcb..7e8f0a9aabf 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2023-06-01" +channel = "1.72" diff --git a/src/array/binary/mod.rs b/src/array/binary/mod.rs index 7247decb300..00f50e34716 100644 --- a/src/array/binary/mod.rs +++ b/src/array/binary/mod.rs @@ -33,9 +33,9 @@ mod data; /// The size of this struct is `O(1)`, as all data is stored behind an [`std::sync::Arc`]. /// # Example /// ``` -/// use arrow2::array::BinaryArray; -/// use arrow2::bitmap::Bitmap; -/// use arrow2::buffer::Buffer; +/// use re_arrow2::array::BinaryArray; +/// use re_arrow2::bitmap::Bitmap; +/// use re_arrow2::buffer::Buffer; /// /// let array = BinaryArray::::from([Some([1, 2].as_ref()), None, Some([3].as_ref())]); /// assert_eq!(array.value(0), &[1, 2]); diff --git a/src/array/boolean/mod.rs b/src/array/boolean/mod.rs index 0b634ee90e3..2def39750b9 100644 --- a/src/array/boolean/mod.rs +++ b/src/array/boolean/mod.rs @@ -19,7 +19,6 @@ mod from; mod iterator; mod mutable; -pub use iterator::*; pub use mutable::*; /// A [`BooleanArray`] is Arrow's semantically equivalent of an immutable `Vec>`. @@ -34,9 +33,9 @@ pub use mutable::*; /// The size of this struct is `O(1)`, as all data is stored behind an [`std::sync::Arc`]. /// # Example /// ``` -/// use arrow2::array::BooleanArray; -/// use arrow2::bitmap::Bitmap; -/// use arrow2::buffer::Buffer; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::bitmap::Bitmap; +/// use re_arrow2::buffer::Buffer; /// /// let array = BooleanArray::from([Some(true), None, Some(false)]); /// assert_eq!(array.value(0), true); diff --git a/src/array/dictionary/mod.rs b/src/array/dictionary/mod.rs index 3a23e670a1d..d62b76bb421 100644 --- a/src/array/dictionary/mod.rs +++ b/src/array/dictionary/mod.rs @@ -1,5 +1,4 @@ -use std::hash::Hash; -use std::hint::unreachable_unchecked; +use std::{hash::Hash, hint::unreachable_unchecked, sync::Arc}; use crate::{ bitmap::{ @@ -292,7 +291,7 @@ impl DictionaryArray { } pub(crate) fn default_data_type(values_datatype: DataType) -> DataType { - DataType::Dictionary(K::KEY_TYPE, Box::new(values_datatype), false) + DataType::Dictionary(K::KEY_TYPE, Arc::new(values_datatype), false) } /// Slices this [`DictionaryArray`]. diff --git a/src/array/dictionary/mutable.rs b/src/array/dictionary/mutable.rs index b48a57a9458..04822135e22 100644 --- a/src/array/dictionary/mutable.rs +++ b/src/array/dictionary/mutable.rs @@ -17,7 +17,7 @@ use super::{DictionaryArray, DictionaryKey}; /// # Example /// Building a UTF8 dictionary with `i32` keys. /// ``` -/// # use arrow2::array::{MutableDictionaryArray, MutableUtf8Array, TryPush}; +/// # use re_arrow2::array::{MutableDictionaryArray, MutableUtf8Array, TryPush}; /// # fn main() -> Result<(), Box> { /// let mut array: MutableDictionaryArray> = MutableDictionaryArray::new(); /// array.try_push(Some("A"))?; @@ -86,7 +86,7 @@ impl MutableDictionaryArray { fn from_value_map(value_map: ValueMap) -> Self { let keys = MutablePrimitiveArray::::new(); let data_type = - DataType::Dictionary(K::KEY_TYPE, Box::new(value_map.data_type().clone()), false); + DataType::Dictionary(K::KEY_TYPE, Arc::new(value_map.data_type().clone()), false); Self { data_type, map: value_map, diff --git a/src/array/fixed_size_list/mod.rs b/src/array/fixed_size_list/mod.rs index 0d335167b20..edd8e1968c7 100644 --- a/src/array/fixed_size_list/mod.rs +++ b/src/array/fixed_size_list/mod.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::{ bitmap::Bitmap, datatypes::{DataType, Field}, @@ -11,7 +13,6 @@ mod data; mod ffi; pub(super) mod fmt; mod iterator; -pub use iterator::*; mod mutable; pub use mutable::*; @@ -204,7 +205,7 @@ impl FixedSizeListArray { /// Returns a [`DataType`] consistent with [`FixedSizeListArray`]. pub fn default_datatype(data_type: DataType, size: usize) -> DataType { - let field = Box::new(Field::new("item", data_type, true)); + let field = Arc::new(Field::new("item", data_type, true)); DataType::FixedSizeList(field, size) } } diff --git a/src/array/fixed_size_list/mutable.rs b/src/array/fixed_size_list/mutable.rs index 1e387a2f70c..60760df8548 100644 --- a/src/array/fixed_size_list/mutable.rs +++ b/src/array/fixed_size_list/mutable.rs @@ -42,7 +42,7 @@ impl MutableFixedSizeListArray { /// Creates a new [`MutableFixedSizeListArray`] from a [`MutableArray`] and size. pub fn new_with_field(values: M, name: &str, nullable: bool, size: usize) -> Self { let data_type = DataType::FixedSizeList( - Box::new(Field::new(name, values.data_type().clone(), nullable)), + Arc::new(Field::new(name, values.data_type().clone(), nullable)), size, ); Self::new_from(values, data_type, size) diff --git a/src/array/list/mod.rs b/src/array/list/mod.rs index b7eda9b4d5c..55b4875cf75 100644 --- a/src/array/list/mod.rs +++ b/src/array/list/mod.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::{ bitmap::Bitmap, datatypes::{DataType, Field}, @@ -188,7 +190,7 @@ impl ListArray { impl ListArray { /// Returns a default [`DataType`]: inner field is named "item" and is nullable pub fn default_datatype(data_type: DataType) -> DataType { - let field = Box::new(Field::new("item", data_type, true)); + let field = Arc::new(Field::new("item", data_type, true)); if O::IS_LARGE { DataType::LargeList(field) } else { diff --git a/src/array/list/mutable.rs b/src/array/list/mutable.rs index d24475e86db..b0046f42594 100644 --- a/src/array/list/mutable.rs +++ b/src/array/list/mutable.rs @@ -127,7 +127,7 @@ impl MutableListArray { /// Creates a new [`MutableListArray`] from a [`MutableArray`]. pub fn new_with_field(values: M, name: &str, nullable: bool) -> Self { - let field = Box::new(Field::new(name, values.data_type().clone(), nullable)); + let field = Arc::new(Field::new(name, values.data_type().clone(), nullable)); let data_type = if O::IS_LARGE { DataType::LargeList(field) } else { diff --git a/src/array/map/mod.rs b/src/array/map/mod.rs index 952695297fa..3668c425fa1 100644 --- a/src/array/map/mod.rs +++ b/src/array/map/mod.rs @@ -12,7 +12,6 @@ mod data; mod ffi; pub(super) mod fmt; mod iterator; -pub use iterator::*; /// An array representing a (key, value), both of arbitrary logical types. #[derive(Clone)] diff --git a/src/array/mod.rs b/src/array/mod.rs index 02735c3d0bb..4807b12b61b 100644 --- a/src/array/mod.rs +++ b/src/array/mod.rs @@ -25,7 +25,7 @@ use crate::{ datatypes::DataType, }; -pub(self) mod physical_binary; +mod physical_binary; /// A trait representing an immutable Arrow array. Arrow arrays are trait objects /// that are infallibly downcasted to concrete types according to the [`Array::data_type`]. diff --git a/src/array/ord.rs b/src/array/ord.rs index 439efa1e21e..823873765bf 100644 --- a/src/array/ord.rs +++ b/src/array/ord.rs @@ -161,9 +161,9 @@ macro_rules! dyn_dict { /// between two [`Array`]. /// # Example /// ``` -/// use arrow2::array::{ord::build_compare, PrimitiveArray}; +/// use re_arrow2::array::{ord::build_compare, PrimitiveArray}; /// -/// # fn main() -> arrow2::error::Result<()> { +/// # fn main() -> re_arrow2::error::Result<()> { /// let array1 = PrimitiveArray::from_slice([1, 2]); /// let array2 = PrimitiveArray::from_slice([3, 4]); /// diff --git a/src/array/primitive/fmt.rs b/src/array/primitive/fmt.rs index 05357ef5876..0108cc646af 100644 --- a/src/array/primitive/fmt.rs +++ b/src/array/primitive/fmt.rs @@ -15,7 +15,11 @@ macro_rules! dyn_primitive { .as_any() .downcast_ref::>() .unwrap(); - Box::new(move |f, index| write!(f, "{}", $expr(array.value(index)))) + Box::new(move |f, index| { + #[allow(clippy::redundant_closure_call)] + let value = $expr(array.value(index)); + write!(f, "{}", value) + }) }}; } diff --git a/src/array/primitive/mod.rs b/src/array/primitive/mod.rs index 04b74a3529b..1b6fdc04c9f 100644 --- a/src/array/primitive/mod.rs +++ b/src/array/primitive/mod.rs @@ -19,7 +19,6 @@ mod ffi; pub(super) mod fmt; mod from_natural; mod iterator; -pub use iterator::*; mod mutable; pub use mutable::*; @@ -35,9 +34,9 @@ pub use mutable::*; /// The size of this struct is `O(1)`, as all data is stored behind an [`std::sync::Arc`]. /// # Example /// ``` -/// use arrow2::array::PrimitiveArray; -/// use arrow2::bitmap::Bitmap; -/// use arrow2::buffer::Buffer; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::bitmap::Bitmap; +/// use re_arrow2::buffer::Buffer; /// /// let array = PrimitiveArray::from([Some(1i32), None, Some(10)]); /// assert_eq!(array.value(0), 1); @@ -102,8 +101,8 @@ impl PrimitiveArray { /// Used to change the arrays' logical type (see example). /// # Example /// ``` - /// use arrow2::array::Int32Array; - /// use arrow2::datatypes::DataType; + /// use re_arrow2::array::Int32Array; + /// use re_arrow2::datatypes::DataType; /// /// let array = Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Date32); /// assert_eq!( @@ -133,7 +132,7 @@ impl PrimitiveArray { /// This function is `O(1)`. /// # Examples /// ``` - /// use arrow2::array::PrimitiveArray; + /// use re_arrow2::array::PrimitiveArray; /// /// let array = PrimitiveArray::from_vec(vec![1, 2, 3]); /// assert_eq!(format!("{:?}", array), "Int32[1, 2, 3]"); diff --git a/src/array/struct_/mod.rs b/src/array/struct_/mod.rs index 767ba8242fc..5ba1944d274 100644 --- a/src/array/struct_/mod.rs +++ b/src/array/struct_/mod.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::{ bitmap::Bitmap, datatypes::{DataType, Field}, @@ -18,8 +20,8 @@ pub use mutable::*; /// multiple [`Array`] with the same number of rows. /// # Example /// ``` -/// use arrow2::array::*; -/// use arrow2::datatypes::*; +/// use re_arrow2::array::*; +/// use re_arrow2::datatypes::*; /// let boolean = BooleanArray::from_slice(&[false, false, true, true]).boxed(); /// let int = Int32Array::from_slice(&[42, 28, 19, 31]).boxed(); /// @@ -28,7 +30,7 @@ pub use mutable::*; /// Field::new("c", DataType::Int32, false), /// ]; /// -/// let array = StructArray::new(DataType::Struct(fields), vec![boolean, int], None); +/// let array = StructArray::new(DataType::Struct(std::sync::Arc::new(fields)), vec![boolean, int], None); /// ``` #[derive(Clone)] pub struct StructArray { @@ -69,7 +71,7 @@ impl StructArray { .try_for_each(|(index, (data_type, child))| { if data_type != child { Err(Error::oos(format!( - "The children DataTypes of a StructArray must equal the children data types. + "The children DataTypes of a StructArray must equal the children data types. However, the field {index} has data type {data_type:?} but the value has data type {child:?}" ))) } else { @@ -153,7 +155,7 @@ impl StructArray { impl StructArray { /// Deconstructs the [`StructArray`] into its individual components. #[must_use] - pub fn into_data(self) -> (Vec, Vec>, Option) { + pub fn into_data(self) -> (Arc>, Vec>, Option) { let Self { data_type, values, diff --git a/src/array/union/mod.rs b/src/array/union/mod.rs index e3e664916f8..624a6d93bc3 100644 --- a/src/array/union/mod.rs +++ b/src/array/union/mod.rs @@ -73,7 +73,7 @@ impl UnionArray { .try_for_each(|(index, (data_type, child))| { if data_type != child { Err(Error::oos(format!( - "The children DataTypes of a UnionArray must equal the children data types. + "The children DataTypes of a UnionArray must equal the children data types. However, the field {index} has data type {data_type:?} but the value has data type {child:?}" ))) } else { @@ -352,7 +352,7 @@ impl UnionArray { fn try_get_all(data_type: &DataType) -> Result { match data_type.to_logical_type() { DataType::Union(fields, ids, mode) => { - Ok((fields, ids.as_ref().map(|x| x.as_ref()), *mode)) + Ok((fields, ids.as_ref().map(|x| x.as_slice()), *mode)) } _ => Err(Error::oos( "The UnionArray requires a logical type of DataType::Union", diff --git a/src/array/utf8/mod.rs b/src/array/utf8/mod.rs index 9440ae43304..fdc5b9a5355 100644 --- a/src/array/utf8/mod.rs +++ b/src/array/utf8/mod.rs @@ -42,9 +42,9 @@ impl> AsRef<[u8]> for StrAsBytes { /// Cloning and slicing this struct is `O(1)`. /// # Example /// ``` -/// use arrow2::bitmap::Bitmap; -/// use arrow2::buffer::Buffer; -/// use arrow2::array::Utf8Array; +/// use re_arrow2::bitmap::Bitmap; +/// use re_arrow2::buffer::Buffer; +/// use re_arrow2::array::Utf8Array; /// # fn main() { /// let array = Utf8Array::::from([Some("hi"), None, Some("there")]); /// assert_eq!(array.value(0), "hi"); diff --git a/src/bitmap/immutable.rs b/src/bitmap/immutable.rs index 6883d3312fb..49a8f14e809 100644 --- a/src/bitmap/immutable.rs +++ b/src/bitmap/immutable.rs @@ -15,7 +15,7 @@ use super::{ /// /// # Examples /// ``` -/// use arrow2::bitmap::{Bitmap, MutableBitmap}; +/// use re_arrow2::bitmap::{Bitmap, MutableBitmap}; /// /// let bitmap = Bitmap::from([true, false, true]); /// assert_eq!(bitmap.iter().collect::>(), vec![true, false, true]); diff --git a/src/bitmap/mutable.rs b/src/bitmap/mutable.rs index 31834f21657..c2d2b5d50c3 100644 --- a/src/bitmap/mutable.rs +++ b/src/bitmap/mutable.rs @@ -22,7 +22,7 @@ use super::Bitmap; /// A [`MutableBitmap`] can be converted to a [`Bitmap`] at `O(1)`. /// # Examples /// ``` -/// use arrow2::bitmap::MutableBitmap; +/// use re_arrow2::bitmap::MutableBitmap; /// /// let bitmap = MutableBitmap::from([true, false, true]); /// assert_eq!(bitmap.iter().collect::>(), vec![true, false, true]); diff --git a/src/buffer/immutable.rs b/src/buffer/immutable.rs index 0da4a41ace4..f3824d483f1 100644 --- a/src/buffer/immutable.rs +++ b/src/buffer/immutable.rs @@ -17,7 +17,7 @@ use super::IntoIter; /// /// # Examples /// ``` -/// use arrow2::buffer::Buffer; +/// use re_arrow2::buffer::Buffer; /// /// let mut buffer: Buffer = vec![1, 2, 3].into(); /// assert_eq!(buffer.as_ref(), [1, 2, 3].as_ref()); diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index 46c0a4d64a3..ba50325cd04 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -7,6 +7,7 @@ use crate::ffi::InternalArrowArray; use std::ops::Deref; pub(crate) enum BytesAllocator { + #[allow(dead_code)] InternalArrowArray(InternalArrowArray), #[cfg(feature = "arrow")] diff --git a/src/compute/aggregate/min_max.rs b/src/compute/aggregate/min_max.rs index da064d9b013..17cb7640788 100644 --- a/src/compute/aggregate/min_max.rs +++ b/src/compute/aggregate/min_max.rs @@ -1,3 +1,5 @@ +#![allow(clippy::redundant_closure_call)] + use crate::bitmap::utils::{BitChunkIterExact, BitChunksExact}; use crate::datatypes::{DataType, PhysicalType, PrimitiveType}; use crate::error::{Error, Result}; @@ -255,7 +257,7 @@ pub fn min_string(array: &Utf8Array) -> Option<&str> { /// Returns the minimum value in the boolean array. /// /// ``` -/// use arrow2::{ +/// use re_arrow2::{ /// array::BooleanArray, /// compute::aggregate::min_boolean, /// }; @@ -283,7 +285,7 @@ pub fn min_boolean(array: &BooleanArray) -> Option { /// Returns the maximum value in the boolean array /// /// ``` -/// use arrow2::{ +/// use re_arrow2::{ /// array::BooleanArray, /// compute::aggregate::max_boolean, /// }; diff --git a/src/compute/aggregate/simd/packed.rs b/src/compute/aggregate/simd/packed.rs index c3ee4ffcf32..809bac7a58f 100644 --- a/src/compute/aggregate/simd/packed.rs +++ b/src/compute/aggregate/simd/packed.rs @@ -1,4 +1,4 @@ -use std::simd::{SimdFloat as _, SimdInt as _, SimdOrd as _, SimdUint as _}; +use std::simd::prelude::*; use crate::types::simd::*; @@ -35,31 +35,37 @@ macro_rules! simd_ord_int { #[inline] fn max_element(self) -> $type { + use std::simd::prelude::*; self.reduce_max() } #[inline] fn min_element(self) -> $type { + use std::simd::prelude::*; self.reduce_min() } #[inline] fn max_lane(self, x: Self) -> Self { + use std::simd::prelude::*; self.simd_max(x) } #[inline] fn min_lane(self, x: Self) -> Self { + use std::simd::prelude::*; self.simd_min(x) } #[inline] fn new_min() -> Self { + use std::simd::prelude::*; Self::splat(Self::MAX) } #[inline] fn new_max() -> Self { + use std::simd::prelude::*; Self::splat(Self::MIN) } } diff --git a/src/compute/arithmetics/basic/add.rs b/src/compute/arithmetics/basic/add.rs index 81f5b7bb039..74012171225 100644 --- a/src/compute/arithmetics/basic/add.rs +++ b/src/compute/arithmetics/basic/add.rs @@ -23,8 +23,8 @@ use super::NativeArithmetics; /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::add; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::add; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([None, Some(6), None, Some(6)]); /// let b = PrimitiveArray::from([Some(5), None, None, Some(6)]); @@ -44,8 +44,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::wrapping_add; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::wrapping_add; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([Some(-100i8), Some(100i8), Some(100i8)]); /// let b = PrimitiveArray::from([Some(0i8), Some(100i8), Some(0i8)]); @@ -67,8 +67,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_add; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::checked_add; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([Some(100i8), Some(100i8), Some(100i8)]); /// let b = PrimitiveArray::from([Some(0i8), Some(100i8), Some(0i8)]); @@ -91,8 +91,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::saturating_add; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::saturating_add; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([Some(100i8)]); /// let b = PrimitiveArray::from([Some(100i8)]); @@ -116,8 +116,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::overflowing_add; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::overflowing_add; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([Some(1i8), Some(100i8)]); /// let b = PrimitiveArray::from([Some(1i8), Some(100i8)]); @@ -191,8 +191,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::add_scalar; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::add_scalar; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([None, Some(6), None, Some(6)]); /// let result = add_scalar(&a, &1i32); @@ -212,8 +212,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::wrapping_add_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::wrapping_add_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[None, Some(100)]); /// let result = wrapping_add_scalar(&a, &100i8); @@ -233,8 +233,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_add_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::checked_add_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[None, Some(100), None, Some(100)]); /// let result = checked_add_scalar(&a, &100i8); @@ -257,8 +257,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::saturating_add_scalar; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::saturating_add_scalar; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([Some(100i8)]); /// let result = saturating_add_scalar(&a, &100i8); @@ -282,8 +282,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::overflowing_add_scalar; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::overflowing_add_scalar; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([Some(1i8), Some(100i8)]); /// let (result, overflow) = overflowing_add_scalar(&a, &100i8); diff --git a/src/compute/arithmetics/basic/div.rs b/src/compute/arithmetics/basic/div.rs index b7f22a0d771..cf53757e9de 100644 --- a/src/compute/arithmetics/basic/div.rs +++ b/src/compute/arithmetics/basic/div.rs @@ -23,8 +23,8 @@ use super::NativeArithmetics; /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::div; -/// use arrow2::array::Int32Array; +/// use re_arrow2::compute::arithmetics::basic::div; +/// use re_arrow2::array::Int32Array; /// /// let a = Int32Array::from(&[Some(10), Some(1), Some(6)]); /// let b = Int32Array::from(&[Some(5), None, Some(6)]); @@ -55,8 +55,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_div; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::checked_div; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(-100i8), Some(10i8)]); /// let b = Int8Array::from(&[Some(100i8), Some(0i8)]); @@ -98,8 +98,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::div_scalar; -/// use arrow2::array::Int32Array; +/// use re_arrow2::compute::arithmetics::basic::div_scalar; +/// use re_arrow2::array::Int32Array; /// /// let a = Int32Array::from(&[None, Some(6), None, Some(6)]); /// let result = div_scalar(&a, &2i32); @@ -170,8 +170,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_div_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::checked_div_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(-100i8)]); /// let result = checked_div_scalar(&a, &100i8); diff --git a/src/compute/arithmetics/basic/mod.rs b/src/compute/arithmetics/basic/mod.rs index 22ed09baf6e..459a3f71e89 100644 --- a/src/compute/arithmetics/basic/mod.rs +++ b/src/compute/arithmetics/basic/mod.rs @@ -46,8 +46,8 @@ impl NativeArithmetics for f64 {} /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::negate; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::negate; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([None, Some(6), None, Some(7)]); /// let result = negate(&a); @@ -65,8 +65,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_negate; -/// use arrow2::array::{Array, PrimitiveArray}; +/// use re_arrow2::compute::arithmetics::basic::checked_negate; +/// use re_arrow2::array::{Array, PrimitiveArray}; /// /// let a = PrimitiveArray::from([None, Some(6), Some(i8::MIN), Some(7)]); /// let result = checked_negate(&a); @@ -85,8 +85,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::wrapping_negate; -/// use arrow2::array::{Array, PrimitiveArray}; +/// use re_arrow2::compute::arithmetics::basic::wrapping_negate; +/// use re_arrow2::array::{Array, PrimitiveArray}; /// /// let a = PrimitiveArray::from([None, Some(6), Some(i8::MIN), Some(7)]); /// let result = wrapping_negate(&a); diff --git a/src/compute/arithmetics/basic/mul.rs b/src/compute/arithmetics/basic/mul.rs index a3b405b845f..c37b1eed1ac 100644 --- a/src/compute/arithmetics/basic/mul.rs +++ b/src/compute/arithmetics/basic/mul.rs @@ -23,8 +23,8 @@ use super::NativeArithmetics; /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::mul; -/// use arrow2::array::Int32Array; +/// use re_arrow2::compute::arithmetics::basic::mul; +/// use re_arrow2::array::Int32Array; /// /// let a = Int32Array::from(&[None, Some(6), None, Some(6)]); /// let b = Int32Array::from(&[Some(5), None, None, Some(6)]); @@ -44,8 +44,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::wrapping_mul; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::wrapping_mul; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([Some(100i8), Some(0x10i8), Some(100i8)]); /// let b = PrimitiveArray::from([Some(0i8), Some(0x10i8), Some(0i8)]); @@ -68,8 +68,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_mul; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::checked_mul; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(100i8), Some(100i8), Some(100i8)]); /// let b = Int8Array::from(&[Some(1i8), Some(100i8), Some(1i8)]); @@ -92,8 +92,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::saturating_mul; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::saturating_mul; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(-100i8)]); /// let b = Int8Array::from(&[Some(100i8)]); @@ -117,8 +117,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::overflowing_mul; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::overflowing_mul; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(1i8), Some(-100i8)]); /// let b = Int8Array::from(&[Some(1i8), Some(100i8)]); @@ -192,8 +192,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::mul_scalar; -/// use arrow2::array::Int32Array; +/// use re_arrow2::compute::arithmetics::basic::mul_scalar; +/// use re_arrow2::array::Int32Array; /// /// let a = Int32Array::from(&[None, Some(6), None, Some(6)]); /// let result = mul_scalar(&a, &2i32); @@ -213,8 +213,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::wrapping_mul_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::wrapping_mul_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[None, Some(0x10)]); /// let result = wrapping_mul_scalar(&a, &0x10); @@ -234,8 +234,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_mul_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::checked_mul_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[None, Some(100), None, Some(100)]); /// let result = checked_mul_scalar(&a, &100i8); @@ -258,8 +258,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::saturating_mul_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::saturating_mul_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(-100i8)]); /// let result = saturating_mul_scalar(&a, &100i8); @@ -283,8 +283,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::overflowing_mul_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::overflowing_mul_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(1i8), Some(100i8)]); /// let (result, overflow) = overflowing_mul_scalar(&a, &100i8); diff --git a/src/compute/arithmetics/basic/pow.rs b/src/compute/arithmetics/basic/pow.rs index 1b67970a030..c3c2467832c 100644 --- a/src/compute/arithmetics/basic/pow.rs +++ b/src/compute/arithmetics/basic/pow.rs @@ -13,8 +13,8 @@ use super::NativeArithmetics; /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::powf_scalar; -/// use arrow2::array::Float32Array; +/// use re_arrow2::compute::arithmetics::basic::powf_scalar; +/// use re_arrow2::array::Float32Array; /// /// let a = Float32Array::from(&[Some(2f32), None]); /// let actual = powf_scalar(&a, 2.0); @@ -34,8 +34,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_powf_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::checked_powf_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(1i8), None, Some(7i8)]); /// let actual = checked_powf_scalar(&a, 8usize); diff --git a/src/compute/arithmetics/basic/rem.rs b/src/compute/arithmetics/basic/rem.rs index 79a6055b8fd..0f584c87110 100644 --- a/src/compute/arithmetics/basic/rem.rs +++ b/src/compute/arithmetics/basic/rem.rs @@ -21,8 +21,8 @@ use super::NativeArithmetics; /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::rem; -/// use arrow2::array::Int32Array; +/// use re_arrow2::compute::arithmetics::basic::rem; +/// use re_arrow2::array::Int32Array; /// /// let a = Int32Array::from(&[Some(10), Some(7)]); /// let b = Int32Array::from(&[Some(5), Some(6)]); @@ -43,8 +43,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_rem; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::checked_rem; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(-100i8), Some(10i8)]); /// let b = Int8Array::from(&[Some(100i8), Some(0i8)]); @@ -84,8 +84,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::rem_scalar; -/// use arrow2::array::Int32Array; +/// use re_arrow2::compute::arithmetics::basic::rem_scalar; +/// use re_arrow2::array::Int32Array; /// /// let a = Int32Array::from(&[None, Some(6), None, Some(7)]); /// let result = rem_scalar(&a, &2i32); @@ -164,8 +164,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_rem_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::checked_rem_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(-100i8)]); /// let result = checked_rem_scalar(&a, &100i8); diff --git a/src/compute/arithmetics/basic/sub.rs b/src/compute/arithmetics/basic/sub.rs index c2c84cae6ee..c369973f4b7 100644 --- a/src/compute/arithmetics/basic/sub.rs +++ b/src/compute/arithmetics/basic/sub.rs @@ -23,8 +23,8 @@ use super::NativeArithmetics; /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::sub; -/// use arrow2::array::Int32Array; +/// use re_arrow2::compute::arithmetics::basic::sub; +/// use re_arrow2::array::Int32Array; /// /// let a = Int32Array::from(&[None, Some(6), None, Some(6)]); /// let b = Int32Array::from(&[Some(5), None, None, Some(6)]); @@ -44,8 +44,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::wrapping_sub; -/// use arrow2::array::PrimitiveArray; +/// use re_arrow2::compute::arithmetics::basic::wrapping_sub; +/// use re_arrow2::array::PrimitiveArray; /// /// let a = PrimitiveArray::from([Some(-100i8), Some(-100i8), Some(100i8)]); /// let b = PrimitiveArray::from([Some(0i8), Some(100i8), Some(0i8)]); @@ -67,8 +67,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_sub; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::checked_sub; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(100i8), Some(-100i8), Some(100i8)]); /// let b = Int8Array::from(&[Some(1i8), Some(100i8), Some(0i8)]); @@ -91,8 +91,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::saturating_sub; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::saturating_sub; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(-100i8)]); /// let b = Int8Array::from(&[Some(100i8)]); @@ -116,8 +116,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::overflowing_sub; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::overflowing_sub; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(1i8), Some(-100i8)]); /// let b = Int8Array::from(&[Some(1i8), Some(100i8)]); @@ -191,8 +191,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::sub_scalar; -/// use arrow2::array::Int32Array; +/// use re_arrow2::compute::arithmetics::basic::sub_scalar; +/// use re_arrow2::array::Int32Array; /// /// let a = Int32Array::from(&[None, Some(6), None, Some(6)]); /// let result = sub_scalar(&a, &1i32); @@ -212,8 +212,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::wrapping_sub_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::wrapping_sub_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[None, Some(-100)]); /// let result = wrapping_sub_scalar(&a, &100i8); @@ -233,8 +233,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::checked_sub_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::checked_sub_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[None, Some(-100), None, Some(-100)]); /// let result = checked_sub_scalar(&a, &100i8); @@ -257,8 +257,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::saturating_sub_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::saturating_sub_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(-100i8)]); /// let result = saturating_sub_scalar(&a, &100i8); @@ -282,8 +282,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::basic::overflowing_sub_scalar; -/// use arrow2::array::Int8Array; +/// use re_arrow2::compute::arithmetics::basic::overflowing_sub_scalar; +/// use re_arrow2::array::Int8Array; /// /// let a = Int8Array::from(&[Some(1i8), Some(-100i8)]); /// let (result, overflow) = overflowing_sub_scalar(&a, &100i8); diff --git a/src/compute/arithmetics/decimal/add.rs b/src/compute/arithmetics/decimal/add.rs index 9f6f529e887..0d9b554c1ba 100644 --- a/src/compute/arithmetics/decimal/add.rs +++ b/src/compute/arithmetics/decimal/add.rs @@ -23,9 +23,9 @@ use super::{adjusted_precision_scale, get_parameters, max_value, number_digits}; /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::add; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::add; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(1i128), Some(1i128), None, Some(2i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(1i128), Some(2i128), None, Some(2i128)]).to(DataType::Decimal(5, 2)); @@ -61,9 +61,9 @@ pub fn add(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> PrimitiveA /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::saturating_add; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::saturating_add; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2)); @@ -105,9 +105,9 @@ pub fn saturating_add( /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::checked_add; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::checked_add; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2)); @@ -169,9 +169,9 @@ impl ArraySaturatingAdd> for PrimitiveArray { /// ``` /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::adaptive_add; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::adaptive_add; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(7, 2)); /// let b = PrimitiveArray::from([Some(11111_111i128)]).to(DataType::Decimal(8, 3)); diff --git a/src/compute/arithmetics/decimal/div.rs b/src/compute/arithmetics/decimal/div.rs index 159c27de2b1..fcfa584b886 100644 --- a/src/compute/arithmetics/decimal/div.rs +++ b/src/compute/arithmetics/decimal/div.rs @@ -23,9 +23,9 @@ use super::{adjusted_precision_scale, get_parameters, max_value, number_digits}; /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::div; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::div; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), Some(2_00i128)]).to(DataType::Decimal(5, 2)); @@ -117,9 +117,9 @@ pub fn div_scalar(lhs: &PrimitiveArray, rhs: &PrimitiveScalar) -> Pr /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::saturating_div; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::saturating_div; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(999_99i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(000_01i128), Some(2_00i128), Some(2_00i128)]).to(DataType::Decimal(5, 2)); @@ -166,9 +166,9 @@ pub fn saturating_div( /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::checked_div; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::checked_div; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(000_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); @@ -228,9 +228,9 @@ impl ArrayCheckedDiv> for PrimitiveArray { /// ``` /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::adaptive_div; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::adaptive_div; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(1000_00i128)]).to(DataType::Decimal(7, 2)); /// let b = PrimitiveArray::from([Some(10_0000i128)]).to(DataType::Decimal(6, 4)); diff --git a/src/compute/arithmetics/decimal/mul.rs b/src/compute/arithmetics/decimal/mul.rs index ac702d2cb3c..b15a8789b01 100644 --- a/src/compute/arithmetics/decimal/mul.rs +++ b/src/compute/arithmetics/decimal/mul.rs @@ -22,9 +22,9 @@ use super::{adjusted_precision_scale, get_parameters, max_value, number_digits}; /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::mul; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::mul; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(1_00i128), Some(1_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); @@ -121,9 +121,9 @@ pub fn mul_scalar(lhs: &PrimitiveArray, rhs: &PrimitiveScalar) -> Pr /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::saturating_mul; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::saturating_mul; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(999_99i128), Some(1_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(10_00i128), Some(2_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); @@ -171,9 +171,9 @@ pub fn saturating_mul( /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::checked_mul; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::checked_mul; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(999_99i128), Some(1_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(10_00i128), Some(2_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); @@ -240,9 +240,9 @@ impl ArraySaturatingMul> for PrimitiveArray { /// ``` /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::adaptive_mul; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::adaptive_mul; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(11111_0i128), Some(1_0i128)]).to(DataType::Decimal(6, 1)); /// let b = PrimitiveArray::from([Some(10_002i128), Some(2_000i128)]).to(DataType::Decimal(5, 3)); diff --git a/src/compute/arithmetics/decimal/sub.rs b/src/compute/arithmetics/decimal/sub.rs index 84afd205433..06bdb98640d 100644 --- a/src/compute/arithmetics/decimal/sub.rs +++ b/src/compute/arithmetics/decimal/sub.rs @@ -20,9 +20,9 @@ use super::{adjusted_precision_scale, get_parameters, max_value, number_digits}; /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::sub; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::sub; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(1i128), Some(1i128), None, Some(2i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(1i128), Some(2i128), None, Some(2i128)]).to(DataType::Decimal(5, 2)); @@ -59,9 +59,9 @@ pub fn sub(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> PrimitiveA /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::saturating_sub; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::saturating_sub; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(-99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2)); @@ -125,9 +125,9 @@ impl ArraySaturatingSub> for PrimitiveArray { /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::checked_sub; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::checked_sub; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(-99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(5, 2)); /// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2)); @@ -168,9 +168,9 @@ pub fn checked_sub(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Pr /// ``` /// # Examples /// ``` -/// use arrow2::compute::arithmetics::decimal::adaptive_sub; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::compute::arithmetics::decimal::adaptive_sub; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::DataType; /// /// let a = PrimitiveArray::from([Some(99_9999i128)]).to(DataType::Decimal(6, 4)); /// let b = PrimitiveArray::from([Some(-00_0001i128)]).to(DataType::Decimal(6, 4)); diff --git a/src/compute/arithmetics/time.rs b/src/compute/arithmetics/time.rs index e049b3820b5..3b5c471a787 100644 --- a/src/compute/arithmetics/time.rs +++ b/src/compute/arithmetics/time.rs @@ -69,9 +69,9 @@ fn create_scale(lhs: &DataType, rhs: &DataType) -> Result { /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::time::add_duration; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::{DataType, TimeUnit}; +/// use re_arrow2::compute::arithmetics::time::add_duration; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::{DataType, TimeUnit}; /// /// let timestamp = PrimitiveArray::from([ /// Some(100000i64), @@ -81,7 +81,7 @@ fn create_scale(lhs: &DataType, rhs: &DataType) -> Result { /// ]) /// .to(DataType::Timestamp( /// TimeUnit::Second, -/// Some("America/New_York".to_string()), +/// Some(std::sync::Arc::new("America/New_york".to_string())), /// )); /// /// let duration = PrimitiveArray::from([Some(10i64), Some(20i64), None, Some(30i64)]) @@ -96,7 +96,7 @@ fn create_scale(lhs: &DataType, rhs: &DataType) -> Result { /// ]) /// .to(DataType::Timestamp( /// TimeUnit::Second, -/// Some("America/New_York".to_string()), +/// Some(std::sync::Arc::new("America/New_york".to_string())), /// )); /// /// assert_eq!(result, expected); @@ -149,9 +149,9 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::time::subtract_duration; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::{DataType, TimeUnit}; +/// use re_arrow2::compute::arithmetics::time::subtract_duration; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::{DataType, TimeUnit}; /// /// let timestamp = PrimitiveArray::from([ /// Some(100000i64), @@ -161,7 +161,7 @@ where /// ]) /// .to(DataType::Timestamp( /// TimeUnit::Second, -/// Some("America/New_York".to_string()), +/// Some(std::sync::Arc::new("America/New_york".to_string())), /// )); /// /// let duration = PrimitiveArray::from([Some(10i64), Some(20i64), None, Some(30i64)]) @@ -176,7 +176,7 @@ where /// ]) /// .to(DataType::Timestamp( /// TimeUnit::Second, -/// Some("America/New_York".to_string()), +/// Some(std::sync::Arc::new("America/New_york".to_string())), /// )); /// /// assert_eq!(result, expected); @@ -228,9 +228,9 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::arithmetics::time::subtract_timestamps; -/// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::{DataType, TimeUnit}; +/// use re_arrow2::compute::arithmetics::time::subtract_timestamps; +/// use re_arrow2::array::PrimitiveArray; +/// use re_arrow2::datatypes::{DataType, TimeUnit}; /// let timestamp_a = PrimitiveArray::from([ /// Some(100_010i64), /// Some(200_020i64), diff --git a/src/compute/boolean.rs b/src/compute/boolean.rs index e34b90c6378..b1bdc8402e8 100644 --- a/src/compute/boolean.rs +++ b/src/compute/boolean.rs @@ -39,8 +39,8 @@ where /// This function panics iff the arrays have different lengths. /// # Examples /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean::and; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean::and; /// /// let a = BooleanArray::from(&[Some(false), Some(true), None]); /// let b = BooleanArray::from(&[Some(true), Some(true), Some(false)]); @@ -81,8 +81,8 @@ pub fn and(lhs: &BooleanArray, rhs: &BooleanArray) -> BooleanArray { /// This function panics iff the arrays have different lengths. /// # Examples /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean::or; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean::or; /// /// let a = BooleanArray::from(vec![Some(false), Some(true), None]); /// let b = BooleanArray::from(vec![Some(true), Some(true), Some(false)]); @@ -122,8 +122,8 @@ pub fn or(lhs: &BooleanArray, rhs: &BooleanArray) -> BooleanArray { /// null. /// # Example /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean::not; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean::not; /// /// let a = BooleanArray::from(vec![Some(false), Some(true), None]); /// let not_a = not(&a); @@ -138,8 +138,8 @@ pub fn not(array: &BooleanArray) -> BooleanArray { /// Returns a non-null [`BooleanArray`] with whether each value of the array is null. /// # Example /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean::is_null; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean::is_null; /// # fn main() { /// let a = BooleanArray::from(vec![Some(false), Some(true), None]); /// let a_is_null = is_null(&a); @@ -160,8 +160,8 @@ pub fn is_null(input: &dyn Array) -> BooleanArray { /// Returns a non-null [`BooleanArray`] with whether each value of the array is not null. /// # Example /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean::is_not_null; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean::is_not_null; /// /// let a = BooleanArray::from(&vec![Some(false), Some(true), None]); /// let a_is_not_null = is_not_null(&a); @@ -183,9 +183,9 @@ pub fn is_not_null(input: &dyn Array) -> BooleanArray { /// is null then the result is also null. /// # Example /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean::and_scalar; -/// use arrow2::scalar::BooleanScalar; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean::and_scalar; +/// use re_arrow2::scalar::BooleanScalar; /// /// let array = BooleanArray::from_slice(&[false, false, true, true]); /// let scalar = BooleanScalar::new(Some(true)); @@ -208,9 +208,9 @@ pub fn and_scalar(array: &BooleanArray, scalar: &BooleanScalar) -> BooleanArray /// is null then the result is also null. /// # Example /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean::or_scalar; -/// use arrow2::scalar::BooleanScalar; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean::or_scalar; +/// use re_arrow2::scalar::BooleanScalar; /// # fn main() { /// let array = BooleanArray::from_slice(&[false, false, true, true]); /// let scalar = BooleanScalar::new(Some(true)); @@ -237,8 +237,8 @@ pub fn or_scalar(array: &BooleanArray, scalar: &BooleanScalar) -> BooleanArray { /// # Example /// /// ``` -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean::any; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean::any; /// /// let a = BooleanArray::from(&[Some(true), Some(false)]); /// let b = BooleanArray::from(&[Some(false), Some(false)]); @@ -266,8 +266,8 @@ pub fn any(array: &BooleanArray) -> bool { /// # Example /// /// ``` -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean::all; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean::all; /// /// let a = BooleanArray::from(&[Some(true), Some(true)]); /// let b = BooleanArray::from(&[Some(false), Some(true)]); diff --git a/src/compute/boolean_kleene.rs b/src/compute/boolean_kleene.rs index b19efeaa78d..a9d1dc0fa3c 100644 --- a/src/compute/boolean_kleene.rs +++ b/src/compute/boolean_kleene.rs @@ -12,8 +12,8 @@ use crate::{ /// # Example /// /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean_kleene::or; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean_kleene::or; /// /// let a = BooleanArray::from(&[Some(true), Some(false), None]); /// let b = BooleanArray::from(&[None, None, None]); @@ -95,8 +95,8 @@ pub fn or(lhs: &BooleanArray, rhs: &BooleanArray) -> BooleanArray { /// # Example /// /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean_kleene::and; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean_kleene::and; /// /// let a = BooleanArray::from(&[Some(true), Some(false), None]); /// let b = BooleanArray::from(&[None, None, None]); @@ -175,9 +175,9 @@ pub fn and(lhs: &BooleanArray, rhs: &BooleanArray) -> BooleanArray { /// # Example /// /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::scalar::BooleanScalar; -/// use arrow2::compute::boolean_kleene::or_scalar; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::scalar::BooleanScalar; +/// use re_arrow2::compute::boolean_kleene::or_scalar; /// /// let array = BooleanArray::from(&[Some(true), Some(false), None]); /// let scalar = BooleanScalar::new(Some(false)); @@ -207,9 +207,9 @@ pub fn or_scalar(array: &BooleanArray, scalar: &BooleanScalar) -> BooleanArray { /// # Example /// /// ```rust -/// use arrow2::array::BooleanArray; -/// use arrow2::scalar::BooleanScalar; -/// use arrow2::compute::boolean_kleene::and_scalar; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::scalar::BooleanScalar; +/// use re_arrow2::compute::boolean_kleene::and_scalar; /// /// let array = BooleanArray::from(&[Some(true), Some(false), None]); /// let scalar = BooleanScalar::new(None); @@ -242,8 +242,8 @@ pub fn and_scalar(array: &BooleanArray, scalar: &BooleanScalar) -> BooleanArray /// # Example /// /// ``` -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean_kleene::any; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean_kleene::any; /// /// let a = BooleanArray::from(&[Some(true), Some(false)]); /// let b = BooleanArray::from(&[Some(false), Some(false)]); @@ -276,8 +276,8 @@ pub fn any(array: &BooleanArray) -> Option { /// # Example /// /// ``` -/// use arrow2::array::BooleanArray; -/// use arrow2::compute::boolean_kleene::all; +/// use re_arrow2::array::BooleanArray; +/// use re_arrow2::compute::boolean_kleene::all; /// /// let a = BooleanArray::from(&[Some(true), Some(true)]); /// let b = BooleanArray::from(&[Some(false), Some(true)]); diff --git a/src/compute/cast/dictionary_to.rs b/src/compute/cast/dictionary_to.rs index 101669f6442..39533bdb065 100644 --- a/src/compute/cast/dictionary_to.rs +++ b/src/compute/cast/dictionary_to.rs @@ -89,7 +89,7 @@ where } else { let data_type = DataType::Dictionary( K2::KEY_TYPE, - Box::new(values.data_type().clone()), + std::sync::Arc::new(values.data_type().clone()), is_ordered, ); // Safety: this is safe because given a type `T` that fits in a `usize`, casting it to type `P` either overflows or also fits in a `usize` @@ -116,7 +116,7 @@ where } else { let data_type = DataType::Dictionary( K2::KEY_TYPE, - Box::new(values.data_type().clone()), + std::sync::Arc::new(values.data_type().clone()), is_ordered, ); // some of the values may not fit in `usize` and thus this needs to be checked diff --git a/src/compute/cast/mod.rs b/src/compute/cast/mod.rs index 688291dd12b..2a037a5803a 100644 --- a/src/compute/cast/mod.rs +++ b/src/compute/cast/mod.rs @@ -409,7 +409,7 @@ fn cast_list_to_fixed_size_list( ); let new_values = cast(sliced_values.as_ref(), inner.data_type(), options)?; Ok(FixedSizeListArray::new( - DataType::FixedSizeList(Box::new(inner.clone()), size), + DataType::FixedSizeList(std::sync::Arc::new(inner.clone()), size), new_values, list.validity().cloned(), )) diff --git a/src/compute/cast/primitive_to.rs b/src/compute/cast/primitive_to.rs index 110288817a7..d34acbdbd01 100644 --- a/src/compute/cast/primitive_to.rs +++ b/src/compute/cast/primitive_to.rs @@ -1,4 +1,5 @@ use std::hash::Hash; +use std::sync::Arc; use num_traits::{AsPrimitive, Float, ToPrimitive}; @@ -406,7 +407,7 @@ pub fn timestamp_to_timestamp( from: &PrimitiveArray, from_unit: TimeUnit, to_unit: TimeUnit, - tz: &Option, + tz: &Option>, ) -> PrimitiveArray { let from_size = time_unit_multiple(from_unit); let to_size = time_unit_multiple(to_unit); diff --git a/src/compute/cast/utf8_to.rs b/src/compute/cast/utf8_to.rs index 6ee38588696..610d1c11e7f 100644 --- a/src/compute/cast/utf8_to.rs +++ b/src/compute/cast/utf8_to.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use chrono::Datelike; use crate::{ @@ -127,7 +129,7 @@ pub fn utf8_to_naive_timestamp_ns(from: &Utf8Array) -> PrimitiveAr pub(super) fn utf8_to_timestamp_ns_dyn( from: &dyn Array, - timezone: String, + timezone: Arc, ) -> Result> { let from = from.as_any().downcast_ref().unwrap(); utf8_to_timestamp_ns::(from, timezone) @@ -138,7 +140,7 @@ pub(super) fn utf8_to_timestamp_ns_dyn( /// [`crate::temporal_conversions::utf8_to_timestamp_ns`] applied for RFC3339 formatting pub fn utf8_to_timestamp_ns( from: &Utf8Array, - timezone: String, + timezone: Arc, ) -> Result> { utf8_to_timestamp_ns_(from, RFC3339, timezone) } diff --git a/src/compute/comparison/mod.rs b/src/compute/comparison/mod.rs index b364ed88222..acfd44b619c 100644 --- a/src/compute/comparison/mod.rs +++ b/src/compute/comparison/mod.rs @@ -14,8 +14,8 @@ //! //! Compare two [`PrimitiveArray`]s: //! ``` -//! use arrow2::array::{BooleanArray, PrimitiveArray}; -//! use arrow2::compute::comparison::primitive::gt; +//! use re_arrow2::array::{BooleanArray, PrimitiveArray}; +//! use re_arrow2::compute::comparison::primitive::gt; //! //! let array1 = PrimitiveArray::::from([Some(1), None, Some(2)]); //! let array2 = PrimitiveArray::::from([Some(1), Some(3), Some(1)]); @@ -25,8 +25,8 @@ //! //! Compare two dynamically-typed [`Array`]s (trait objects): //! ``` -//! use arrow2::array::{Array, BooleanArray, PrimitiveArray}; -//! use arrow2::compute::comparison::eq; +//! use re_arrow2::array::{Array, BooleanArray, PrimitiveArray}; +//! use re_arrow2::compute::comparison::eq; //! //! let array1: &dyn Array = &PrimitiveArray::::from(&[Some(10.0), None, Some(20.0)]); //! let array2: &dyn Array = &PrimitiveArray::::from(&[Some(10.0), None, Some(10.0)]); @@ -36,8 +36,8 @@ //! //! Compare (not equal) a [`Utf8Array`] to a word: //! ``` -//! use arrow2::array::{BooleanArray, Utf8Array}; -//! use arrow2::compute::comparison::utf8::neq_scalar; +//! use re_arrow2::array::{BooleanArray, Utf8Array}; +//! use re_arrow2::compute::comparison::utf8::neq_scalar; //! //! let array = Utf8Array::::from([Some("compute"), None, Some("compare")]); //! let result = neq_scalar(&array, "compare"); diff --git a/src/compute/comparison/simd/packed.rs b/src/compute/comparison/simd/packed.rs index 937e3bb81a2..731227b5caa 100644 --- a/src/compute/comparison/simd/packed.rs +++ b/src/compute/comparison/simd/packed.rs @@ -1,5 +1,5 @@ use std::convert::TryInto; -use std::simd::{SimdPartialEq, SimdPartialOrd, ToBitMask}; +use std::simd::prelude::*; use crate::types::simd::*; use crate::types::{days_ms, f16, i256, months_days_ns}; @@ -29,34 +29,40 @@ macro_rules! simd8 { impl Simd8PartialEq for $md { #[inline] fn eq(self, other: Self) -> u8 { - self.simd_eq(other).to_bitmask() + use std::simd::prelude::*; + self.simd_eq(other).to_bitmask().try_into().unwrap() } #[inline] fn neq(self, other: Self) -> u8 { - self.simd_ne(other).to_bitmask() + use std::simd::prelude::*; + self.simd_ne(other).to_bitmask().try_into().unwrap() } } impl Simd8PartialOrd for $md { #[inline] fn lt_eq(self, other: Self) -> u8 { - self.simd_le(other).to_bitmask() + use std::simd::prelude::*; + self.simd_le(other).to_bitmask().try_into().unwrap() } #[inline] fn lt(self, other: Self) -> u8 { - self.simd_lt(other).to_bitmask() + use std::simd::prelude::*; + self.simd_lt(other).to_bitmask().try_into().unwrap() } #[inline] fn gt_eq(self, other: Self) -> u8 { - self.simd_ge(other).to_bitmask() + use std::simd::prelude::*; + self.simd_ge(other).to_bitmask().try_into().unwrap() } #[inline] fn gt(self, other: Self) -> u8 { - self.simd_gt(other).to_bitmask() + use std::simd::prelude::*; + self.simd_gt(other).to_bitmask().try_into().unwrap() } } }; diff --git a/src/compute/concatenate.rs b/src/compute/concatenate.rs index 1cab5767164..46ea9a08cb0 100644 --- a/src/compute/concatenate.rs +++ b/src/compute/concatenate.rs @@ -3,8 +3,8 @@ //! Example: //! //! ``` -//! use arrow2::array::Utf8Array; -//! use arrow2::compute::concatenate::concatenate; +//! use re_arrow2::array::Utf8Array; +//! use re_arrow2::compute::concatenate::concatenate; //! //! let arr = concatenate(&[ //! &Utf8Array::::from_slice(["hello", "world"]), diff --git a/src/compute/filter.rs b/src/compute/filter.rs index 7ba260e702f..36171aa1f9c 100644 --- a/src/compute/filter.rs +++ b/src/compute/filter.rs @@ -258,9 +258,9 @@ pub fn build_filter(filter: &BooleanArray) -> Result { /// /// # Example /// ```rust -/// # use arrow2::array::{Int32Array, PrimitiveArray, BooleanArray}; -/// # use arrow2::error::Result; -/// # use arrow2::compute::filter::filter; +/// # use re_arrow2::array::{Int32Array, PrimitiveArray, BooleanArray}; +/// # use re_arrow2::error::Result; +/// # use re_arrow2::compute::filter::filter; /// # fn main() -> Result<()> { /// let array = PrimitiveArray::from_slice([5, 6, 7, 8, 9]); /// let filter_array = BooleanArray::from_slice(&vec![true, false, false, true, false]); diff --git a/src/compute/hash.rs b/src/compute/hash.rs index 5f914917507..d8760a2ffe6 100644 --- a/src/compute/hash.rs +++ b/src/compute/hash.rs @@ -125,8 +125,8 @@ pub fn hash(array: &dyn Array) -> Result> { /// /// # Examples /// ``` -/// use arrow2::compute::hash::can_hash; -/// use arrow2::datatypes::{DataType}; +/// use re_arrow2::compute::hash::can_hash; +/// use re_arrow2::datatypes::{DataType}; /// /// let data_type = DataType::Int8; /// assert_eq!(can_hash(&data_type), true); diff --git a/src/compute/if_then_else.rs b/src/compute/if_then_else.rs index 86c46b29d04..630325a8c53 100644 --- a/src/compute/if_then_else.rs +++ b/src/compute/if_then_else.rs @@ -7,9 +7,9 @@ use crate::error::{Error, Result}; /// Returns `None` if the predicate is `None`. /// # Example /// ```rust -/// # use arrow2::error::Result; -/// use arrow2::compute::if_then_else::if_then_else; -/// use arrow2::array::{Int32Array, BooleanArray}; +/// # use re_arrow2::error::Result; +/// use re_arrow2::compute::if_then_else::if_then_else; +/// use re_arrow2::array::{Int32Array, BooleanArray}; /// /// # fn main() -> Result<()> { /// let lhs = Int32Array::from_slice(&[1, 2, 3]); diff --git a/src/compute/length.rs b/src/compute/length.rs index 9dc7e0b1c12..24b0e7e3e95 100644 --- a/src/compute/length.rs +++ b/src/compute/length.rs @@ -68,8 +68,8 @@ pub fn length(array: &dyn Array) -> Result> { /// /// # Examples /// ``` -/// use arrow2::compute::length::can_length; -/// use arrow2::datatypes::{DataType}; +/// use re_arrow2::compute::length::can_length; +/// use re_arrow2::datatypes::{DataType}; /// /// let data_type = DataType::Utf8; /// assert_eq!(can_length(&data_type), true); diff --git a/src/compute/like.rs b/src/compute/like.rs index d52e9c5e9ff..d8c4f84ddb6 100644 --- a/src/compute/like.rs +++ b/src/compute/like.rs @@ -109,8 +109,8 @@ fn a_like_utf8 bool>( /// * any of the patterns is not valid /// # Example /// ``` -/// use arrow2::array::{Utf8Array, BooleanArray}; -/// use arrow2::compute::like::like_utf8; +/// use re_arrow2::array::{Utf8Array, BooleanArray}; +/// use re_arrow2::compute::like::like_utf8; /// /// let strings = Utf8Array::::from_slice(&["Arrow", "Arrow", "Arrow", "Arrow", "Ar"]); /// let patterns = Utf8Array::::from_slice(&["A%", "B%", "%r_ow", "A_", "A_"]); @@ -190,8 +190,8 @@ fn a_like_utf8_scalar bool>( /// * any of the patterns is not valid /// # Example /// ``` -/// use arrow2::array::{Utf8Array, BooleanArray}; -/// use arrow2::compute::like::like_utf8_scalar; +/// use re_arrow2::array::{Utf8Array, BooleanArray}; +/// use re_arrow2::compute::like::like_utf8_scalar; /// /// let array = Utf8Array::::from_slice(&["Arrow", "Arrow", "Arrow", "BA"]); /// @@ -267,8 +267,8 @@ fn a_like_binary bool>( /// * any of the patterns is not valid /// # Example /// ``` -/// use arrow2::array::{BinaryArray, BooleanArray}; -/// use arrow2::compute::like::like_binary; +/// use re_arrow2::array::{BinaryArray, BooleanArray}; +/// use re_arrow2::compute::like::like_binary; /// /// let strings = BinaryArray::::from_slice(&["Arrow", "Arrow", "Arrow", "Arrow", "Ar"]); /// let patterns = BinaryArray::::from_slice(&["A%", "B%", "%r_ow", "A_", "A_"]); @@ -341,8 +341,8 @@ fn a_like_binary_scalar bool>( /// * any of the patterns is not valid /// # Example /// ``` -/// use arrow2::array::{BinaryArray, BooleanArray}; -/// use arrow2::compute::like::like_binary_scalar; +/// use re_arrow2::array::{BinaryArray, BooleanArray}; +/// use re_arrow2::compute::like::like_binary_scalar; /// /// let array = BinaryArray::::from_slice(&["Arrow", "Arrow", "Arrow", "BA"]); /// diff --git a/src/compute/merge_sort/mod.rs b/src/compute/merge_sort/mod.rs index f57b09bb4a2..8038e3f5c36 100644 --- a/src/compute/merge_sort/mod.rs +++ b/src/compute/merge_sort/mod.rs @@ -127,9 +127,9 @@ pub fn take_arrays>( /// * the arrays have a [`crate::datatypes::DataType`] that has no order relationship /// # Example /// ```rust -/// use arrow2::array::Int32Array; -/// use arrow2::compute::merge_sort::{merge_sort, SortOptions}; -/// # use arrow2::error::Result; +/// use re_arrow2::array::Int32Array; +/// use re_arrow2::compute::merge_sort::{merge_sort, SortOptions}; +/// # use re_arrow2::error::Result; /// # fn main() -> Result<()> { /// let a = Int32Array::from_slice(&[2, 4, 6]); /// let b = Int32Array::from_slice(&[0, 1, 3]); @@ -166,9 +166,9 @@ pub fn merge_sort( /// In other words, `pairs.i.0[j]` must be an array coming from a batch of equal len arrays. /// # Example /// ```rust -/// use arrow2::array::Int32Array; -/// use arrow2::compute::merge_sort::{slices, SortOptions}; -/// # use arrow2::error::Result; +/// use re_arrow2::array::Int32Array; +/// use re_arrow2::compute::merge_sort::{slices, SortOptions}; +/// # use re_arrow2::error::Result; /// # fn main() -> Result<()> { /// let a = Int32Array::from_slice(&[2, 4, 6]); /// let b = Int32Array::from_slice(&[0, 1, 3]); diff --git a/src/compute/nullif.rs b/src/compute/nullif.rs index b93e518da7f..4ef377c7705 100644 --- a/src/compute/nullif.rs +++ b/src/compute/nullif.rs @@ -19,9 +19,9 @@ use super::utils::combine_validities; /// * The arguments do not have the same length /// # Example /// ```rust -/// # use arrow2::array::Int32Array; -/// # use arrow2::datatypes::DataType; -/// # use arrow2::compute::nullif::primitive_nullif; +/// # use re_arrow2::array::Int32Array; +/// # use re_arrow2::datatypes::DataType; +/// # use re_arrow2::compute::nullif::primitive_nullif; /// # fn main() { /// let lhs = Int32Array::from(&[None, None, Some(1), Some(1), Some(1)]); /// let rhs = Int32Array::from(&[None, Some(1), None, Some(1), Some(0)]); @@ -53,9 +53,9 @@ where /// * The arguments do not have the same logical type /// # Example /// ```rust -/// # use arrow2::array::Int32Array; -/// # use arrow2::datatypes::DataType; -/// # use arrow2::compute::nullif::primitive_nullif_scalar; +/// # use re_arrow2::array::Int32Array; +/// # use re_arrow2::datatypes::DataType; +/// # use re_arrow2::compute::nullif::primitive_nullif_scalar; /// # fn main() { /// let lhs = Int32Array::from(&[None, None, Some(1), Some(0), Some(1)]); /// let result = primitive_nullif_scalar(&lhs, 0); @@ -89,9 +89,9 @@ where /// * The physical type is not supported for this operation (use [`can_nullif`] to check) /// # Example /// ```rust -/// # use arrow2::array::Int32Array; -/// # use arrow2::datatypes::DataType; -/// # use arrow2::compute::nullif::nullif; +/// # use re_arrow2::array::Int32Array; +/// # use re_arrow2::datatypes::DataType; +/// # use re_arrow2::compute::nullif::nullif; /// # fn main() { /// let lhs = Int32Array::from(&[None, None, Some(1), Some(1), Some(1)]); /// let rhs = Int32Array::from(&[None, Some(1), None, Some(1), Some(0)]); @@ -127,10 +127,10 @@ pub fn nullif(lhs: &dyn Array, rhs: &dyn Array) -> Box { /// * The physical type is not supported for this operation (use [`can_nullif`] to check) /// # Example /// ```rust -/// # use arrow2::array::Int32Array; -/// # use arrow2::scalar::PrimitiveScalar; -/// # use arrow2::datatypes::DataType; -/// # use arrow2::compute::nullif::nullif_scalar; +/// # use re_arrow2::array::Int32Array; +/// # use re_arrow2::scalar::PrimitiveScalar; +/// # use re_arrow2::datatypes::DataType; +/// # use re_arrow2::compute::nullif::nullif_scalar; /// # fn main() { /// let lhs = Int32Array::from(&[None, None, Some(1), Some(0), Some(1)]); /// let rhs = PrimitiveScalar::::from(Some(0)); diff --git a/src/compute/regex_match.rs b/src/compute/regex_match.rs index f0cd2acadec..fcddab6d556 100644 --- a/src/compute/regex_match.rs +++ b/src/compute/regex_match.rs @@ -50,8 +50,8 @@ pub fn regex_match(values: &Utf8Array, regex: &Utf8Array) -> Re /// Regex matches /// # Example /// ``` -/// use arrow2::array::{Utf8Array, BooleanArray}; -/// use arrow2::compute::regex_match::regex_match_scalar; +/// use re_arrow2::array::{Utf8Array, BooleanArray}; +/// use re_arrow2::compute::regex_match::regex_match_scalar; /// /// let strings = Utf8Array::::from_slice(&vec!["ArAow", "A_B", "AAA"]); /// diff --git a/src/compute/sort/lex_sort.rs b/src/compute/sort/lex_sort.rs index c598cfd3abd..fc5a9abb4d1 100644 --- a/src/compute/sort/lex_sort.rs +++ b/src/compute/sort/lex_sort.rs @@ -32,9 +32,9 @@ pub struct SortColumn<'a> { /// /// ``` /// use std::convert::From; -/// use arrow2::array::{Utf8Array, Int64Array, Array}; -/// use arrow2::compute::sort::{SortColumn, SortOptions, lexsort}; -/// use arrow2::datatypes::DataType; +/// use re_arrow2::array::{Utf8Array, Int64Array, Array}; +/// use re_arrow2::compute::sort::{SortColumn, SortOptions, lexsort}; +/// use re_arrow2::datatypes::DataType; /// /// let int64 = Int64Array::from(&[None, Some(-2), Some(89), Some(-64), Some(101)]); /// let utf8 = Utf8Array::::from(&vec![Some("hello"), Some("world"), Some(","), Some("foobar"), Some("!")]); diff --git a/src/compute/sort/mod.rs b/src/compute/sort/mod.rs index be85c9f6cbd..09bbbcb94e8 100644 --- a/src/compute/sort/mod.rs +++ b/src/compute/sort/mod.rs @@ -230,8 +230,8 @@ fn sort_dict( /// /// # Examples /// ``` -/// use arrow2::compute::sort::can_sort; -/// use arrow2::datatypes::{DataType}; +/// use re_arrow2::compute::sort::can_sort; +/// use re_arrow2::datatypes::{DataType}; /// /// let data_type = DataType::Int8; /// assert_eq!(can_sort(&data_type), true); @@ -345,7 +345,7 @@ where let mut values = if options.nulls_first { null_indices.into_iter().chain(values).collect::>() } else { - values.chain(null_indices.into_iter()).collect::>() + values.chain(null_indices).collect::>() }; values.truncate(limit.unwrap_or(values.len())); diff --git a/src/compute/sort/row/interner.rs b/src/compute/sort/row/interner.rs index 77c53a06843..a7ef90bd200 100644 --- a/src/compute/sort/row/interner.rs +++ b/src/compute/sort/row/interner.rs @@ -415,7 +415,7 @@ mod tests { #[test] fn test_intern_duplicates() { // Unsorted with duplicates - let values = vec![0_u8, 1, 8, 4, 1, 0]; + let values = [0_u8, 1, 8, 4, 1, 0]; let mut interner = OrderPreservingInterner::default(); let interned = interner.intern(values.iter().map(std::slice::from_ref).map(Some)); diff --git a/src/compute/substring.rs b/src/compute/substring.rs index 2919b3037b9..376b7af6aa6 100644 --- a/src/compute/substring.rs +++ b/src/compute/substring.rs @@ -171,8 +171,8 @@ pub fn substring(array: &dyn Array, start: i64, length: &Option) -> Result< /// /// # Examples /// ``` -/// use arrow2::compute::substring::can_substring; -/// use arrow2::datatypes::{DataType}; +/// use re_arrow2::compute::substring::can_substring; +/// use re_arrow2::datatypes::{DataType}; /// /// let data_type = DataType::Utf8; /// assert_eq!(can_substring(&data_type), true); diff --git a/src/compute/take/mod.rs b/src/compute/take/mod.rs index 3acf47dc7a1..73d4d67faa6 100644 --- a/src/compute/take/mod.rs +++ b/src/compute/take/mod.rs @@ -103,8 +103,8 @@ pub fn take(values: &dyn Array, indices: &PrimitiveArray) -> Result /// /// # Examples /// ``` -/// use arrow2::compute::take::can_take; -/// use arrow2::datatypes::{DataType}; +/// use re_arrow2::compute::take::can_take; +/// use re_arrow2::datatypes::{DataType}; /// /// let data_type = DataType::Int8; /// assert_eq!(can_take(&data_type), true); diff --git a/src/compute/temporal.rs b/src/compute/temporal.rs index 60e573da4ba..410ae0eeb1a 100644 --- a/src/compute/temporal.rs +++ b/src/compute/temporal.rs @@ -331,8 +331,8 @@ where /// /// # Examples /// ``` -/// use arrow2::compute::temporal::can_year; -/// use arrow2::datatypes::{DataType}; +/// use re_arrow2::compute::temporal::can_year; +/// use re_arrow2::datatypes::{DataType}; /// /// assert_eq!(can_year(&DataType::Date32), true); /// assert_eq!(can_year(&DataType::Int8), false); @@ -372,8 +372,8 @@ fn can_date(data_type: &DataType) -> bool { /// /// # Examples /// ``` -/// use arrow2::compute::temporal::can_hour; -/// use arrow2::datatypes::{DataType, TimeUnit}; +/// use re_arrow2::compute::temporal::can_hour; +/// use re_arrow2::datatypes::{DataType, TimeUnit}; /// /// assert_eq!(can_hour(&DataType::Time32(TimeUnit::Second)), true); /// assert_eq!(can_hour(&DataType::Int8), false); diff --git a/src/compute/utf8.rs b/src/compute/utf8.rs index 2e480016ef5..6ffb71b438c 100644 --- a/src/compute/utf8.rs +++ b/src/compute/utf8.rs @@ -44,8 +44,8 @@ pub fn upper(array: &dyn Array) -> Result> { /// /// # Examples /// ``` -/// use arrow2::compute::utf8::can_upper; -/// use arrow2::datatypes::{DataType}; +/// use re_arrow2::compute::utf8::can_upper; +/// use re_arrow2::datatypes::{DataType}; /// /// let data_type = DataType::Utf8; /// assert_eq!(can_upper(&data_type), true); @@ -86,8 +86,8 @@ pub fn lower(array: &dyn Array) -> Result> { /// /// # Examples /// ``` -/// use arrow2::compute::utf8::can_lower; -/// use arrow2::datatypes::{DataType}; +/// use re_arrow2::compute::utf8::can_lower; +/// use re_arrow2::datatypes::{DataType}; /// /// let data_type = DataType::Utf8; /// assert_eq!(can_lower(&data_type), true); diff --git a/src/compute/window.rs b/src/compute/window.rs index b9200fe3a37..2ebb8fb6265 100644 --- a/src/compute/window.rs +++ b/src/compute/window.rs @@ -30,8 +30,8 @@ use crate::{ /// a negative value shifts the array to the left. /// # Examples /// ``` -/// use arrow2::array::Int32Array; -/// use arrow2::compute::window::shift; +/// use re_arrow2::array::Int32Array; +/// use re_arrow2::compute::window::shift; /// /// let array = Int32Array::from(&[Some(1), None, Some(3)]); /// let result = shift(&array, -1).unwrap(); diff --git a/src/datatypes/mod.rs b/src/datatypes/mod.rs index 626b292ad81..17bc65ae04c 100644 --- a/src/datatypes/mod.rs +++ b/src/datatypes/mod.rs @@ -20,6 +20,47 @@ pub type Metadata = BTreeMap; /// typedef fpr [Option<(String, Option)>] descr pub(crate) type Extension = Option<(String, Option)>; +/// An extension trait to polyfill [`Arc::unwrap_or_clone`] from the nightly stdlib. +pub trait ArcExt { + /// If we have the only reference to `T` then unwrap it. Otherwise, clone `T` and return the + /// clone. + /// + /// Assuming `arc_t` is of type `Arc`, this function is functionally equivalent to + /// `(*arc_t).clone()`, but will avoid cloning the inner value where possible. + /// + /// # Examples + /// + /// ``` + /// # use std::{ptr, sync::Arc}; + /// # use re_arrow2::datatype::ArcExt; + /// let inner = String::from("test"); + /// let ptr = inner.as_ptr(); + /// + /// let arc = Arc::new(inner); + /// let inner = Arc::unwrap_or_clone(arc); + /// // The inner value was not cloned + /// assert!(ptr::eq(ptr, inner.as_ptr())); + /// + /// let arc = Arc::new(inner); + /// let arc2 = arc.clone(); + /// let inner = Arc::unwrap_or_clone(arc); + /// // Because there were 2 references, we had to clone the inner value. + /// assert!(!ptr::eq(ptr, inner.as_ptr())); + /// // `arc2` is the last reference, so when we unwrap it we get back + /// // the original `String`. + /// let inner = Arc::unwrap_or_clone(arc2); + /// assert!(ptr::eq(ptr, inner.as_ptr())); + /// ``` + fn unwrap_or_clone_polyfill(this: Self) -> T; +} + +impl ArcExt for Arc { + #[inline] + fn unwrap_or_clone_polyfill(this: Self) -> T { + Arc::try_unwrap(this).unwrap_or_else(|arc| (*arc).clone()) + } +} + /// The set of supported logical types in this crate. /// /// Each variant uniquely identifies a logical type, which define specific semantics to the data @@ -70,7 +111,7 @@ pub enum DataType { /// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30 /// When the timezone is not specified, the timestamp is considered to have no timezone /// and is represented _as is_ - Timestamp(TimeUnit, Option), + Timestamp(TimeUnit, Option>), /// An [`i32`] representing the elapsed time since UNIX epoch (1970-01-01) /// in days. Date32, @@ -100,16 +141,16 @@ pub enum DataType { /// A variable-length UTF-8 encoded string whose offsets are represented as [`i64`]. LargeUtf8, /// A list of some logical data type whose offsets are represented as [`i32`]. - List(Box), + List(Arc), /// A list of some logical data type with a fixed number of elements. - FixedSizeList(Box, usize), + FixedSizeList(Arc, usize), /// A list of some logical data type whose offsets are represented as [`i64`]. - LargeList(Box), + LargeList(Arc), /// A nested [`DataType`] with a given number of [`Field`]s. - Struct(Vec), + Struct(Arc>), /// A nested datatype that can represent slots of differing types. /// Third argument represents mode - Union(Vec, Option>, UnionMode), + Union(Arc>, Option>>, UnionMode), /// A nested type that is represented as /// /// List> @@ -135,7 +176,7 @@ pub enum DataType { /// The metadata is structured so that Arrow systems without special handling /// for Map can make Map an alias for List. The "layout" attribute for the Map /// field must have the same contents as a List. - Map(Box, bool), + Map(Arc, bool), /// A dictionary encoded array (`key_type`, `value_type`), where /// each array element is an index of `key_type` into an /// associated dictionary of `value_type`. @@ -148,7 +189,7 @@ pub enum DataType { /// arrays or a limited set of primitive types as integers. /// /// The `bool` value indicates the `Dictionary` is sorted if set to `true`. - Dictionary(IntegerType, Box, bool), + Dictionary(IntegerType, Arc, bool), /// Decimal value with precision and scale /// precision is the number of digits in the number and /// scale is the number of decimal places. @@ -157,7 +198,7 @@ pub enum DataType { /// Decimal backed by 256 bits Decimal256(usize, usize), /// Extension type. - Extension(String, Box, Option), + Extension(String, Arc, Option>), } #[cfg(feature = "arrow")] @@ -180,7 +221,10 @@ impl From for arrow_schema::DataType { DataType::Float16 => Self::Float16, DataType::Float32 => Self::Float32, DataType::Float64 => Self::Float64, - DataType::Timestamp(unit, tz) => Self::Timestamp(unit.into(), tz.map(Into::into)), + DataType::Timestamp(unit, tz) => Self::Timestamp( + unit.into(), + tz.map(Arc::unwrap_or_clone_polyfill).map(Into::into), + ), DataType::Date32 => Self::Date32, DataType::Date64 => Self::Date64, DataType::Time32(unit) => Self::Time32(unit.into()), @@ -192,30 +236,45 @@ impl From for arrow_schema::DataType { DataType::LargeBinary => Self::LargeBinary, DataType::Utf8 => Self::Utf8, DataType::LargeUtf8 => Self::LargeUtf8, - DataType::List(f) => Self::List(Arc::new((*f).into())), + DataType::List(f) => Self::List(Arc::new(Arc::unwrap_or_clone_polyfill(f).into())), DataType::FixedSizeList(f, size) => { - Self::FixedSizeList(Arc::new((*f).into()), size as _) + Self::FixedSizeList(Arc::new(Arc::unwrap_or_clone_polyfill(f).into()), size as _) + } + DataType::LargeList(f) => { + Self::LargeList(Arc::new(Arc::unwrap_or_clone_polyfill(f).into())) } - DataType::LargeList(f) => Self::LargeList(Arc::new((*f).into())), - DataType::Struct(f) => Self::Struct(f.into_iter().map(ArrowField::from).collect()), + DataType::Struct(f) => Self::Struct( + Arc::unwrap_or_clone_polyfill(f) + .into_iter() + .map(ArrowField::from) + .collect(), + ), DataType::Union(fields, Some(ids), mode) => { - let ids = ids.into_iter().map(|x| x as _); - let fields = fields.into_iter().map(ArrowField::from); + let ids = Arc::unwrap_or_clone_polyfill(ids) + .into_iter() + .map(|x| x as _); + let fields = Arc::unwrap_or_clone_polyfill(fields) + .into_iter() + .map(ArrowField::from); Self::Union(UnionFields::new(ids, fields), mode.into()) } DataType::Union(fields, None, mode) => { let ids = 0..fields.len() as i8; - let fields = fields.into_iter().map(ArrowField::from); + let fields = Arc::unwrap_or_clone_polyfill(fields) + .into_iter() + .map(ArrowField::from); Self::Union(UnionFields::new(ids, fields), mode.into()) } - DataType::Map(f, ordered) => Self::Map(Arc::new((*f).into()), ordered), + DataType::Map(f, ordered) => { + Self::Map(Arc::new(Arc::unwrap_or_clone_polyfill(f).into()), ordered) + } DataType::Dictionary(key, value, _) => Self::Dictionary( Box::new(DataType::from(key).into()), - Box::new((*value).into()), + Box::new(Arc::unwrap_or_clone_polyfill(value).into()), ), DataType::Decimal(precision, scale) => Self::Decimal128(precision as _, scale as _), DataType::Decimal256(precision, scale) => Self::Decimal256(precision as _, scale as _), - DataType::Extension(_, d, _) => (*d).into(), + DataType::Extension(_, d, _) => Arc::unwrap_or_clone_polyfill(d).into(), } } } @@ -239,7 +298,7 @@ impl From for DataType { DataType::Float32 => Self::Float32, DataType::Float64 => Self::Float64, DataType::Timestamp(unit, tz) => { - Self::Timestamp(unit.into(), tz.map(|x| x.to_string())) + Self::Timestamp(unit.into(), tz.map(|tz| Arc::new(tz.to_string()))) } DataType::Date32 => Self::Date32, DataType::Date64 => Self::Date64, @@ -252,16 +311,16 @@ impl From for DataType { DataType::LargeBinary => Self::LargeBinary, DataType::Utf8 => Self::Utf8, DataType::LargeUtf8 => Self::LargeUtf8, - DataType::List(f) => Self::List(Box::new(f.into())), - DataType::FixedSizeList(f, size) => Self::FixedSizeList(Box::new(f.into()), size as _), - DataType::LargeList(f) => Self::LargeList(Box::new(f.into())), - DataType::Struct(f) => Self::Struct(f.into_iter().map(Into::into).collect()), + DataType::List(f) => Self::List(Arc::new(f.into())), + DataType::FixedSizeList(f, size) => Self::FixedSizeList(Arc::new(f.into()), size as _), + DataType::LargeList(f) => Self::LargeList(Arc::new(f.into())), + DataType::Struct(f) => Self::Struct(Arc::new(f.into_iter().map(Into::into).collect())), DataType::Union(fields, mode) => { - let ids = fields.iter().map(|(x, _)| x as _).collect(); - let fields = fields.iter().map(|(_, f)| f.into()).collect(); + let ids = Arc::new(fields.iter().map(|(x, _)| x as _).collect()); + let fields = Arc::new(fields.iter().map(|(_, f)| f.into()).collect()); Self::Union(fields, Some(ids), mode.into()) } - DataType::Map(f, ordered) => Self::Map(Box::new(f.into()), ordered), + DataType::Map(f, ordered) => Self::Map(std::sync::Arc::new(f.into()), ordered), DataType::Dictionary(key, value) => { let key = match *key { DataType::Int8 => IntegerType::Int8, @@ -274,7 +333,7 @@ impl From for DataType { DataType::UInt64 => IntegerType::UInt64, d => panic!("illegal dictionary key type: {d}"), }; - Self::Dictionary(key, Box::new((*value).into()), false) + Self::Dictionary(key, Arc::new((*value).into()), false) } DataType::Decimal128(precision, scale) => Self::Decimal(precision as _, scale as _), DataType::Decimal256(precision, scale) => Self::Decimal256(precision as _, scale as _), diff --git a/src/doc/lib.md b/src/doc/lib.md index 9708a6cd3d1..afeffddf20d 100644 --- a/src/doc/lib.md +++ b/src/doc/lib.md @@ -11,12 +11,12 @@ Below is an example of some of the things you can do with it: ```rust use std::sync::Arc; -use arrow2::array::*; -use arrow2::datatypes::{Field, DataType, Schema}; -use arrow2::compute::arithmetics; -use arrow2::error::Result; -use arrow2::io::parquet::write::*; -use arrow2::chunk::Chunk; +use re_arrow2::array::*; +use re_arrow2::datatypes::{Field, DataType, Schema}; +use re_arrow2::compute::arithmetics; +use re_arrow2::error::Result; +// use re_arrow2::io::parquet::write::*; +use re_arrow2::chunk::Chunk; fn main() -> Result<()> { // declare arrays @@ -36,32 +36,33 @@ fn main() -> Result<()> { // declare chunk let chunk = Chunk::new(vec![a.arced(), b.arced()]); - // write to parquet (probably the fastest implementation of writing to parquet out there) + // // write to parquet (probably the fastest implementation of writing to parquet out there) - let options = WriteOptions { - write_statistics: true, - compression: CompressionOptions::Snappy, - version: Version::V1, - data_pagesize_limit: None, - }; + // let options = WriteOptions { + // write_statistics: true, + // compression: CompressionOptions::Snappy, + // version: Version::V1, + // data_pagesize_limit: None, + // }; - let row_groups = RowGroupIterator::try_new( - vec![Ok(chunk)].into_iter(), - &schema, - options, - vec![vec![Encoding::Plain], vec![Encoding::Plain]], - )?; + // let row_groups = RowGroupIterator::try_new( + // vec![Ok(chunk)].into_iter(), + // &schema, + // options, + // vec![vec![Encoding::Plain], vec![Encoding::Plain]], + // )?; - // anything implementing `std::io::Write` works - let mut file = vec![]; + // // anything implementing `std::io::Write` works + // let mut file = vec![]; - let mut writer = FileWriter::try_new(file, schema, options)?; + // let mut writer = FileWriter::try_new(file, schema, options)?; + + // // Write the file. + // for group in row_groups { + // writer.write(group?)?; + // } + // let _ = writer.end(None)?; - // Write the file. - for group in row_groups { - writer.write(group?)?; - } - let _ = writer.end(None)?; Ok(()) } ``` @@ -83,5 +84,5 @@ functionality, such as: * `compute` to operate on arrays (addition, sum, sort, etc.) The feature `simd` (not part of `full`) produces more explicit SIMD instructions -via [`std::simd`](https://doc.rust-lang.org/nightly/std/simd/index.html), but requires the +via [`std::simd`](https://doc.rust-lang.org/nightly/std/simd/index.html), but requires the nightly channel. diff --git a/src/ffi/mmap.rs b/src/ffi/mmap.rs index 0f879d4fdca..cbc03dbcba7 100644 --- a/src/ffi/mmap.rs +++ b/src/ffi/mmap.rs @@ -105,7 +105,7 @@ pub unsafe fn slice(slice: &[T]) -> PrimitiveArray { let validity = None; let data: &[u8] = bytemuck::cast_slice(slice); - let ptr = data.as_ptr() as *const u8; + let ptr = data.as_ptr(); let data = Arc::new(data); // safety: the underlying assumption of this function: the array will not be used @@ -147,7 +147,7 @@ pub unsafe fn bitmap(data: &[u8], offset: usize, length: usize) -> Result Result { let indices = to_integer_type(schema.format())?; let values = to_field(dictionary)?; let is_ordered = schema.flags & 1 == 1; - DataType::Dictionary(indices, Box::new(values.data_type().clone()), is_ordered) + DataType::Dictionary( + indices, + std::sync::Arc::new(values.data_type().clone()), + is_ordered, + ) } else { to_data_type(schema)? }; let (metadata, extension) = unsafe { metadata_from_bytes(schema.metadata) }; let data_type = if let Some((name, extension_metadata)) = extension { - DataType::Extension(name, Box::new(data_type), extension_metadata) + DataType::Extension(name, Arc::new(data_type), extension_metadata.map(Arc::new)) } else { data_type }; @@ -263,23 +267,23 @@ unsafe fn to_data_type(schema: &ArrowSchema) -> Result { "tiD" => DataType::Interval(IntervalUnit::DayTime), "+l" => { let child = schema.child(0); - DataType::List(Box::new(to_field(child)?)) + DataType::List(Arc::new(to_field(child)?)) } "+L" => { let child = schema.child(0); - DataType::LargeList(Box::new(to_field(child)?)) + DataType::LargeList(Arc::new(to_field(child)?)) } "+m" => { let child = schema.child(0); let is_sorted = (schema.flags & 4) != 0; - DataType::Map(Box::new(to_field(child)?), is_sorted) + DataType::Map(std::sync::Arc::new(to_field(child)?), is_sorted) } "+s" => { let children = (0..schema.n_children as usize) .map(|x| to_field(schema.child(x))) .collect::>>()?; - DataType::Struct(children) + DataType::Struct(Arc::new(children)) } other => { match other.splitn(2, ':').collect::>()[..] { @@ -290,10 +294,18 @@ unsafe fn to_data_type(schema: &ArrowSchema) -> Result { ["tsn", ""] => DataType::Timestamp(TimeUnit::Nanosecond, None), // Timestamps with timezone - ["tss", tz] => DataType::Timestamp(TimeUnit::Second, Some(tz.to_string())), - ["tsm", tz] => DataType::Timestamp(TimeUnit::Millisecond, Some(tz.to_string())), - ["tsu", tz] => DataType::Timestamp(TimeUnit::Microsecond, Some(tz.to_string())), - ["tsn", tz] => DataType::Timestamp(TimeUnit::Nanosecond, Some(tz.to_string())), + ["tss", tz] => { + DataType::Timestamp(TimeUnit::Second, Some(Arc::new(tz.to_string()))) + } + ["tsm", tz] => { + DataType::Timestamp(TimeUnit::Millisecond, Some(Arc::new(tz.to_string()))) + } + ["tsu", tz] => { + DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::new(tz.to_string()))) + } + ["tsn", tz] => { + DataType::Timestamp(TimeUnit::Nanosecond, Some(Arc::new(tz.to_string()))) + } ["w", size_raw] => { // Example: "w:42" fixed-width binary [42 bytes] @@ -308,7 +320,7 @@ unsafe fn to_data_type(schema: &ArrowSchema) -> Result { .parse::() .map_err(|_| Error::OutOfSpec("size is not a valid integer".to_string()))?; let child = to_field(schema.child(0))?; - DataType::FixedSizeList(Box::new(child), size) + DataType::FixedSizeList(Arc::new(child), size) } ["d", raw] => { // Decimal @@ -373,7 +385,7 @@ unsafe fn to_data_type(schema: &ArrowSchema) -> Result { let fields = (0..schema.n_children as usize) .map(|x| to_field(schema.child(x))) .collect::>>()?; - DataType::Union(fields, Some(type_ids), mode) + DataType::Union(Arc::new(fields), Some(Arc::new(type_ids)), mode) } _ => { return Err(Error::OutOfSpec(format!( @@ -436,7 +448,7 @@ fn to_format(data_type: &DataType) -> String { format!( "ts{}:{}", unit, - tz.as_ref().map(|x| x.as_ref()).unwrap_or("") + tz.as_ref().map(|x| x.as_str()).unwrap_or("") ) } DataType::Decimal(precision, scale) => format!("d:{precision},{scale}"), @@ -568,40 +580,43 @@ mod tests { DataType::Binary, DataType::LargeBinary, DataType::FixedSizeBinary(2), - DataType::List(Box::new(Field::new("example", DataType::Boolean, false))), - DataType::FixedSizeList(Box::new(Field::new("example", DataType::Boolean, false)), 2), - DataType::LargeList(Box::new(Field::new("example", DataType::Boolean, false))), - DataType::Struct(vec![ + DataType::List(Arc::new(Field::new("example", DataType::Boolean, false))), + DataType::FixedSizeList(Arc::new(Field::new("example", DataType::Boolean, false)), 2), + DataType::LargeList(Arc::new(Field::new("example", DataType::Boolean, false))), + DataType::Struct(Arc::new(vec![ Field::new("a", DataType::Int64, true), Field::new( "b", - DataType::List(Box::new(Field::new("item", DataType::Int32, true))), + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), true, ), - ]), - DataType::Map(Box::new(Field::new("a", DataType::Int64, true)), true), + ])), + DataType::Map( + std::sync::Arc::new(Field::new("a", DataType::Int64, true)), + true, + ), DataType::Union( - vec![ + Arc::new(vec![ Field::new("a", DataType::Int64, true), Field::new( "b", - DataType::List(Box::new(Field::new("item", DataType::Int32, true))), + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), true, ), - ], - Some(vec![1, 2]), + ]), + Some(Arc::new(vec![1, 2])), UnionMode::Dense, ), DataType::Union( - vec![ + Arc::new(vec![ Field::new("a", DataType::Int64, true), Field::new( "b", - DataType::List(Box::new(Field::new("item", DataType::Int32, true))), + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), true, ), - ], - Some(vec![0, 1]), + ]), + Some(Arc::new(vec![0, 1])), UnionMode::Sparse, ), ]; @@ -612,7 +627,10 @@ mod tests { TimeUnit::Nanosecond, ] { dts.push(DataType::Timestamp(time_unit, None)); - dts.push(DataType::Timestamp(time_unit, Some("00:00".to_string()))); + dts.push(DataType::Timestamp( + time_unit, + Some(Arc::new("00:00".to_string())), + )); dts.push(DataType::Duration(time_unit)); } for interval_type in [ diff --git a/src/io/avro/read/nested.rs b/src/io/avro/read/nested.rs index 056d9a8f836..cc752e976bb 100644 --- a/src/io/avro/read/nested.rs +++ b/src/io/avro/read/nested.rs @@ -129,7 +129,7 @@ impl FixedItemsUtf8Dictionary { Self { data_type: DataType::Dictionary( IntegerType::Int32, - Box::new(values.data_type().clone()), + std::sync::Arc::new(values.data_type().clone()), false, ), keys: MutablePrimitiveArray::::with_capacity(capacity), diff --git a/src/io/avro/read/schema.rs b/src/io/avro/read/schema.rs index 3a920286c03..9c70b48e405 100644 --- a/src/io/avro/read/schema.rs +++ b/src/io/avro/read/schema.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use avro_schema::schema::{Enum, Fixed, Record, Schema as AvroSchema}; use crate::datatypes::*; @@ -52,10 +54,10 @@ fn schema_to_field(schema: &AvroSchema, name: Option<&str>, props: Metadata) -> Some(logical) => match logical { avro_schema::schema::LongLogical::Time => DataType::Time64(TimeUnit::Microsecond), avro_schema::schema::LongLogical::TimestampMillis => { - DataType::Timestamp(TimeUnit::Millisecond, Some("00:00".to_string())) + DataType::Timestamp(TimeUnit::Millisecond, Some(Arc::new("00:00".to_string()))) } avro_schema::schema::LongLogical::TimestampMicros => { - DataType::Timestamp(TimeUnit::Microsecond, Some("00:00".to_string())) + DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::new("00:00".to_string()))) } avro_schema::schema::LongLogical::LocalTimestampMillis => { DataType::Timestamp(TimeUnit::Millisecond, None) @@ -77,7 +79,7 @@ fn schema_to_field(schema: &AvroSchema, name: Option<&str>, props: Metadata) -> None => DataType::Binary, }, AvroSchema::String(_) => DataType::Utf8, - AvroSchema::Array(item_schema) => DataType::List(Box::new(schema_to_field( + AvroSchema::Array(item_schema) => DataType::List(std::sync::Arc::new(schema_to_field( item_schema, Some("item"), // default name for list items Metadata::default(), @@ -103,7 +105,7 @@ fn schema_to_field(schema: &AvroSchema, name: Option<&str>, props: Metadata) -> .iter() .map(|s| schema_to_field(s, None, Metadata::default())) .collect::>>()?; - DataType::Union(fields, None, UnionMode::Dense) + DataType::Union(Arc::new(fields), None, UnionMode::Dense) } } AvroSchema::Record(Record { fields, .. }) => { @@ -117,12 +119,12 @@ fn schema_to_field(schema: &AvroSchema, name: Option<&str>, props: Metadata) -> schema_to_field(&field.schema, Some(&field.name), props) }) .collect::>()?; - DataType::Struct(fields) + DataType::Struct(std::sync::Arc::new(fields)) } AvroSchema::Enum { .. } => { return Ok(Field::new( name.unwrap_or_default(), - DataType::Dictionary(IntegerType::Int32, Box::new(DataType::Utf8), false), + DataType::Dictionary(IntegerType::Int32, Arc::new(DataType::Utf8), false), false, )) } diff --git a/src/io/csv/utils.rs b/src/io/csv/utils.rs index fa8c01f43d7..7cf8d66595a 100644 --- a/src/io/csv/utils.rs +++ b/src/io/csv/utils.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::datatypes::{DataType, Field, TimeUnit}; use ahash::AHashSet; @@ -27,15 +29,18 @@ fn is_naive_datetime(string: &str) -> bool { string.parse::().is_ok() } -fn is_datetime(string: &str) -> Option { +fn is_datetime(string: &str) -> Option> { let mut parsed = chrono::format::Parsed::new(); let fmt = chrono::format::StrftimeItems::new(RFC3339); if chrono::format::parse(&mut parsed, string, fmt).is_ok() { - parsed.offset.map(|x| { - let hours = x / 60 / 60; - let minutes = x / 60 - hours * 60; - format!("{hours:03}:{minutes:02}") - }) + parsed + .offset + .map(|x| { + let hours = x / 60 / 60; + let minutes = x / 60 - hours * 60; + format!("{hours:03}:{minutes:02}") + }) + .map(Arc::new) } else { None } diff --git a/src/io/flight/mod.rs b/src/io/flight/mod.rs index 943f1487304..4d27dbe8b9f 100644 --- a/src/io/flight/mod.rs +++ b/src/io/flight/mod.rs @@ -30,7 +30,7 @@ pub fn serialize_batch( options: &WriteOptions, ) -> Result<(Vec, FlightData)> { if fields.len() != chunk.arrays().len() { - return Err(Error::InvalidArgumentError("The argument `fields` must be consistent with the columns' schema. Use e.g. &arrow2::io::flight::default_ipc_fields(&schema.fields)".to_string())); + return Err(Error::InvalidArgumentError("The argument `fields` must be consistent with the columns' schema. Use e.g. &re_arrow2::io::flight::default_ipc_fields(&schema.fields)".to_string())); } let mut dictionary_tracker = DictionaryTracker { diff --git a/src/io/ipc/mod.rs b/src/io/ipc/mod.rs index 2bb233a1474..f708251849a 100644 --- a/src/io/ipc/mod.rs +++ b/src/io/ipc/mod.rs @@ -29,12 +29,12 @@ //! # Examples //! Read and write to a file: //! ``` -//! use arrow2::io::ipc::{{read::{FileReader, read_file_metadata}}, {write::{FileWriter, WriteOptions}}}; +//! use re_arrow2::io::ipc::{{read::{FileReader, read_file_metadata}}, {write::{FileWriter, WriteOptions}}}; //! # use std::fs::File; -//! # use arrow2::datatypes::{Field, Schema, DataType}; -//! # use arrow2::array::{Int32Array, Array}; -//! # use arrow2::chunk::Chunk; -//! # use arrow2::error::Error; +//! # use re_arrow2::datatypes::{Field, Schema, DataType}; +//! # use re_arrow2::array::{Int32Array, Array}; +//! # use re_arrow2::chunk::Chunk; +//! # use re_arrow2::error::Error; //! // Setup the writer //! let path = "example.arrow".to_string(); //! let mut file = File::create(&path)?; diff --git a/src/io/ipc/read/schema.rs b/src/io/ipc/read/schema.rs index 7ec87eaa334..4d68f1281ad 100644 --- a/src/io/ipc/read/schema.rs +++ b/src/io/ipc/read/schema.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use arrow_format::ipc::{ planus::ReadAsRoot, FieldRef, FixedSizeListRef, MapRef, TimeRef, TimestampRef, UnionRef, }; @@ -104,7 +106,7 @@ fn deserialize_time(time: TimeRef) -> Result<(DataType, IpcField)> { } fn deserialize_timestamp(timestamp: TimestampRef) -> Result<(DataType, IpcField)> { - let timezone = timestamp.timezone()?.map(|tz| tz.to_string()); + let timezone = timestamp.timezone()?.map(|tz| Arc::new(tz.to_string())); let time_unit = deserialize_timeunit(timestamp.unit()?)?; Ok(( DataType::Timestamp(time_unit, timezone), @@ -131,7 +133,10 @@ fn deserialize_union(union_: UnionRef, field: FieldRef) -> Result<(DataType, Ipc fields: ipc_fields, dictionary_id: None, }; - Ok((DataType::Union(fields, ids, mode), ipc_field)) + Ok(( + DataType::Union(Arc::new(fields), ids.map(Arc::new), mode), + ipc_field, + )) } fn deserialize_map(map: MapRef, field: FieldRef) -> Result<(DataType, IpcField)> { @@ -145,7 +150,7 @@ fn deserialize_map(map: MapRef, field: FieldRef) -> Result<(DataType, IpcField)> .ok_or_else(|| Error::oos("IPC: Map must contain one child"))??; let (field, ipc_field) = deserialize_field(inner)?; - let data_type = DataType::Map(Box::new(field), is_sorted); + let data_type = DataType::Map(std::sync::Arc::new(field), is_sorted); Ok(( data_type, IpcField { @@ -170,7 +175,7 @@ fn deserialize_struct(field: FieldRef) -> Result<(DataType, IpcField)> { fields: ipc_fields, dictionary_id: None, }; - Ok((DataType::Struct(fields), ipc_field)) + Ok((DataType::Struct(std::sync::Arc::new(fields)), ipc_field)) } fn deserialize_list(field: FieldRef) -> Result<(DataType, IpcField)> { @@ -183,7 +188,7 @@ fn deserialize_list(field: FieldRef) -> Result<(DataType, IpcField)> { let (field, ipc_field) = deserialize_field(inner)?; Ok(( - DataType::List(Box::new(field)), + DataType::List(std::sync::Arc::new(field)), IpcField { fields: vec![ipc_field], dictionary_id: None, @@ -201,7 +206,7 @@ fn deserialize_large_list(field: FieldRef) -> Result<(DataType, IpcField)> { let (field, ipc_field) = deserialize_field(inner)?; Ok(( - DataType::LargeList(Box::new(field)), + DataType::LargeList(std::sync::Arc::new(field)), IpcField { fields: vec![ipc_field], dictionary_id: None, @@ -227,7 +232,7 @@ fn deserialize_fixed_size_list( .map_err(|_| Error::from(OutOfSpecKind::NegativeFooterLength))?; Ok(( - DataType::FixedSizeList(Box::new(field), size), + DataType::FixedSizeList(std::sync::Arc::new(field), size), IpcField { fields: vec![ipc_field], dictionary_id: None, @@ -250,7 +255,7 @@ fn get_data_type( let (inner, mut ipc_field) = get_data_type(field, extension, false)?; ipc_field.dictionary_id = Some(dictionary.id()?); return Ok(( - DataType::Dictionary(index_type, Box::new(inner), dictionary.is_ordered()?), + DataType::Dictionary(index_type, Arc::new(inner), dictionary.is_ordered()?), ipc_field, )); } @@ -260,7 +265,7 @@ fn get_data_type( let (name, metadata) = extension; let (data_type, fields) = get_data_type(field, None, false)?; return Ok(( - DataType::Extension(name, Box::new(data_type), metadata), + DataType::Extension(name, Arc::new(data_type), metadata.map(Arc::new)), fields, )); } diff --git a/src/io/ipc/write/file_async.rs b/src/io/ipc/write/file_async.rs index 6bf77536640..9141e968644 100644 --- a/src/io/ipc/write/file_async.rs +++ b/src/io/ipc/write/file_async.rs @@ -24,11 +24,11 @@ type WriteOutput = (usize, Option, Vec, Option); /// /// ``` /// use futures::{SinkExt, TryStreamExt, io::Cursor}; -/// use arrow2::array::{Array, Int32Array}; -/// use arrow2::datatypes::{DataType, Field, Schema}; -/// use arrow2::chunk::Chunk; -/// use arrow2::io::ipc::write::file_async::FileSink; -/// use arrow2::io::ipc::read::file_async::{read_file_metadata_async, FileStream}; +/// use re_arrow2::array::{Array, Int32Array}; +/// use re_arrow2::datatypes::{DataType, Field, Schema}; +/// use re_arrow2::chunk::Chunk; +/// use re_arrow2::io::ipc::write::file_async::FileSink; +/// use re_arrow2::io::ipc::read::file_async::{read_file_metadata_async, FileStream}; /// # futures::executor::block_on(async move { /// let schema = Schema::from(vec![ /// Field::new("values", DataType::Int32, true), @@ -56,7 +56,7 @@ type WriteOutput = (usize, Option, Vec, Option); /// let metadata = read_file_metadata_async(&mut buffer).await?; /// let mut stream = FileStream::new(buffer, metadata, None, None); /// let chunks = stream.try_collect::>().await?; -/// # arrow2::error::Result::Ok(()) +/// # re_arrow2::error::Result::Ok(()) /// # }).unwrap(); /// ``` pub struct FileSink<'a, W: AsyncWrite + Unpin + Send + 'a> { diff --git a/src/io/ipc/write/mod.rs b/src/io/ipc/write/mod.rs index 6fc355c85ca..df27f9949b5 100644 --- a/src/io/ipc/write/mod.rs +++ b/src/io/ipc/write/mod.rs @@ -8,7 +8,7 @@ pub(crate) mod writer; pub use common::{Compression, Record, WriteOptions}; pub use schema::schema_to_bytes; pub use serialize::write; -pub(self) use serialize::write_dictionary; +use serialize::write_dictionary; pub use stream::StreamWriter; pub use writer::FileWriter; diff --git a/src/io/ipc/write/schema.rs b/src/io/ipc/write/schema.rs index 1c4dab8e393..e1fa2b97aba 100644 --- a/src/io/ipc/write/schema.rs +++ b/src/io/ipc/write/schema.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use arrow_format::ipc::planus::Builder; use crate::datatypes::{ @@ -71,14 +73,14 @@ fn write_metadata(metadata: &Metadata, kv_vec: &mut Vec, + metadata: &Option>, kv_vec: &mut Vec, ) { // metadata if let Some(metadata) = metadata { let entry = arrow_format::ipc::KeyValue { key: Some("ARROW:extension:metadata".to_string()), - value: Some(metadata.clone()), + value: Some(metadata.to_string()), }; kv_vec.push(entry); } @@ -228,7 +230,7 @@ fn serialize_type(data_type: &DataType) -> arrow_format::ipc::Type { })), Timestamp(unit, tz) => ipc::Type::Timestamp(Box::new(ipc::Timestamp { unit: serialize_time_unit(unit), - timezone: tz.as_ref().cloned(), + timezone: tz.as_ref().map(|tz| tz.as_str().to_owned()), })), Interval(unit) => ipc::Type::Interval(Box::new(ipc::Interval { unit: match unit { @@ -247,7 +249,7 @@ fn serialize_type(data_type: &DataType) -> arrow_format::ipc::Type { UnionMode::Dense => ipc::UnionMode::Dense, UnionMode::Sparse => ipc::UnionMode::Sparse, }, - type_ids: type_ids.clone(), + type_ids: type_ids.as_ref().map(|type_ids| type_ids.to_vec()), })), Map(_, keys_sorted) => ipc::Type::Map(Box::new(ipc::Map { keys_sorted: *keys_sorted, diff --git a/src/io/ipc/write/stream_async.rs b/src/io/ipc/write/stream_async.rs index df651461fea..75727a1c799 100644 --- a/src/io/ipc/write/stream_async.rs +++ b/src/io/ipc/write/stream_async.rs @@ -21,10 +21,10 @@ use crate::error::{Error, Result}; /// /// ``` /// use futures::SinkExt; -/// use arrow2::array::{Array, Int32Array}; -/// use arrow2::datatypes::{DataType, Field, Schema}; -/// use arrow2::chunk::Chunk; -/// # use arrow2::io::ipc::write::stream_async::StreamSink; +/// use re_arrow2::array::{Array, Int32Array}; +/// use re_arrow2::datatypes::{DataType, Field, Schema}; +/// use re_arrow2::chunk::Chunk; +/// # use re_arrow2::io::ipc::write::stream_async::StreamSink; /// # futures::executor::block_on(async move { /// let schema = Schema::from(vec![ /// Field::new("values", DataType::Int32, true), @@ -44,7 +44,7 @@ use crate::error::{Error, Result}; /// sink.feed(chunk.into()).await?; /// } /// sink.close().await?; -/// # arrow2::error::Result::Ok(()) +/// # re_arrow2::error::Result::Ok(()) /// # }).unwrap(); /// ``` pub struct StreamSink<'a, W: AsyncWrite + Unpin + Send + 'a> { diff --git a/src/io/json/read/deserialize.rs b/src/io/json/read/deserialize.rs index 048e29a3005..531b37f9f5e 100644 --- a/src/io/json/read/deserialize.rs +++ b/src/io/json/read/deserialize.rs @@ -599,7 +599,7 @@ pub fn deserialize(json: &Value, data_type: DataType) -> Result, match json { Value::Array(rows) => match data_type { DataType::List(inner) | DataType::LargeList(inner) => { - Ok(_deserialize(rows, inner.data_type)) + Ok(_deserialize(rows, inner.data_type.clone())) } _ => Err(Error::nyi("read an Array from a non-Array data type")), }, diff --git a/src/io/json/read/infer_schema.rs b/src/io/json/read/infer_schema.rs index c4ba23f56fe..71142de1210 100644 --- a/src/io/json/read/infer_schema.rs +++ b/src/io/json/read/infer_schema.rs @@ -1,4 +1,5 @@ use std::borrow::Borrow; +use std::sync::Arc; use indexmap::map::IndexMap as HashMap; use indexmap::set::IndexSet as HashSet; @@ -77,7 +78,7 @@ fn infer_object(inner: &HashMap) -> Result { Ok(Field::new(key, dt, true)) }) .collect::>>()?; - Ok(DataType::Struct(fields)) + Ok(DataType::Struct(std::sync::Arc::new(fields))) } fn infer_array(values: &[Value]) -> Result { @@ -100,7 +101,7 @@ fn infer_array(values: &[Value]) -> Result { Ok(if dt == DataType::Null { dt } else { - DataType::List(Box::new(Field::new(ITEM_NAME, dt, true))) + DataType::List(std::sync::Arc::new(Field::new(ITEM_NAME, dt, true))) }) } @@ -136,7 +137,7 @@ pub(crate) fn coerce_data_type>(datatypes: &[A]) -> DataType // all are structs => union of all fields (that may have equal names) let fields = datatypes.iter().fold(vec![], |mut acc, dt| { if let Struct(new_fields) = dt.borrow() { - acc.extend(new_fields); + acc.extend(new_fields.as_slice()); }; acc }); @@ -165,7 +166,7 @@ pub(crate) fn coerce_data_type>(datatypes: &[A]) -> DataType Field::new(name, coerce_data_type(&dts), true) }) .collect(); - return Struct(fields); + return Struct(Arc::new(fields)); } else if datatypes.len() > 2 { return Utf8; } @@ -175,15 +176,15 @@ pub(crate) fn coerce_data_type>(datatypes: &[A]) -> DataType (lhs, rhs) if lhs == rhs => lhs.clone(), (List(lhs), List(rhs)) => { let inner = coerce_data_type(&[lhs.data_type(), rhs.data_type()]); - List(Box::new(Field::new(ITEM_NAME, inner, true))) + List(std::sync::Arc::new(Field::new(ITEM_NAME, inner, true))) } (scalar, List(list)) => { let inner = coerce_data_type(&[scalar, list.data_type()]); - List(Box::new(Field::new(ITEM_NAME, inner, true))) + List(std::sync::Arc::new(Field::new(ITEM_NAME, inner, true))) } (List(list), scalar) => { let inner = coerce_data_type(&[scalar, list.data_type()]); - List(Box::new(Field::new(ITEM_NAME, inner, true))) + List(std::sync::Arc::new(Field::new(ITEM_NAME, inner, true))) } (Float64, Int64) => Float64, (Int64, Float64) => Float64, @@ -204,25 +205,31 @@ mod test { assert_eq!( coerce_data_type(&[ Float64, - List(Box::new(Field::new(ITEM_NAME, Float64, true))) + List(std::sync::Arc::new(Field::new(ITEM_NAME, Float64, true))) ]), - List(Box::new(Field::new(ITEM_NAME, Float64, true))), + List(std::sync::Arc::new(Field::new(ITEM_NAME, Float64, true))), ); assert_eq!( - coerce_data_type(&[Float64, List(Box::new(Field::new(ITEM_NAME, Int64, true)))]), - List(Box::new(Field::new(ITEM_NAME, Float64, true))), + coerce_data_type(&[ + Float64, + List(std::sync::Arc::new(Field::new(ITEM_NAME, Int64, true))) + ]), + List(std::sync::Arc::new(Field::new(ITEM_NAME, Float64, true))), ); assert_eq!( - coerce_data_type(&[Int64, List(Box::new(Field::new(ITEM_NAME, Int64, true)))]), - List(Box::new(Field::new(ITEM_NAME, Int64, true))), + coerce_data_type(&[ + Int64, + List(std::sync::Arc::new(Field::new(ITEM_NAME, Int64, true))) + ]), + List(std::sync::Arc::new(Field::new(ITEM_NAME, Int64, true))), ); // boolean and number are incompatible, return utf8 assert_eq!( coerce_data_type(&[ Boolean, - List(Box::new(Field::new(ITEM_NAME, Float64, true))) + List(std::sync::Arc::new(Field::new(ITEM_NAME, Float64, true))) ]), - List(Box::new(Field::new(ITEM_NAME, Utf8, true))), + List(std::sync::Arc::new(Field::new(ITEM_NAME, Utf8, true))), ); } diff --git a/src/io/json_integration/read/schema.rs b/src/io/json_integration/read/schema.rs index 6a54dafc2b1..c2b08762570 100644 --- a/src/io/json_integration/read/schema.rs +++ b/src/io/json_integration/read/schema.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use serde_derive::Deserialize; use serde_json::Value; @@ -209,7 +211,7 @@ fn to_data_type(item: &Value, mut children: Vec) -> Result { Some(Value::String(tz)) => Ok(Some(tz.clone())), _ => Err(Error::OutOfSpec("timezone must be a string".to_string())), }?; - DataType::Timestamp(unit, tz) + DataType::Timestamp(unit, tz.map(Arc::new)) } "date" => match item.get("unit") { Some(p) if p == "DAY" => DataType::Date32, @@ -243,12 +245,12 @@ fn to_data_type(item: &Value, mut children: Vec) -> Result { } }, "int" => to_int(item).map(|x| x.into())?, - "list" => DataType::List(Box::new(children.pop().unwrap())), - "largelist" => DataType::LargeList(Box::new(children.pop().unwrap())), + "list" => DataType::List(std::sync::Arc::new(children.pop().unwrap())), + "largelist" => DataType::LargeList(std::sync::Arc::new(children.pop().unwrap())), "fixedsizelist" => { if let Some(Value::Number(size)) = item.get("listSize") { DataType::FixedSizeList( - Box::new(children.pop().unwrap()), + Arc::new(children.pop().unwrap()), size.as_i64().unwrap() as usize, ) } else { @@ -257,7 +259,7 @@ fn to_data_type(item: &Value, mut children: Vec) -> Result { )); } } - "struct" => DataType::Struct(children), + "struct" => DataType::Struct(Arc::new(children)), "union" => { let mode = if let Some(Value::String(mode)) = item.get("mode") { UnionMode::sparse(mode == "SPARSE") @@ -265,11 +267,13 @@ fn to_data_type(item: &Value, mut children: Vec) -> Result { return Err(Error::OutOfSpec("union requires mode".to_string())); }; let ids = if let Some(Value::Array(ids)) = item.get("typeIds") { - Some(ids.iter().map(|x| x.as_i64().unwrap() as i32).collect()) + Some(Arc::new( + ids.iter().map(|x| x.as_i64().unwrap() as i32).collect(), + )) } else { return Err(Error::OutOfSpec("union requires ids".to_string())); }; - DataType::Union(children, ids, mode) + DataType::Union(Arc::new(children), ids, mode) } "map" => { let sorted_keys = if let Some(Value::Bool(sorted_keys)) = item.get("keysSorted") { @@ -277,7 +281,7 @@ fn to_data_type(item: &Value, mut children: Vec) -> Result { } else { return Err(Error::OutOfSpec("sorted keys not defined".to_string())); }; - DataType::Map(Box::new(children.pop().unwrap()), sorted_keys) + DataType::Map(std::sync::Arc::new(children.pop().unwrap()), sorted_keys) } other => { return Err(Error::NotYetImplemented(format!( @@ -368,7 +372,7 @@ fn deserialize_field(value: &Value) -> Result { let data_type = to_data_type(type_, children)?; let data_type = if let Some((name, metadata)) = extension { - DataType::Extension(name, Box::new(data_type), metadata) + DataType::Extension(name, Arc::new(data_type), metadata.map(Arc::new)) } else { data_type }; @@ -390,7 +394,7 @@ fn deserialize_field(value: &Value) -> Result { )); } }; - DataType::Dictionary(index_type, Box::new(data_type), is_ordered) + DataType::Dictionary(index_type, Arc::new(data_type), is_ordered) } else { data_type }; diff --git a/src/io/orc/read/mod.rs b/src/io/orc/read/mod.rs index 3fe4abb7f63..32c4465c0bf 100644 --- a/src/io/orc/read/mod.rs +++ b/src/io/orc/read/mod.rs @@ -1,9 +1,10 @@ //! APIs to read from [ORC format](https://orc.apache.org). use std::io::Read; +use std::sync::Arc; use crate::array::{Array, BinaryArray, BooleanArray, Int64Array, PrimitiveArray, Utf8Array}; use crate::bitmap::{Bitmap, MutableBitmap}; -use crate::datatypes::{DataType, Field, Schema}; +use crate::datatypes::{ArcExt, DataType, Field, Schema}; use crate::error::Error; use crate::offset::{Offset, Offsets}; use crate::types::NativeType; @@ -21,7 +22,7 @@ pub fn infer_schema(footer: &Footer) -> Result { let dt = infer_dt(&footer.types[0], types)?; if let DataType::Struct(fields) = dt { - Ok(fields.into()) + Ok(Arc::unwrap_or_clone_polyfill(fields).into()) } else { Err(Error::ExternalFormat( "ORC root type must be a struct".to_string(), @@ -57,7 +58,7 @@ fn infer_dt(type_: &Type, types: &[Type]) -> Result { .map(|dt| Field::new(name, dt, true)) }) .collect::, Error>>()?; - DataType::Struct(sub_types) + DataType::Struct(Arc::new(sub_types)) } kind => return Err(Error::nyi(format!("Reading {kind:?} from ORC"))), }; diff --git a/src/io/parquet/read/deserialize/binary/dictionary.rs b/src/io/parquet/read/deserialize/binary/dictionary.rs index 6f883528ef8..df8d3988c8c 100644 --- a/src/io/parquet/read/deserialize/binary/dictionary.rs +++ b/src/io/parquet/read/deserialize/binary/dictionary.rs @@ -1,11 +1,11 @@ -use std::collections::VecDeque; +use std::{collections::VecDeque, sync::Arc}; use parquet2::page::DictPage; use crate::{ array::{Array, BinaryArray, DictionaryArray, DictionaryKey, Utf8Array}, bitmap::MutableBitmap, - datatypes::{DataType, PhysicalType}, + datatypes::{ArcExt, DataType, PhysicalType}, error::Result, io::parquet::read::deserialize::nested_utils::{InitNested, NestedState}, offset::Offset, @@ -53,7 +53,7 @@ where fn read_dict(data_type: DataType, dict: &DictPage) -> Box { let data_type = match data_type { - DataType::Dictionary(_, values, _) => *values, + DataType::Dictionary(_, values, _) => Arc::unwrap_or_clone_polyfill(values), _ => data_type, }; diff --git a/src/io/parquet/read/deserialize/fixed_size_binary/dictionary.rs b/src/io/parquet/read/deserialize/fixed_size_binary/dictionary.rs index 680834ad270..27ef41312e2 100644 --- a/src/io/parquet/read/deserialize/fixed_size_binary/dictionary.rs +++ b/src/io/parquet/read/deserialize/fixed_size_binary/dictionary.rs @@ -1,11 +1,11 @@ -use std::collections::VecDeque; +use std::{collections::VecDeque, sync::Arc}; use parquet2::page::DictPage; use crate::{ array::{Array, DictionaryArray, DictionaryKey, FixedSizeBinaryArray}, bitmap::MutableBitmap, - datatypes::DataType, + datatypes::{ArcExt, DataType}, error::Result, io::parquet::read::deserialize::nested_utils::{InitNested, NestedState}, }; @@ -48,7 +48,7 @@ where fn read_dict(data_type: DataType, dict: &DictPage) -> Box { let data_type = match data_type { - DataType::Dictionary(_, values, _) => *values, + DataType::Dictionary(_, values, _) => Arc::unwrap_or_clone_polyfill(values), _ => data_type, }; diff --git a/src/io/parquet/read/deserialize/primitive/dictionary.rs b/src/io/parquet/read/deserialize/primitive/dictionary.rs index 16fec526112..b46cc6f7286 100644 --- a/src/io/parquet/read/deserialize/primitive/dictionary.rs +++ b/src/io/parquet/read/deserialize/primitive/dictionary.rs @@ -1,11 +1,11 @@ -use std::collections::VecDeque; +use std::{collections::VecDeque, sync::Arc}; use parquet2::{page::DictPage, types::NativeType as ParquetNativeType}; use crate::{ array::{Array, DictionaryArray, DictionaryKey, PrimitiveArray}, bitmap::MutableBitmap, - datatypes::DataType, + datatypes::{ArcExt, DataType}, error::Result, types::NativeType, }; @@ -24,7 +24,7 @@ where F: Copy + Fn(P) -> T, { let data_type = match data_type { - DataType::Dictionary(_, values, _) => *values, + DataType::Dictionary(_, values, _) => Arc::unwrap_or_clone_polyfill(values), _ => data_type, }; let values = deserialize_plain(&dict.buffer, op); diff --git a/src/io/parquet/read/deserialize/struct_.rs b/src/io/parquet/read/deserialize/struct_.rs index dd5776948cd..72209260df7 100644 --- a/src/io/parquet/read/deserialize/struct_.rs +++ b/src/io/parquet/read/deserialize/struct_.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::array::{Array, StructArray}; use crate::datatypes::{DataType, Field}; use crate::error::Error; @@ -7,12 +9,12 @@ use super::nested_utils::{NestedArrayIter, NestedState}; /// An iterator adapter over [`NestedArrayIter`] assumed to be encoded as Struct arrays pub struct StructIterator<'a> { iters: Vec>, - fields: Vec, + fields: Arc>, } impl<'a> StructIterator<'a> { /// Creates a new [`StructIterator`] with `iters` and `fields`. - pub fn new(iters: Vec>, fields: Vec) -> Self { + pub fn new(iters: Vec>, fields: Arc>) -> Self { assert_eq!(iters.len(), fields.len()); Self { iters, fields } } diff --git a/src/io/parquet/read/schema/convert.rs b/src/io/parquet/read/schema/convert.rs index 1ec43acd4e6..3e36f536b71 100644 --- a/src/io/parquet/read/schema/convert.rs +++ b/src/io/parquet/read/schema/convert.rs @@ -1,4 +1,7 @@ //! This module has entry points, [`parquet_to_arrow_schema`] and the more configurable [`parquet_to_arrow_schema_with_options`]. + +use std::sync::Arc; + use parquet2::schema::{ types::{ FieldInfo, GroupConvertedType, GroupLogicalType, IntegerType, ParquetType, PhysicalType, @@ -107,12 +110,14 @@ fn from_int64( match unit { ParquetTimeUnit::Milliseconds => { - DataType::Timestamp(TimeUnit::Millisecond, timezone) + DataType::Timestamp(TimeUnit::Millisecond, timezone.map(Arc::new)) } ParquetTimeUnit::Microseconds => { - DataType::Timestamp(TimeUnit::Microsecond, timezone) + DataType::Timestamp(TimeUnit::Microsecond, timezone.map(Arc::new)) + } + ParquetTimeUnit::Nanoseconds => { + DataType::Timestamp(TimeUnit::Nanosecond, timezone.map(Arc::new)) } - ParquetTimeUnit::Nanoseconds => DataType::Timestamp(TimeUnit::Nanosecond, timezone), } } (Some(Time { unit, .. }), _) => match unit { @@ -222,7 +227,7 @@ fn to_primitive_type(primitive_type: &PrimitiveType, options: &SchemaInferenceOp let base_type = to_primitive_type_inner(primitive_type, options); if primitive_type.field_info.repetition == Repetition::Repeated { - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new( &primitive_type.field_info.name, base_type, is_nullable(&primitive_type.field_info), @@ -261,7 +266,7 @@ fn to_struct(fields: &[ParquetType], options: &SchemaInferenceOptions) -> Option if fields.is_empty() { None } else { - Some(DataType::Struct(fields)) + Some(DataType::Struct(std::sync::Arc::new(fields))) } } @@ -269,7 +274,7 @@ fn to_struct(fields: &[ParquetType], options: &SchemaInferenceOptions) -> Option /// Returns [`None`] if all its fields are empty fn to_map(fields: &[ParquetType], options: &SchemaInferenceOptions) -> Option { let inner = to_field(&fields[0], options)?; - Some(DataType::Map(Box::new(inner), false)) + Some(DataType::Map(Arc::new(inner), false)) } /// Entry point for converting parquet group type. @@ -285,7 +290,7 @@ fn to_group_type( ) -> Option { debug_assert!(!fields.is_empty()); if field_info.repetition == Repetition::Repeated { - Some(DataType::List(Box::new(Field::new( + Some(DataType::List(std::sync::Arc::new(Field::new( &field_info.name, to_struct(fields, options)?, is_nullable(field_info), @@ -362,7 +367,7 @@ fn to_list( ), }; - Some(DataType::List(Box::new(Field::new( + Some(DataType::List(std::sync::Arc::new(Field::new( list_item_name, item_type, item_is_optional, @@ -561,7 +566,11 @@ mod tests { { arrow_fields.push(Field::new( "my_list", - DataType::List(Box::new(Field::new("element", DataType::Utf8, true))), + DataType::List(std::sync::Arc::new(Field::new( + "element", + DataType::Utf8, + true, + ))), false, )); } @@ -575,7 +584,11 @@ mod tests { { arrow_fields.push(Field::new( "my_list", - DataType::List(Box::new(Field::new("element", DataType::Utf8, false))), + DataType::List(std::sync::Arc::new(Field::new( + "element", + DataType::Utf8, + false, + ))), true, )); } @@ -593,11 +606,18 @@ mod tests { // } // } { - let arrow_inner_list = - DataType::List(Box::new(Field::new("element", DataType::Int32, false))); + let arrow_inner_list = DataType::List(std::sync::Arc::new(Field::new( + "element", + DataType::Int32, + false, + ))); arrow_fields.push(Field::new( "array_of_arrays", - DataType::List(Box::new(Field::new("element", arrow_inner_list, false))), + DataType::List(std::sync::Arc::new(Field::new( + "element", + arrow_inner_list, + false, + ))), true, )); } @@ -611,7 +631,7 @@ mod tests { { arrow_fields.push(Field::new( "my_list", - DataType::List(Box::new(Field::new("element", DataType::Utf8, false))), + DataType::List(Arc::new(Field::new("element", DataType::Utf8, false))), true, )); } @@ -623,7 +643,7 @@ mod tests { { arrow_fields.push(Field::new( "my_list", - DataType::List(Box::new(Field::new("element", DataType::Int32, false))), + DataType::List(Arc::new(Field::new("element", DataType::Int32, false))), true, )); } @@ -636,13 +656,13 @@ mod tests { // }; // } { - let arrow_struct = DataType::Struct(vec![ + let arrow_struct = DataType::Struct(Arc::new(vec![ Field::new("str", DataType::Utf8, false), Field::new("num", DataType::Int32, false), - ]); + ])); arrow_fields.push(Field::new( "my_list", - DataType::List(Box::new(Field::new("element", arrow_struct, false))), + DataType::List(Arc::new(Field::new("element", arrow_struct, false))), true, )); } @@ -655,10 +675,11 @@ mod tests { // } // Special case: group is named array { - let arrow_struct = DataType::Struct(vec![Field::new("str", DataType::Utf8, false)]); + let arrow_struct = + DataType::Struct(Arc::new(vec![Field::new("str", DataType::Utf8, false)])); arrow_fields.push(Field::new( "my_list", - DataType::List(Box::new(Field::new("array", arrow_struct, false))), + DataType::List(Arc::new(Field::new("array", arrow_struct, false))), true, )); } @@ -671,10 +692,11 @@ mod tests { // } // Special case: group named ends in _tuple { - let arrow_struct = DataType::Struct(vec![Field::new("str", DataType::Utf8, false)]); + let arrow_struct = + DataType::Struct(Arc::new(vec![Field::new("str", DataType::Utf8, false)])); arrow_fields.push(Field::new( "my_list", - DataType::List(Box::new(Field::new("my_list_tuple", arrow_struct, false))), + DataType::List(Arc::new(Field::new("my_list_tuple", arrow_struct, false))), true, )); } @@ -684,7 +706,7 @@ mod tests { { arrow_fields.push(Field::new( "name", - DataType::List(Box::new(Field::new("name", DataType::Int32, false))), + DataType::List(Arc::new(Field::new("name", DataType::Int32, false))), false, )); } @@ -722,7 +744,7 @@ mod tests { ]; arrow_fields.push(Field::new( "events", - DataType::List(Box::new(Field::new( + DataType::List(Arc::new(Field::new( "array", DataType::Struct(struct_fields), false, @@ -771,7 +793,11 @@ mod tests { { arrow_fields.push(Field::new( "my_list1", - DataType::List(Box::new(Field::new("element", DataType::Utf8, true))), + DataType::List(std::sync::Arc::new(Field::new( + "element", + DataType::Utf8, + true, + ))), false, )); } @@ -785,7 +811,11 @@ mod tests { { arrow_fields.push(Field::new( "my_list2", - DataType::List(Box::new(Field::new("element", DataType::Utf8, false))), + DataType::List(std::sync::Arc::new(Field::new( + "element", + DataType::Utf8, + false, + ))), true, )); } @@ -799,7 +829,11 @@ mod tests { { arrow_fields.push(Field::new( "my_list3", - DataType::List(Box::new(Field::new("element", DataType::Utf8, false))), + DataType::List(std::sync::Arc::new(Field::new( + "element", + DataType::Utf8, + false, + ))), false, )); } @@ -815,10 +849,10 @@ mod tests { fn test_nested_schema() -> Result<()> { let mut arrow_fields = Vec::new(); { - let group1_fields = vec![ + let group1_fields = Arc::new(vec![ Field::new("leaf1", DataType::Boolean, false), Field::new("leaf2", DataType::Int32, false), - ]; + ]); let group1_struct = Field::new("group1", DataType::Struct(group1_fields), false); arrow_fields.push(group1_struct); @@ -851,9 +885,9 @@ mod tests { let inner_group_list = Field::new( "innerGroup", - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new( "innerGroup", - DataType::Struct(vec![Field::new("leaf3", DataType::Int32, true)]), + DataType::Struct(Arc::new(vec![Field::new("leaf3", DataType::Int32, true)])), false, ))), false, @@ -861,12 +895,12 @@ mod tests { let outer_group_list = Field::new( "outerGroup", - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new( "outerGroup", - DataType::Struct(vec![ + DataType::Struct(Arc::new(vec![ Field::new("leaf2", DataType::Int32, true), inner_group_list, - ]), + ])), false, ))), false, @@ -930,7 +964,7 @@ mod tests { Field::new("string", DataType::Utf8, true), Field::new( "bools", - DataType::List(Box::new(Field::new("bools", DataType::Boolean, false))), + DataType::List(Arc::new(Field::new("bools", DataType::Boolean, false))), false, ), Field::new("date", DataType::Date32, true), @@ -949,7 +983,7 @@ mod tests { ), Field::new( "ts_nano", - DataType::Timestamp(TimeUnit::Nanosecond, Some("+00:00".to_string())), + DataType::Timestamp(TimeUnit::Nanosecond, Some(Arc::new("+00:00".to_string()))), false, ), ]; @@ -1012,12 +1046,20 @@ mod tests { Field::new("string", DataType::Utf8, true), Field::new( "bools", - DataType::List(Box::new(Field::new("element", DataType::Boolean, true))), + DataType::List(std::sync::Arc::new(Field::new( + "element", + DataType::Boolean, + true, + ))), true, ), Field::new( "bools_non_null", - DataType::List(Box::new(Field::new("element", DataType::Boolean, false))), + DataType::List(std::sync::Arc::new(Field::new( + "element", + DataType::Boolean, + false, + ))), false, ), Field::new("date", DataType::Date32, true), @@ -1035,15 +1077,19 @@ mod tests { ), Field::new( "struct", - DataType::Struct(vec![ + DataType::Struct(Arc::new(vec![ Field::new("bools", DataType::Boolean, false), Field::new("uint32", DataType::UInt32, false), Field::new( "int32", - DataType::List(Box::new(Field::new("element", DataType::Int32, true))), + DataType::List(std::sync::Arc::new(Field::new( + "element", + DataType::Int32, + true, + ))), false, ), - ]), + ])), false, ), Field::new("dictionary_strings", DataType::Utf8, false), diff --git a/src/io/parquet/read/statistics/mod.rs b/src/io/parquet/read/statistics/mod.rs index 8514913d569..99c050e14cb 100644 --- a/src/io/parquet/read/statistics/mod.rs +++ b/src/io/parquet/read/statistics/mod.rs @@ -208,25 +208,25 @@ fn make_mutable(data_type: &DataType, capacity: usize) -> Result DataType { if let DataType::Struct(fields) = data_type.to_logical_type() { - DataType::Struct( + DataType::Struct(Arc::new( fields .iter() .map(|f| Field::new(&f.name, create_dt(&f.data_type), f.is_nullable)) .collect(), - ) + )) } else if let DataType::Map(f, ordered) = data_type.to_logical_type() { DataType::Map( - Box::new(Field::new(&f.name, create_dt(&f.data_type), f.is_nullable)), + Arc::new(Field::new(&f.name, create_dt(&f.data_type), f.is_nullable)), *ordered, ) } else if let DataType::List(f) = data_type.to_logical_type() { - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new( &f.name, create_dt(&f.data_type), f.is_nullable, ))) } else if let DataType::LargeList(f) = data_type.to_logical_type() { - DataType::LargeList(Box::new(Field::new( + DataType::LargeList(std::sync::Arc::new(Field::new( &f.name, create_dt(&f.data_type), f.is_nullable, diff --git a/src/io/parquet/write/mod.rs b/src/io/parquet/write/mod.rs index 6ef1864c6f3..2b7ef4f76d0 100644 --- a/src/io/parquet/write/mod.rs +++ b/src/io/parquet/write/mod.rs @@ -843,7 +843,7 @@ fn transverse_recursive T + Clone>( } Struct => { if let DataType::Struct(fields) = data_type.to_logical_type() { - for field in fields { + for field in fields.as_slice() { transverse_recursive(&field.data_type, map.clone(), encodings) } } else { @@ -853,7 +853,7 @@ fn transverse_recursive T + Clone>( Map => { if let DataType::Map(field, _) = data_type.to_logical_type() { if let DataType::Struct(fields) = field.data_type.to_logical_type() { - for field in fields { + for field in fields.as_slice() { transverse_recursive(&field.data_type, map.clone(), encodings) } } else { @@ -872,12 +872,12 @@ fn transverse_recursive T + Clone>( /// This is used to assign an [`Encoding`] to every parquet column based on the columns' type (see example) /// # Example /// ``` -/// use arrow2::io::parquet::write::{transverse, Encoding}; -/// use arrow2::datatypes::{DataType, Field}; +/// use re_arrow2::io::parquet::write::{transverse, Encoding}; +/// use re_arrow2::datatypes::{DataType, Field}; /// /// let dt = DataType::Struct(vec![ /// Field::new("a", DataType::Int64, true), -/// Field::new("b", DataType::List(Box::new(Field::new("item", DataType::Int32, true))), true), +/// Field::new("b", DataType::List(std::sync::Arc::new(Field::new("item", DataType::Int32, true))), true), /// ]); /// /// let encodings = transverse(&dt, |dt| Encoding::Plain); diff --git a/src/io/parquet/write/pages.rs b/src/io/parquet/write/pages.rs index 10aea638a22..0644f6b48d7 100644 --- a/src/io/parquet/write/pages.rs +++ b/src/io/parquet/write/pages.rs @@ -258,6 +258,8 @@ pub fn array_to_columns + Send + Sync>( #[cfg(test)] mod tests { + use std::sync::Arc; + use parquet2::schema::types::{GroupLogicalType, PrimitiveConvertedType, PrimitiveLogicalType}; use parquet2::schema::Repetition; @@ -280,7 +282,7 @@ mod tests { ]; let array = StructArray::new( - DataType::Struct(fields), + DataType::Struct(std::sync::Arc::new(fields)), vec![boolean.clone(), int.clone()], Some(Bitmap::from([true, true, false, true])), ); @@ -344,7 +346,7 @@ mod tests { ]; let array = StructArray::new( - DataType::Struct(fields), + DataType::Struct(std::sync::Arc::new(fields)), vec![boolean.clone(), int.clone()], Some(Bitmap::from([true, true, false, true])), ); @@ -355,7 +357,7 @@ mod tests { ]; let array = StructArray::new( - DataType::Struct(fields), + DataType::Struct(std::sync::Arc::new(fields)), vec![Box::new(array.clone()), Box::new(array)], None, ); @@ -447,13 +449,17 @@ mod tests { ]; let array = StructArray::new( - DataType::Struct(fields), + DataType::Struct(std::sync::Arc::new(fields)), vec![boolean.clone(), int.clone()], Some(Bitmap::from([true, true, false, true])), ); let array = ListArray::new( - DataType::List(Box::new(Field::new("l", array.data_type().clone(), true))), + DataType::List(std::sync::Arc::new(Field::new( + "l", + array.data_type().clone(), + true, + ))), vec![0i32, 2, 4].try_into().unwrap(), Box::new(array), None, @@ -540,12 +546,12 @@ mod tests { #[test] fn test_map() { - let kv_type = DataType::Struct(vec![ + let kv_type = DataType::Struct(Arc::new(vec![ Field::new("k", DataType::Utf8, false), Field::new("v", DataType::Int32, false), - ]); + ])); let kv_field = Field::new("kv", kv_type.clone(), false); - let map_type = DataType::Map(Box::new(kv_field), false); + let map_type = DataType::Map(std::sync::Arc::new(kv_field), false); let key_array = Utf8Array::::from_slice(["k1", "k2", "k3", "k4", "k5", "k6"]).boxed(); let val_array = Int32Array::from_slice([42, 28, 19, 31, 21, 17]).boxed(); diff --git a/src/io/parquet/write/sink.rs b/src/io/parquet/write/sink.rs index 1eeb83e21b2..bec103deed0 100644 --- a/src/io/parquet/write/sink.rs +++ b/src/io/parquet/write/sink.rs @@ -20,11 +20,11 @@ use super::{Encoding, SchemaDescriptor, WriteOptions}; /// /// ``` /// use futures::SinkExt; -/// use arrow2::array::{Array, Int32Array}; -/// use arrow2::datatypes::{DataType, Field, Schema}; -/// use arrow2::chunk::Chunk; -/// use arrow2::io::parquet::write::{Encoding, WriteOptions, CompressionOptions, Version}; -/// # use arrow2::io::parquet::write::FileSink; +/// use re_arrow2::array::{Array, Int32Array}; +/// use re_arrow2::datatypes::{DataType, Field, Schema}; +/// use re_arrow2::chunk::Chunk; +/// use re_arrow2::io::parquet::write::{Encoding, WriteOptions, CompressionOptions, Version}; +/// # use re_arrow2::io::parquet::write::FileSink; /// # futures::executor::block_on(async move { /// /// let schema = Schema::from(vec![ @@ -53,7 +53,7 @@ use super::{Encoding, SchemaDescriptor, WriteOptions}; /// } /// sink.metadata.insert(String::from("key"), Some(String::from("value"))); /// sink.close().await?; -/// # arrow2::error::Result::Ok(()) +/// # re_arrow2::error::Result::Ok(()) /// # }).unwrap(); /// ``` pub struct FileSink<'a, W: AsyncWrite + Send + Unpin> { diff --git a/src/scalar/primitive.rs b/src/scalar/primitive.rs index bc469eb98e7..a7bdf40ca50 100644 --- a/src/scalar/primitive.rs +++ b/src/scalar/primitive.rs @@ -15,12 +15,14 @@ impl PrimitiveScalar { #[inline] pub fn new(data_type: DataType, value: Option) -> Self { if !data_type.to_physical_type().eq_primitive(T::PRIMITIVE) { - Err(Error::InvalidArgumentError(format!( - "Type {} does not support logical type {:?}", - std::any::type_name::(), - data_type - ))) - .unwrap() + panic!( + "{:?}", + Error::InvalidArgumentError(format!( + "Type {} does not support logical type {:?}", + std::any::type_name::(), + data_type + )) + ); } Self { value, data_type } } diff --git a/src/temporal_conversions.rs b/src/temporal_conversions.rs index f2864c34179..7f4aae3ec2f 100644 --- a/src/temporal_conversions.rs +++ b/src/temporal_conversions.rs @@ -1,5 +1,7 @@ //! Conversion methods for dates and times. +use std::sync::Arc; + use chrono::{ format::{parse, Parsed, StrftimeItems}, Datelike, Duration, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, @@ -398,7 +400,7 @@ pub fn utf8_to_naive_timestamp_scalar(value: &str, fmt: &str, tu: &TimeUnit) -> fn utf8_to_timestamp_ns_impl( array: &Utf8Array, fmt: &str, - timezone: String, + timezone: Arc, tz: T, ) -> PrimitiveArray { let iter = array @@ -423,9 +425,9 @@ pub fn parse_offset_tz(timezone: &str) -> Result { fn chrono_tz_utf_to_timestamp_ns( array: &Utf8Array, fmt: &str, - timezone: String, + timezone: Arc, ) -> Result> { - let tz = parse_offset_tz(&timezone)?; + let tz = parse_offset_tz(timezone.as_str())?; Ok(utf8_to_timestamp_ns_impl(array, fmt, timezone, tz)) } @@ -433,7 +435,7 @@ fn chrono_tz_utf_to_timestamp_ns( fn chrono_tz_utf_to_timestamp_ns( _: &Utf8Array, _: &str, - timezone: String, + timezone: Arc, ) -> Result> { Err(Error::InvalidArgumentError(format!( "timezone \"{timezone}\" cannot be parsed (feature chrono-tz is not active)", @@ -451,7 +453,7 @@ fn chrono_tz_utf_to_timestamp_ns( pub fn utf8_to_timestamp_ns( array: &Utf8Array, fmt: &str, - timezone: String, + timezone: Arc, ) -> Result> { let tz = parse_offset(timezone.as_str()); diff --git a/src/types/bit_chunk.rs b/src/types/bit_chunk.rs index 796a608b1b8..a4dc51c43ab 100644 --- a/src/types/bit_chunk.rs +++ b/src/types/bit_chunk.rs @@ -56,7 +56,7 @@ bit_chunk!(u64); /// to the first slot, as defined by the arrow specification. /// # Example /// ``` -/// use arrow2::types::BitChunkIter; +/// use re_arrow2::types::BitChunkIter; /// let a = 0b00010000u8; /// let iter = BitChunkIter::new(a, 7); /// let r = iter.collect::>(); @@ -109,7 +109,7 @@ unsafe impl crate::trusted_len::TrustedLen for BitChunkIter {} /// See for details /// # Example /// ``` -/// use arrow2::types::BitChunkOnes; +/// use re_arrow2::types::BitChunkOnes; /// let a = 0b00010000u8; /// let iter = BitChunkOnes::new(a); /// let r = iter.collect::>(); diff --git a/src/types/simd/packed.rs b/src/types/simd/packed.rs index 0d95b68882a..048d3b5d757 100644 --- a/src/types/simd/packed.rs +++ b/src/types/simd/packed.rs @@ -1,7 +1,7 @@ pub use std::simd::{ f32x16, f32x8, f64x8, i16x32, i16x8, i32x16, i32x8, i64x8, i8x64, i8x8, mask32x16 as m32x16, - mask64x8 as m64x8, mask8x64 as m8x64, u16x32, u16x8, u32x16, u32x8, u64x8, u8x64, u8x8, - SimdPartialEq, + mask64x8 as m64x8, mask8x64 as m8x64, prelude::*, u16x32, u16x8, u32x16, u32x8, u64x8, u8x64, + u8x8, }; /// Vector of 32 16-bit masks diff --git a/testing/arrow-testing b/testing/arrow-testing deleted file mode 160000 index e8ce32338f2..00000000000 --- a/testing/arrow-testing +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e8ce32338f2dfeca3a5126f7677bdee159604000 diff --git a/testing/parquet-testing b/testing/parquet-testing deleted file mode 160000 index 8e7badc6a38..00000000000 --- a/testing/parquet-testing +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8e7badc6a3817a02e06d17b5d8ab6b6dc356e890 diff --git a/tests/it/array/binary/mod.rs b/tests/it/array/binary/mod.rs index 1418fce8918..bb6e374c415 100644 --- a/tests/it/array/binary/mod.rs +++ b/tests/it/array/binary/mod.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::{Array, BinaryArray}, bitmap::Bitmap, buffer::Buffer, @@ -74,7 +74,7 @@ fn try_from_trusted_len_iter() { let iter = std::iter::repeat(b"hello".as_ref()) .take(2) .map(Some) - .map(arrow2::error::Result::Ok); + .map(re_arrow2::error::Result::Ok); let a = BinaryArray::::try_from_trusted_len_iter(iter).unwrap(); assert_eq!(a.len(), 2); } diff --git a/tests/it/array/binary/mutable.rs b/tests/it/array/binary/mutable.rs index 0d388b24162..a62462c316d 100644 --- a/tests/it/array/binary/mutable.rs +++ b/tests/it/array/binary/mutable.rs @@ -1,8 +1,8 @@ use std::ops::Deref; -use arrow2::array::{BinaryArray, MutableArray, MutableBinaryArray, TryExtendFromSelf}; -use arrow2::bitmap::Bitmap; -use arrow2::error::Error; +use re_arrow2::array::{BinaryArray, MutableArray, MutableBinaryArray, TryExtendFromSelf}; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::error::Error; #[test] fn new() { diff --git a/tests/it/array/binary/mutable_values.rs b/tests/it/array/binary/mutable_values.rs index 0bf532bc21c..25cd6abc15a 100644 --- a/tests/it/array/binary/mutable_values.rs +++ b/tests/it/array/binary/mutable_values.rs @@ -1,6 +1,6 @@ -use arrow2::array::MutableArray; -use arrow2::array::MutableBinaryValuesArray; -use arrow2::datatypes::DataType; +use re_arrow2::array::MutableArray; +use re_arrow2::array::MutableBinaryValuesArray; +use re_arrow2::datatypes::DataType; #[test] fn capacity() { diff --git a/tests/it/array/binary/to_mutable.rs b/tests/it/array/binary/to_mutable.rs index 1773c83a362..56ddba21a7e 100644 --- a/tests/it/array/binary/to_mutable.rs +++ b/tests/it/array/binary/to_mutable.rs @@ -1,4 +1,4 @@ -use arrow2::{array::BinaryArray, bitmap::Bitmap, buffer::Buffer, datatypes::DataType}; +use re_arrow2::{array::BinaryArray, bitmap::Bitmap, buffer::Buffer, datatypes::DataType}; #[test] fn not_shared() { diff --git a/tests/it/array/boolean/mod.rs b/tests/it/array/boolean/mod.rs index cd6ad0c77af..f78bd39d23a 100644 --- a/tests/it/array/boolean/mod.rs +++ b/tests/it/array/boolean/mod.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::{Array, BooleanArray}, bitmap::Bitmap, datatypes::DataType, @@ -109,7 +109,7 @@ fn try_from_trusted_len_iter() { let iter = std::iter::repeat(true) .take(2) .map(Some) - .map(arrow2::error::Result::Ok); + .map(re_arrow2::error::Result::Ok); let a = BooleanArray::try_from_trusted_len_iter(iter.clone()).unwrap(); assert_eq!(a.len(), 2); let a = unsafe { BooleanArray::try_from_trusted_len_iter_unchecked(iter).unwrap() }; diff --git a/tests/it/array/boolean/mutable.rs b/tests/it/array/boolean/mutable.rs index 1f1d85631a5..cbd3c94ed47 100644 --- a/tests/it/array/boolean/mutable.rs +++ b/tests/it/array/boolean/mutable.rs @@ -1,7 +1,7 @@ -use arrow2::array::{MutableArray, MutableBooleanArray, TryExtendFromSelf}; -use arrow2::bitmap::MutableBitmap; -use arrow2::datatypes::DataType; -use arrow2::error::Result; +use re_arrow2::array::{MutableArray, MutableBooleanArray, TryExtendFromSelf}; +use re_arrow2::bitmap::MutableBitmap; +use re_arrow2::datatypes::DataType; +use re_arrow2::error::Result; #[test] fn set() { diff --git a/tests/it/array/dictionary/mod.rs b/tests/it/array/dictionary/mod.rs index 0ee0c374764..d18ac327a1b 100644 --- a/tests/it/array/dictionary/mod.rs +++ b/tests/it/array/dictionary/mod.rs @@ -1,12 +1,17 @@ mod mutable; -use arrow2::{array::*, datatypes::DataType}; +use std::sync::Arc; + +use re_arrow2::{array::*, datatypes::DataType}; #[test] fn try_new_ok() { let values = Utf8Array::::from_slice(["a", "aa"]); - let data_type = - DataType::Dictionary(i32::KEY_TYPE, Box::new(values.data_type().clone()), false); + let data_type = DataType::Dictionary( + i32::KEY_TYPE, + std::sync::Arc::new(values.data_type().clone()), + false, + ); let array = DictionaryArray::try_new( data_type, PrimitiveArray::from_vec(vec![1, 0]), @@ -27,8 +32,11 @@ fn try_new_ok() { #[test] fn try_new_incorrect_key() { let values = Utf8Array::::from_slice(["a", "aa"]); - let data_type = - DataType::Dictionary(i16::KEY_TYPE, Box::new(values.data_type().clone()), false); + let data_type = DataType::Dictionary( + i16::KEY_TYPE, + std::sync::Arc::new(values.data_type().clone()), + false, + ); let r = DictionaryArray::try_new( data_type, @@ -47,8 +55,11 @@ fn try_new_nulls() { let value: &[&str] = &[]; let values = Utf8Array::::from_slice(value); - let data_type = - DataType::Dictionary(u32::KEY_TYPE, Box::new(values.data_type().clone()), false); + let data_type = DataType::Dictionary( + u32::KEY_TYPE, + std::sync::Arc::new(values.data_type().clone()), + false, + ); let r = DictionaryArray::try_new(data_type, keys, values.boxed()).is_ok(); assert!(r); @@ -72,7 +83,7 @@ fn try_new_incorrect_dt() { #[test] fn try_new_incorrect_values_dt() { let values = Utf8Array::::from_slice(["a", "aa"]); - let data_type = DataType::Dictionary(i32::KEY_TYPE, Box::new(DataType::LargeUtf8), false); + let data_type = DataType::Dictionary(i32::KEY_TYPE, Arc::new(DataType::LargeUtf8), false); let r = DictionaryArray::try_new( data_type, @@ -106,7 +117,7 @@ fn try_new_out_of_bounds_neg() { #[test] fn new_null() { - let dt = DataType::Dictionary(i16::KEY_TYPE, Box::new(DataType::Int32), false); + let dt = DataType::Dictionary(i16::KEY_TYPE, Arc::new(DataType::Int32), false); let array = DictionaryArray::::new_null(dt, 2); assert_eq!(format!("{array:?}"), "DictionaryArray[None, None]"); @@ -114,7 +125,7 @@ fn new_null() { #[test] fn new_empty() { - let dt = DataType::Dictionary(i16::KEY_TYPE, Box::new(DataType::Int32), false); + let dt = DataType::Dictionary(i16::KEY_TYPE, Arc::new(DataType::Int32), false); let array = DictionaryArray::::new_empty(dt); assert_eq!(format!("{array:?}"), "DictionaryArray[]"); diff --git a/tests/it/array/dictionary/mutable.rs b/tests/it/array/dictionary/mutable.rs index a7845114d9b..450172ae1c3 100644 --- a/tests/it/array/dictionary/mutable.rs +++ b/tests/it/array/dictionary/mutable.rs @@ -3,9 +3,9 @@ use std::collections::HashSet; use std::fmt::Debug; use std::hash::Hash; -use arrow2::array::indexable::{AsIndexed, Indexable}; -use arrow2::array::*; -use arrow2::error::Result; +use re_arrow2::array::indexable::{AsIndexed, Indexable}; +use re_arrow2::array::*; +use re_arrow2::error::Result; #[test] fn primitive() -> Result<()> { diff --git a/tests/it/array/equal/boolean.rs b/tests/it/array/equal/boolean.rs index 9a43f226e6c..0e102f39d8f 100644 --- a/tests/it/array/equal/boolean.rs +++ b/tests/it/array/equal/boolean.rs @@ -1,4 +1,4 @@ -use arrow2::array::*; +use re_arrow2::array::*; use super::test_equal; diff --git a/tests/it/array/equal/dictionary.rs b/tests/it/array/equal/dictionary.rs index 8e25d083e1a..18b0ee47470 100644 --- a/tests/it/array/equal/dictionary.rs +++ b/tests/it/array/equal/dictionary.rs @@ -1,4 +1,4 @@ -use arrow2::array::*; +use re_arrow2::array::*; use super::test_equal; diff --git a/tests/it/array/equal/fixed_size_list.rs b/tests/it/array/equal/fixed_size_list.rs index 3df32c574f2..864e4f5b47c 100644 --- a/tests/it/array/equal/fixed_size_list.rs +++ b/tests/it/array/equal/fixed_size_list.rs @@ -1,4 +1,4 @@ -use arrow2::array::{ +use re_arrow2::array::{ FixedSizeListArray, MutableFixedSizeListArray, MutablePrimitiveArray, TryExtend, }; diff --git a/tests/it/array/equal/list.rs b/tests/it/array/equal/list.rs index d54c59de6e9..70676c020d1 100644 --- a/tests/it/array/equal/list.rs +++ b/tests/it/array/equal/list.rs @@ -1,6 +1,6 @@ -use arrow2::array::{Int32Array, ListArray, MutableListArray, MutablePrimitiveArray, TryExtend}; -use arrow2::bitmap::Bitmap; -use arrow2::datatypes::DataType; +use re_arrow2::array::{Int32Array, ListArray, MutableListArray, MutablePrimitiveArray, TryExtend}; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::datatypes::DataType; use super::test_equal; diff --git a/tests/it/array/equal/mod.rs b/tests/it/array/equal/mod.rs index f479b478843..c78a84748d5 100644 --- a/tests/it/array/equal/mod.rs +++ b/tests/it/array/equal/mod.rs @@ -1,4 +1,4 @@ -use arrow2::array::*; +use re_arrow2::array::*; mod dictionary; mod fixed_size_list; diff --git a/tests/it/array/equal/primitive.rs b/tests/it/array/equal/primitive.rs index 58793bc8c3b..62c96daba8d 100644 --- a/tests/it/array/equal/primitive.rs +++ b/tests/it/array/equal/primitive.rs @@ -1,4 +1,4 @@ -use arrow2::array::*; +use re_arrow2::array::*; use super::test_equal; diff --git a/tests/it/array/equal/utf8.rs b/tests/it/array/equal/utf8.rs index 7d4e725c0bf..712bf221c7c 100644 --- a/tests/it/array/equal/utf8.rs +++ b/tests/it/array/equal/utf8.rs @@ -1,5 +1,5 @@ -use arrow2::array::*; -use arrow2::offset::Offset; +use re_arrow2::array::*; +use re_arrow2::offset::Offset; use super::{binary_cases, test_equal}; diff --git a/tests/it/array/fixed_size_binary/mod.rs b/tests/it/array/fixed_size_binary/mod.rs index c5524248ff5..5572639f52e 100644 --- a/tests/it/array/fixed_size_binary/mod.rs +++ b/tests/it/array/fixed_size_binary/mod.rs @@ -1,4 +1,6 @@ -use arrow2::{array::FixedSizeBinaryArray, bitmap::Bitmap, buffer::Buffer, datatypes::DataType}; +use std::sync::Arc; + +use re_arrow2::{array::FixedSizeBinaryArray, bitmap::Bitmap, buffer::Buffer, datatypes::DataType}; mod mutable; @@ -89,7 +91,7 @@ fn to() { let extension = DataType::Extension( "a".to_string(), - Box::new(DataType::FixedSizeBinary(2)), + Arc::new(DataType::FixedSizeBinary(2)), None, ); let _ = a.to(extension); diff --git a/tests/it/array/fixed_size_binary/mutable.rs b/tests/it/array/fixed_size_binary/mutable.rs index ad2ea25b3fd..cb402a67de2 100644 --- a/tests/it/array/fixed_size_binary/mutable.rs +++ b/tests/it/array/fixed_size_binary/mutable.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::bitmap::{Bitmap, MutableBitmap}; -use arrow2::datatypes::DataType; +use re_arrow2::array::*; +use re_arrow2::bitmap::{Bitmap, MutableBitmap}; +use re_arrow2::datatypes::DataType; #[test] fn basic() { diff --git a/tests/it/array/fixed_size_list/mod.rs b/tests/it/array/fixed_size_list/mod.rs index c2a4b11e62c..394022ed141 100644 --- a/tests/it/array/fixed_size_list/mod.rs +++ b/tests/it/array/fixed_size_list/mod.rs @@ -1,6 +1,8 @@ mod mutable; -use arrow2::{ +use std::sync::Arc; + +use re_arrow2::{ array::*, bitmap::Bitmap, datatypes::{DataType, Field}, @@ -11,7 +13,7 @@ fn data() -> FixedSizeListArray { FixedSizeListArray::try_new( DataType::FixedSizeList( - Box::new(Field::new("a", values.data_type().clone(), true)), + Arc::new(Field::new("a", values.data_type().clone(), true)), 2, ), values.boxed(), @@ -53,7 +55,7 @@ fn debug() { #[test] fn empty() { let array = FixedSizeListArray::new_empty(DataType::FixedSizeList( - Box::new(Field::new("a", DataType::Int32, true)), + Arc::new(Field::new("a", DataType::Int32, true)), 2, )); assert_eq!(array.values().len(), 0); @@ -63,7 +65,10 @@ fn empty() { #[test] fn null() { let array = FixedSizeListArray::new_null( - DataType::FixedSizeList(Box::new(Field::new("a", DataType::Int32, true)), 2), + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("a", DataType::Int32, true)), + 2, + ), 2, ); assert_eq!(array.values().len(), 4); @@ -74,7 +79,10 @@ fn null() { fn wrong_size() { let values = Int32Array::from_slice([10, 20, 0]); assert!(FixedSizeListArray::try_new( - DataType::FixedSizeList(Box::new(Field::new("a", DataType::Int32, true)), 2), + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("a", DataType::Int32, true)), + 2 + ), values.boxed(), None ) @@ -85,7 +93,10 @@ fn wrong_size() { fn wrong_len() { let values = Int32Array::from_slice([10, 20, 0]); assert!(FixedSizeListArray::try_new( - DataType::FixedSizeList(Box::new(Field::new("a", DataType::Int32, true)), 2), + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("a", DataType::Int32, true)), + 2 + ), values.boxed(), Some([true, false, false].into()), // it should be 2 ) diff --git a/tests/it/array/fixed_size_list/mutable.rs b/tests/it/array/fixed_size_list/mutable.rs index f7a8784dfce..0d113833538 100644 --- a/tests/it/array/fixed_size_list/mutable.rs +++ b/tests/it/array/fixed_size_list/mutable.rs @@ -1,5 +1,7 @@ -use arrow2::array::*; -use arrow2::datatypes::{DataType, Field}; +use std::sync::Arc; + +use re_arrow2::array::*; +use re_arrow2::datatypes::{DataType, Field}; #[test] fn primitive() { @@ -46,7 +48,7 @@ fn new_with_field() { assert_eq!( list.data_type(), &DataType::FixedSizeList( - Box::new(Field::new("custom_items", DataType::Int32, false)), + Arc::new(Field::new("custom_items", DataType::Int32, false)), 3 ) ); diff --git a/tests/it/array/growable/binary.rs b/tests/it/array/growable/binary.rs index 5b8bb3e9349..3f818a876f1 100644 --- a/tests/it/array/growable/binary.rs +++ b/tests/it/array/growable/binary.rs @@ -1,4 +1,4 @@ -use arrow2::array::{ +use re_arrow2::array::{ growable::{Growable, GrowableBinary}, BinaryArray, }; diff --git a/tests/it/array/growable/boolean.rs b/tests/it/array/growable/boolean.rs index b52110b5d94..22291a9b896 100644 --- a/tests/it/array/growable/boolean.rs +++ b/tests/it/array/growable/boolean.rs @@ -1,5 +1,5 @@ -use arrow2::array::growable::{Growable, GrowableBoolean}; -use arrow2::array::BooleanArray; +use re_arrow2::array::growable::{Growable, GrowableBoolean}; +use re_arrow2::array::BooleanArray; #[test] fn test_bool() { diff --git a/tests/it/array/growable/dictionary.rs b/tests/it/array/growable/dictionary.rs index d4c75922de2..9b60691b190 100644 --- a/tests/it/array/growable/dictionary.rs +++ b/tests/it/array/growable/dictionary.rs @@ -1,6 +1,6 @@ -use arrow2::array::growable::{Growable, GrowableDictionary}; -use arrow2::array::*; -use arrow2::error::Result; +use re_arrow2::array::growable::{Growable, GrowableDictionary}; +use re_arrow2::array::*; +use re_arrow2::error::Result; #[test] fn test_single() -> Result<()> { diff --git a/tests/it/array/growable/fixed_binary.rs b/tests/it/array/growable/fixed_binary.rs index c3bcb630551..31743d6c310 100644 --- a/tests/it/array/growable/fixed_binary.rs +++ b/tests/it/array/growable/fixed_binary.rs @@ -1,4 +1,4 @@ -use arrow2::array::{ +use re_arrow2::array::{ growable::{Growable, GrowableFixedSizeBinary}, FixedSizeBinaryArray, }; diff --git a/tests/it/array/growable/fixed_size_list.rs b/tests/it/array/growable/fixed_size_list.rs index d48e40338c5..91193871f31 100644 --- a/tests/it/array/growable/fixed_size_list.rs +++ b/tests/it/array/growable/fixed_size_list.rs @@ -1,4 +1,4 @@ -use arrow2::array::{ +use re_arrow2::array::{ growable::{Growable, GrowableFixedSizeList}, FixedSizeListArray, MutableFixedSizeListArray, MutablePrimitiveArray, TryExtend, }; diff --git a/tests/it/array/growable/list.rs b/tests/it/array/growable/list.rs index 45006b6e3d6..ac578d86aa4 100644 --- a/tests/it/array/growable/list.rs +++ b/tests/it/array/growable/list.rs @@ -1,4 +1,6 @@ -use arrow2::{ +use std::sync::Arc; + +use re_arrow2::{ array::{ growable::{Growable, GrowableList}, Array, ListArray, MutableListArray, MutablePrimitiveArray, TryExtend, @@ -23,7 +25,7 @@ fn extension() { let array = create_list_array(data); let data_type = - DataType::Extension("ext".to_owned(), Box::new(array.data_type().clone()), None); + DataType::Extension("ext".to_owned(), Arc::new(array.data_type().clone()), None); let array_ext = ListArray::new( data_type, array.offsets().clone(), diff --git a/tests/it/array/growable/map.rs b/tests/it/array/growable/map.rs index de9069f68bc..04baf3117a0 100644 --- a/tests/it/array/growable/map.rs +++ b/tests/it/array/growable/map.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::{ growable::{Growable, GrowableMap}, Array, MapArray, PrimitiveArray, StructArray, Utf8Array, @@ -29,7 +29,10 @@ fn some_values() -> (DataType, Vec>) { Field::new("key", DataType::Utf8, true), Field::new("val", DataType::Int32, true), ]; - (DataType::Struct(fields), vec![strings, ints]) + ( + DataType::Struct(std::sync::Arc::new(fields)), + vec![strings, ints], + ) } #[test] @@ -38,7 +41,7 @@ fn basic() { let kv_array = StructArray::new(fields.clone(), values, None).boxed(); let kv_field = Field::new("kv", fields, false); - let data_type = DataType::Map(Box::new(kv_field), false); + let data_type = DataType::Map(std::sync::Arc::new(kv_field), false); let offsets = OffsetsBuffer::try_from(vec![0, 1, 2, 4, 6]).unwrap(); let array = MapArray::new(data_type.clone(), offsets, kv_array.clone(), None); @@ -62,7 +65,7 @@ fn offset() { let kv_array = StructArray::new(fields.clone(), values, None).boxed(); let kv_field = Field::new("kv", fields, false); - let data_type = DataType::Map(Box::new(kv_field), false); + let data_type = DataType::Map(std::sync::Arc::new(kv_field), false); let offsets = OffsetsBuffer::try_from(vec![0, 1, 2, 4, 6]).unwrap(); let array = MapArray::new(data_type.clone(), offsets, kv_array.clone(), None).sliced(1, 3); @@ -86,7 +89,7 @@ fn nulls() { let kv_array = StructArray::new(fields.clone(), values, None).boxed(); let kv_field = Field::new("kv", fields, false); - let data_type = DataType::Map(Box::new(kv_field), false); + let data_type = DataType::Map(std::sync::Arc::new(kv_field), false); let offsets = OffsetsBuffer::try_from(vec![0, 1, 2, 4, 6]).unwrap(); let array = MapArray::new( diff --git a/tests/it/array/growable/mod.rs b/tests/it/array/growable/mod.rs index d4b034a13e6..4597f06321b 100644 --- a/tests/it/array/growable/mod.rs +++ b/tests/it/array/growable/mod.rs @@ -11,9 +11,11 @@ mod struct_; mod union; mod utf8; -use arrow2::array::growable::make_growable; -use arrow2::array::*; -use arrow2::datatypes::{DataType, Field}; +use std::sync::Arc; + +use re_arrow2::array::growable::make_growable; +use re_arrow2::array::*; +use re_arrow2::datatypes::{DataType, Field}; #[test] fn test_make_growable() { @@ -49,18 +51,18 @@ fn test_make_growable_extension() { .unwrap(); make_growable(&[&array], false, 2); - let data_type = DataType::Extension("ext".to_owned(), Box::new(DataType::Int32), None); + let data_type = DataType::Extension("ext".to_owned(), Arc::new(DataType::Int32), None); let array = Int32Array::from_slice([1, 2]).to(data_type.clone()); let array_grown = make_growable(&[&array], false, 2).as_box(); assert_eq!(array_grown.data_type(), &data_type); let data_type = DataType::Extension( "ext".to_owned(), - Box::new(DataType::Struct(vec![Field::new( + Arc::new(DataType::Struct(Arc::new(vec![Field::new( "a", DataType::Int32, false, - )])), + )]))), None, ); let array = StructArray::new( diff --git a/tests/it/array/growable/null.rs b/tests/it/array/growable/null.rs index 1298f3f5686..a27691f51fd 100644 --- a/tests/it/array/growable/null.rs +++ b/tests/it/array/growable/null.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::{ growable::{Growable, GrowableNull}, NullArray, diff --git a/tests/it/array/growable/primitive.rs b/tests/it/array/growable/primitive.rs index f308a0c49b5..f0fcd0b1efa 100644 --- a/tests/it/array/growable/primitive.rs +++ b/tests/it/array/growable/primitive.rs @@ -1,4 +1,4 @@ -use arrow2::array::{ +use re_arrow2::array::{ growable::{Growable, GrowablePrimitive}, PrimitiveArray, }; diff --git a/tests/it/array/growable/struct_.rs b/tests/it/array/growable/struct_.rs index 9596f23961d..1408cf978f1 100644 --- a/tests/it/array/growable/struct_.rs +++ b/tests/it/array/growable/struct_.rs @@ -1,9 +1,9 @@ -use arrow2::array::{ +use re_arrow2::array::{ growable::{Growable, GrowableStruct}, Array, PrimitiveArray, StructArray, Utf8Array, }; -use arrow2::bitmap::Bitmap; -use arrow2::datatypes::{DataType, Field}; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::datatypes::{DataType, Field}; fn some_values() -> (DataType, Vec>) { let strings: Box = Box::new(Utf8Array::::from([ @@ -24,7 +24,10 @@ fn some_values() -> (DataType, Vec>) { Field::new("f1", DataType::Utf8, true), Field::new("f2", DataType::Int32, true), ]; - (DataType::Struct(fields), vec![strings, ints]) + ( + DataType::Struct(std::sync::Arc::new(fields)), + vec![strings, ints], + ) } #[test] diff --git a/tests/it/array/growable/union.rs b/tests/it/array/growable/union.rs index 21cbbebabf4..6d151c7f4d6 100644 --- a/tests/it/array/growable/union.rs +++ b/tests/it/array/growable/union.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::{ growable::{Growable, GrowableUnion}, *, @@ -13,7 +13,7 @@ fn sparse() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let types = vec![0, 0, 1].into(); let fields = vec![ Int32Array::from(&[Some(1), None, Some(2)]).boxed(), @@ -45,7 +45,7 @@ fn dense() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Dense); let types = vec![0, 0, 1].into(); let fields = vec![ Int32Array::from(&[Some(1), None, Some(2)]).boxed(), @@ -74,8 +74,10 @@ fn dense() -> Result<()> { #[test] fn complex_dense() -> Result<()> { - let fixed_size_type = - DataType::FixedSizeList(Box::new(Field::new("i", DataType::UInt16, true)), 3); + let fixed_size_type = DataType::FixedSizeList( + std::sync::Arc::new(Field::new("i", DataType::UInt16, true)), + 3, + ); let fields = vec![ Field::new("a", DataType::Int32, true), @@ -83,7 +85,7 @@ fn complex_dense() -> Result<()> { Field::new("c", fixed_size_type.clone(), true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Dense); // UnionArray[1, [11, 12, 13], abcd, [21, 22, 23], 2] let types = vec![0, 2, 1, 2, 0].into(); diff --git a/tests/it/array/growable/utf8.rs b/tests/it/array/growable/utf8.rs index 2568116dc3a..1a2d59a3dd5 100644 --- a/tests/it/array/growable/utf8.rs +++ b/tests/it/array/growable/utf8.rs @@ -1,4 +1,4 @@ -use arrow2::array::{ +use re_arrow2::array::{ growable::{Growable, GrowableUtf8}, Utf8Array, }; diff --git a/tests/it/array/list/mod.rs b/tests/it/array/list/mod.rs index 4cbf4dbbece..887ba9cf5ce 100644 --- a/tests/it/array/list/mod.rs +++ b/tests/it/array/list/mod.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::buffer::Buffer; -use arrow2::datatypes::DataType; +use re_arrow2::array::*; +use re_arrow2::buffer::Buffer; +use re_arrow2::datatypes::DataType; mod mutable; diff --git a/tests/it/array/list/mutable.rs b/tests/it/array/list/mutable.rs index e72167277fa..b35403c6e4b 100644 --- a/tests/it/array/list/mutable.rs +++ b/tests/it/array/list/mutable.rs @@ -1,4 +1,4 @@ -use arrow2::{array::*, bitmap::Bitmap, buffer::Buffer, datatypes::DataType}; +use re_arrow2::{array::*, bitmap::Bitmap, buffer::Buffer, datatypes::DataType}; #[test] fn basics() { diff --git a/tests/it/array/map/mod.rs b/tests/it/array/map/mod.rs index 285bc8b39b3..2c84dfab221 100644 --- a/tests/it/array/map/mod.rs +++ b/tests/it/array/map/mod.rs @@ -1,15 +1,20 @@ -use arrow2::{ +use std::sync::Arc; + +use re_arrow2::{ array::*, datatypes::{DataType, Field}, }; #[test] fn basics() { - let dt = DataType::Struct(vec![ + let dt = DataType::Struct(Arc::new(vec![ Field::new("a", DataType::Utf8, true), Field::new("b", DataType::Utf8, true), - ]); - let data_type = DataType::Map(Box::new(Field::new("a", dt.clone(), true)), false); + ])); + let data_type = DataType::Map( + std::sync::Arc::new(Field::new("a", dt.clone(), true)), + false, + ); let field = StructArray::new( dt.clone(), diff --git a/tests/it/array/mod.rs b/tests/it/array/mod.rs index 85318ba628a..d08f31cf9f5 100644 --- a/tests/it/array/mod.rs +++ b/tests/it/array/mod.rs @@ -13,9 +13,11 @@ mod struct_; mod union; mod utf8; -use arrow2::array::{clone, new_empty_array, new_null_array, Array, PrimitiveArray}; -use arrow2::bitmap::Bitmap; -use arrow2::datatypes::{DataType, Field, UnionMode}; +use std::sync::Arc; + +use re_arrow2::array::{clone, new_empty_array, new_null_array, Array, PrimitiveArray}; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::datatypes::{DataType, Field, UnionMode}; #[test] fn nulls() { @@ -24,7 +26,7 @@ fn nulls() { DataType::Float64, DataType::Utf8, DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), + DataType::List(std::sync::Arc::new(Field::new("a", DataType::Binary, true))), ]; let a = datatypes .into_iter() @@ -34,12 +36,12 @@ fn nulls() { // unions' null count is always 0 let datatypes = vec![ DataType::Union( - vec![Field::new("a", DataType::Binary, true)], + Arc::new(vec![Field::new("a", DataType::Binary, true)]), None, UnionMode::Dense, ), DataType::Union( - vec![Field::new("a", DataType::Binary, true)], + Arc::new(vec![Field::new("a", DataType::Binary, true)]), None, UnionMode::Sparse, ), @@ -57,23 +59,23 @@ fn empty() { DataType::Float64, DataType::Utf8, DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new("a", DataType::Binary, true))), + DataType::List(std::sync::Arc::new(Field::new( "a", - DataType::Extension("ext".to_owned(), Box::new(DataType::Int32), None), + DataType::Extension("ext".to_owned(), Arc::new(DataType::Int32), None), true, ))), DataType::Union( - vec![Field::new("a", DataType::Binary, true)], + Arc::new(vec![Field::new("a", DataType::Binary, true)]), None, UnionMode::Sparse, ), DataType::Union( - vec![Field::new("a", DataType::Binary, true)], + Arc::new(vec![Field::new("a", DataType::Binary, true)]), None, UnionMode::Dense, ), - DataType::Struct(vec![Field::new("a", DataType::Int32, true)]), + DataType::Struct(Arc::new(vec![Field::new("a", DataType::Int32, true)])), ]; let a = datatypes.into_iter().all(|x| new_empty_array(x).len() == 0); assert!(a); @@ -86,22 +88,22 @@ fn empty_extension() { DataType::Float64, DataType::Utf8, DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), + DataType::List(std::sync::Arc::new(Field::new("a", DataType::Binary, true))), DataType::Union( - vec![Field::new("a", DataType::Binary, true)], + Arc::new(vec![Field::new("a", DataType::Binary, true)]), None, UnionMode::Sparse, ), DataType::Union( - vec![Field::new("a", DataType::Binary, true)], + Arc::new(vec![Field::new("a", DataType::Binary, true)]), None, UnionMode::Dense, ), - DataType::Struct(vec![Field::new("a", DataType::Int32, true)]), + DataType::Struct(Arc::new(vec![Field::new("a", DataType::Int32, true)])), ]; let a = datatypes .into_iter() - .map(|dt| DataType::Extension("ext".to_owned(), Box::new(dt), None)) + .map(|dt| DataType::Extension("ext".to_owned(), Arc::new(dt), None)) .all(|x| { let a = new_empty_array(x); a.len() == 0 && matches!(a.data_type(), DataType::Extension(_, _, _)) @@ -116,7 +118,7 @@ fn test_clone() { DataType::Float64, DataType::Utf8, DataType::Binary, - DataType::List(Box::new(Field::new("a", DataType::Binary, true))), + DataType::List(std::sync::Arc::new(Field::new("a", DataType::Binary, true))), ]; let a = datatypes .into_iter() diff --git a/tests/it/array/ord.rs b/tests/it/array/ord.rs index 3ce249af451..096916d4975 100644 --- a/tests/it/array/ord.rs +++ b/tests/it/array/ord.rs @@ -1,9 +1,9 @@ use std::cmp::Ordering; -use arrow2::array::ord::build_compare; -use arrow2::array::*; -use arrow2::datatypes::DataType; -use arrow2::error::Result; +use re_arrow2::array::ord::build_compare; +use re_arrow2::array::*; +use re_arrow2::datatypes::DataType; +use re_arrow2::error::Result; #[test] fn i32() -> Result<()> { diff --git a/tests/it/array/primitive/fmt.rs b/tests/it/array/primitive/fmt.rs index c04c1e1c5e5..f17aaf0b6c1 100644 --- a/tests/it/array/primitive/fmt.rs +++ b/tests/it/array/primitive/fmt.rs @@ -1,4 +1,6 @@ -use arrow2::{ +use std::sync::Arc; + +use re_arrow2::{ array::*, datatypes::*, types::{days_ms, months_days_ns}, @@ -118,7 +120,7 @@ fn debug_timestamp_ns() { fn debug_timestamp_tz_ns() { let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Timestamp( TimeUnit::Nanosecond, - Some("+02:00".to_string()), + Some(Arc::new("+02:00".to_string())), )); assert_eq!( format!("{array:?}"), @@ -130,7 +132,7 @@ fn debug_timestamp_tz_ns() { fn debug_timestamp_tz_not_parsable() { let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Timestamp( TimeUnit::Nanosecond, - Some("aa".to_string()), + Some(Arc::new("aa".to_string())), )); assert_eq!( format!("{array:?}"), @@ -143,7 +145,7 @@ fn debug_timestamp_tz_not_parsable() { fn debug_timestamp_tz1_ns() { let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Timestamp( TimeUnit::Nanosecond, - Some("Europe/Lisbon".to_string()), + Some(Arc::new("Europe/Lisbon".to_string())), )); assert_eq!( format!("{array:?}"), diff --git a/tests/it/array/primitive/mod.rs b/tests/it/array/primitive/mod.rs index e8a78435311..2c84b763c2b 100644 --- a/tests/it/array/primitive/mod.rs +++ b/tests/it/array/primitive/mod.rs @@ -1,6 +1,6 @@ use std::iter::FromIterator; -use arrow2::{array::*, bitmap::Bitmap, buffer::Buffer, datatypes::*, types::months_days_ns}; +use re_arrow2::{array::*, bitmap::Bitmap, buffer::Buffer, datatypes::*, types::months_days_ns}; mod fmt; mod mutable; diff --git a/tests/it/array/primitive/mutable.rs b/tests/it/array/primitive/mutable.rs index d7e0b86c061..368409fdf73 100644 --- a/tests/it/array/primitive/mutable.rs +++ b/tests/it/array/primitive/mutable.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::*, bitmap::{Bitmap, MutableBitmap}, datatypes::DataType, diff --git a/tests/it/array/primitive/to_mutable.rs b/tests/it/array/primitive/to_mutable.rs index ee3a1f8ee34..524723e1fbd 100644 --- a/tests/it/array/primitive/to_mutable.rs +++ b/tests/it/array/primitive/to_mutable.rs @@ -1,7 +1,7 @@ -use arrow2::array::PrimitiveArray; -use arrow2::bitmap::Bitmap; -use arrow2::datatypes::DataType; use either::Either; +use re_arrow2::array::PrimitiveArray; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::datatypes::DataType; #[test] fn array_to_mutable() { diff --git a/tests/it/array/struct_/iterator.rs b/tests/it/array/struct_/iterator.rs index be4a5eefbb4..921cf3fd9b5 100644 --- a/tests/it/array/struct_/iterator.rs +++ b/tests/it/array/struct_/iterator.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::datatypes::*; -use arrow2::scalar::new_scalar; +use re_arrow2::array::*; +use re_arrow2::datatypes::*; +use re_arrow2::scalar::new_scalar; #[test] fn test_simple_iter() { @@ -13,7 +13,7 @@ fn test_simple_iter() { ]; let array = StructArray::new( - DataType::Struct(fields), + DataType::Struct(std::sync::Arc::new(fields)), vec![boolean.clone(), int.clone()], None, ); diff --git a/tests/it/array/struct_/mod.rs b/tests/it/array/struct_/mod.rs index cd32eee3f75..47c30298427 100644 --- a/tests/it/array/struct_/mod.rs +++ b/tests/it/array/struct_/mod.rs @@ -1,9 +1,9 @@ mod iterator; mod mutable; -use arrow2::array::*; -use arrow2::bitmap::Bitmap; -use arrow2::datatypes::*; +use re_arrow2::array::*; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::datatypes::*; #[test] fn debug() { @@ -16,7 +16,7 @@ fn debug() { ]; let array = StructArray::new( - DataType::Struct(fields), + DataType::Struct(std::sync::Arc::new(fields)), vec![boolean.clone(), int.clone()], Some(Bitmap::from([true, true, false, true])), ); diff --git a/tests/it/array/struct_/mutable.rs b/tests/it/array/struct_/mutable.rs index 19f2f12f15e..9aad50d7315 100644 --- a/tests/it/array/struct_/mutable.rs +++ b/tests/it/array/struct_/mutable.rs @@ -1,4 +1,6 @@ -use arrow2::{ +use std::sync::Arc; + +use re_arrow2::{ array::*, datatypes::{DataType, Field}, }; @@ -7,7 +9,7 @@ use arrow2::{ fn push() { let c1 = Box::new(MutablePrimitiveArray::::new()) as Box; let values = vec![c1]; - let data_type = DataType::Struct(vec![Field::new("f1", DataType::Int32, true)]); + let data_type = DataType::Struct(Arc::new(vec![Field::new("f1", DataType::Int32, true)])); let mut a = MutableStructArray::new(data_type, values); a.value::>(0) diff --git a/tests/it/array/union.rs b/tests/it/array/union.rs index 4a8c3aee214..ba63934fc61 100644 --- a/tests/it/array/union.rs +++ b/tests/it/array/union.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::*, buffer::Buffer, datatypes::*, @@ -25,7 +25,7 @@ fn sparse_debug() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let types = vec![0, 0, 1].into(); let fields = vec![ Int32Array::from(&[Some(1), None, Some(2)]).boxed(), @@ -45,7 +45,7 @@ fn dense_debug() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Dense); let types = vec![0, 0, 1].into(); let fields = vec![ Int32Array::from(&[Some(1), None, Some(2)]).boxed(), @@ -66,7 +66,7 @@ fn slice() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let types = Buffer::from(vec![0, 0, 1]); let fields = vec![ Int32Array::from(&[Some(1), None, Some(2)]).boxed(), @@ -94,7 +94,7 @@ fn iter_sparse() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let types = Buffer::from(vec![0, 0, 1]); let fields = vec![ Int32Array::from(&[Some(1), None, Some(2)]).boxed(), @@ -127,7 +127,7 @@ fn iter_dense() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Dense); let types = Buffer::from(vec![0, 0, 1]); let offsets = Buffer::::from(vec![0, 1, 0]); let fields = vec![ @@ -161,7 +161,7 @@ fn iter_sparse_slice() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let types = Buffer::from(vec![0, 0, 1]); let fields = vec![ Int32Array::from(&[Some(1), Some(3), Some(2)]).boxed(), @@ -187,7 +187,7 @@ fn iter_dense_slice() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Dense); let types = Buffer::from(vec![0, 0, 1]); let offsets = Buffer::::from(vec![0, 1, 0]); let fields = vec![ @@ -214,7 +214,7 @@ fn scalar() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Dense); let types = Buffer::from(vec![0, 0, 1]); let offsets = Buffer::::from(vec![0, 1, 0]); let fields = vec![ @@ -271,7 +271,7 @@ fn dense_without_offsets_is_error() { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Dense); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Dense); let types = vec![0, 0, 1].into(); let fields = vec![ Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), @@ -287,7 +287,7 @@ fn fields_must_match() { Field::new("a", DataType::Int64, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let types = vec![0, 0, 1].into(); let fields = vec![ Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), @@ -303,7 +303,7 @@ fn sparse_with_offsets_is_error() { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let fields = vec![ Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), @@ -321,7 +321,7 @@ fn offsets_must_be_in_bounds() { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let fields = vec![ Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), @@ -340,7 +340,7 @@ fn sparse_with_wrong_offsets1_is_error() { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let fields = vec![ Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), @@ -359,7 +359,7 @@ fn types_must_be_in_bounds() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let fields = vec![ Int32Array::from([Some(1), Some(3), Some(2)]).boxed(), Utf8Array::::from([Some("a"), Some("b"), Some("c")]).boxed(), diff --git a/tests/it/array/utf8/mod.rs b/tests/it/array/utf8/mod.rs index 9a437bb8681..7a35c2639df 100644 --- a/tests/it/array/utf8/mod.rs +++ b/tests/it/array/utf8/mod.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::*, bitmap::Bitmap, buffer::Buffer, datatypes::DataType, error::Result, offset::OffsetsBuffer, }; diff --git a/tests/it/array/utf8/mutable.rs b/tests/it/array/utf8/mutable.rs index b33fb59966e..52939fb4433 100644 --- a/tests/it/array/utf8/mutable.rs +++ b/tests/it/array/utf8/mutable.rs @@ -1,6 +1,6 @@ -use arrow2::array::{MutableArray, MutableUtf8Array, TryExtendFromSelf, Utf8Array}; -use arrow2::bitmap::Bitmap; -use arrow2::datatypes::DataType; +use re_arrow2::array::{MutableArray, MutableUtf8Array, TryExtendFromSelf, Utf8Array}; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::datatypes::DataType; #[test] fn capacities() { @@ -135,7 +135,7 @@ fn test_extend_values() { fn test_extend() { let mut array = MutableUtf8Array::::new(); - array.extend([Some("hi"), None, Some("there"), None].into_iter()); + array.extend([Some("hi"), None, Some("there"), None]); let array: Utf8Array = array.into(); @@ -149,7 +149,7 @@ fn test_extend() { fn as_arc() { let mut array = MutableUtf8Array::::new(); - array.extend([Some("hi"), None, Some("there"), None].into_iter()); + array.extend([Some("hi"), None, Some("there"), None]); assert_eq!( Utf8Array::::from([Some("hi"), None, Some("there"), None]), diff --git a/tests/it/array/utf8/mutable_values.rs b/tests/it/array/utf8/mutable_values.rs index 6bf04726a36..fde91fe1861 100644 --- a/tests/it/array/utf8/mutable_values.rs +++ b/tests/it/array/utf8/mutable_values.rs @@ -1,6 +1,6 @@ -use arrow2::array::MutableArray; -use arrow2::array::MutableUtf8ValuesArray; -use arrow2::datatypes::DataType; +use re_arrow2::array::MutableArray; +use re_arrow2::array::MutableUtf8ValuesArray; +use re_arrow2::datatypes::DataType; #[test] fn capacity() { diff --git a/tests/it/array/utf8/to_mutable.rs b/tests/it/array/utf8/to_mutable.rs index 97ee0fb2055..a181b9306b8 100644 --- a/tests/it/array/utf8/to_mutable.rs +++ b/tests/it/array/utf8/to_mutable.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::Utf8Array, bitmap::Bitmap, buffer::Buffer, datatypes::DataType, offset::OffsetsBuffer, }; diff --git a/tests/it/arrow.rs b/tests/it/arrow.rs index 633a2a196f5..7da7c92cf6b 100644 --- a/tests/it/arrow.rs +++ b/tests/it/arrow.rs @@ -1,10 +1,12 @@ -use arrow2::array::*; -use arrow2::bitmap::Bitmap; -use arrow2::datatypes::{DataType, Field, IntegerType, TimeUnit, UnionMode}; -use arrow2::offset::Offsets; +use std::sync::Arc; + use arrow_array::ArrayRef; use arrow_data::ArrayDataBuilder; use proptest::num::i32; +use re_arrow2::array::*; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::datatypes::{DataType, Field, IntegerType, TimeUnit, UnionMode}; +use re_arrow2::offset::Offsets; fn test_arrow2_roundtrip(array: &dyn arrow_array::Array) { let arrow2 = Box::::from(array); @@ -90,7 +92,7 @@ fn test_primitive() { let array = PrimitiveArray::new(data_type, vec![1, 2, 3].into(), None); test_conversion(&array); - let data_type = DataType::Timestamp(TimeUnit::Second, Some("UTC".into())); + let data_type = DataType::Timestamp(TimeUnit::Second, Some(Arc::new("UTC".into()))); let nulls = Bitmap::from_iter([true, true, false]); let array = PrimitiveArray::new(data_type, vec![1_i64, 24, 0].into(), Some(nulls)); test_conversion(&array); @@ -134,17 +136,17 @@ fn make_struct() -> StructArray { let a1 = BinaryArray::::from_iter([Some("s".as_bytes()), Some(b"sd\xFFfk\x23"), None]); let a2 = BinaryArray::::from_iter([Some("45848".as_bytes()), Some(b"\x03\xFF"), None]); - let data_type = DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())); + let data_type = DataType::Timestamp(TimeUnit::Millisecond, Some(Arc::new("UTC".into()))); let nulls = Bitmap::from_iter([true, true, false]); let a3 = PrimitiveArray::new(data_type, vec![1_i64, 24, 0].into(), Some(nulls)); let nulls = [true, true, false].into_iter().collect(); StructArray::new( - DataType::Struct(vec![ + DataType::Struct(Arc::new(vec![ Field::new("a1", a1.data_type().clone(), true), Field::new("a2", a2.data_type().clone(), true), Field::new("a3", a3.data_type().clone(), true), - ]), + ])), vec![Box::new(a1), Box::new(a2), Box::new(a3)], Some(nulls), ) @@ -169,7 +171,11 @@ fn test_list() { let validity = [true, true, false, false, true].into_iter().collect(); let offsets = Offsets::try_from_iter(vec![0, 2, 2, 2, 0]).unwrap(); - let data_type = DataType::List(Box::new(Field::new("element", DataType::Utf8, true))); + let data_type = DataType::List(std::sync::Arc::new(Field::new( + "element", + DataType::Utf8, + true, + ))); let list = ListArray::::new( data_type.clone(), offsets.into(), @@ -184,7 +190,11 @@ fn test_list() { let validity = [true, true, false, false, true].into_iter().collect(); let offsets = Offsets::try_from_iter(vec![0, 2, 2, 2, 0]).unwrap(); - let data_type = DataType::LargeList(Box::new(Field::new("element", DataType::Utf8, true))); + let data_type = DataType::LargeList(std::sync::Arc::new(Field::new( + "element", + DataType::Utf8, + true, + ))); let list = ListArray::::new( data_type.clone(), offsets.into(), @@ -204,7 +214,7 @@ fn test_list_struct() { let validity = [true, true, false, true].into_iter().collect(); let offsets = Offsets::try_from_iter(vec![0, 1, 0, 2]).unwrap(); let list = ListArray::::new( - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new( "element", values.data_type().clone(), true, @@ -225,7 +235,7 @@ fn test_dictionary() { let dictionary = DictionaryArray::try_new( DataType::Dictionary( IntegerType::Int16, - Box::new(values.data_type().clone()), + std::sync::Arc::new(values.data_type().clone()), false, ), keys, @@ -257,7 +267,10 @@ fn test_fixed_size_list() { let nulls = [true, true, false, true].into_iter().collect(); let array = FixedSizeListArray::new( - DataType::FixedSizeList(Box::new(Field::new("element", DataType::Int64, true)), 2), + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("element", DataType::Int64, true)), + 2, + ), Box::new(values), Some(nulls), ); @@ -274,10 +287,10 @@ fn test_map() { ); let values = PrimitiveArray::::from_iter([Some(1), None, Some(3), Some(1), None]); let fields = StructArray::new( - DataType::Struct(vec![ + DataType::Struct(Arc::new(vec![ Field::new("keys", DataType::Utf8, false), // Cannot be nullable Field::new("values", DataType::Int32, true), - ]), + ])), vec![Box::new(keys), Box::new(values)], None, // Cannot be nullable ); @@ -285,7 +298,7 @@ fn test_map() { let validity = [true, true, false, false].into_iter().collect(); let offsets = Offsets::try_from_iter(vec![0, 2, 0, 2]).unwrap(); let data_type = DataType::Map( - Box::new(Field::new("entries", fields.data_type().clone(), true)), + Arc::new(Field::new("entries", fields.data_type().clone(), true)), false, ); let map = MapArray::new( @@ -303,10 +316,10 @@ fn test_map() { #[test] fn test_dense_union() { - let fields = vec![ + let fields = Arc::new(vec![ Field::new("a1", DataType::Int32, true), Field::new("a2", DataType::Int64, true), - ]; + ]); let a1 = PrimitiveArray::from_iter([Some(2), None]); let a2 = PrimitiveArray::from_iter([Some(2_i64), None, Some(3)]); @@ -314,7 +327,7 @@ fn test_dense_union() { let types = vec![1, 0, 0, 1, 1]; let offsets = vec![0, 0, 1, 1, 2]; let union = UnionArray::new( - DataType::Union(fields.clone(), Some(vec![0, 1]), UnionMode::Dense), + DataType::Union(fields.clone(), Some(Arc::new(vec![0, 1])), UnionMode::Dense), types.into(), vec![Box::new(a1.clone()), Box::new(a2.clone())], Some(offsets.into()), @@ -325,7 +338,7 @@ fn test_dense_union() { let types = vec![1, 4, 4, 1, 1]; let offsets = vec![0, 0, 1, 1, 2]; let union = UnionArray::new( - DataType::Union(fields, Some(vec![4, 1]), UnionMode::Dense), + DataType::Union(fields, Some(Arc::new(vec![4, 1])), UnionMode::Dense), types.into(), vec![Box::new(a1), Box::new(a2)], Some(offsets.into()), @@ -336,17 +349,21 @@ fn test_dense_union() { #[test] fn test_sparse_union() { - let fields = vec![ + let fields = Arc::new(vec![ Field::new("a1", DataType::Int32, true), Field::new("a2", DataType::Int64, true), - ]; + ]); let a1 = PrimitiveArray::from_iter([None, Some(2), None, None, None]); let a2 = PrimitiveArray::from_iter([Some(2_i64), None, None, None, Some(3)]); let types = vec![1, 0, 0, 1, 1]; let union = UnionArray::new( - DataType::Union(fields.clone(), Some(vec![0, 1]), UnionMode::Sparse), + DataType::Union( + fields.clone(), + Some(Arc::new(vec![0, 1])), + UnionMode::Sparse, + ), types.into(), vec![Box::new(a1.clone()), Box::new(a2.clone())], None, @@ -356,7 +373,7 @@ fn test_sparse_union() { let types = vec![1, 4, 4, 1, 1]; let union = UnionArray::new( - DataType::Union(fields, Some(vec![4, 1]), UnionMode::Sparse), + DataType::Union(fields, Some(Arc::new(vec![4, 1])), UnionMode::Sparse), types.into(), vec![Box::new(a1), Box::new(a2)], None, diff --git a/tests/it/bitmap/assign_ops.rs b/tests/it/bitmap/assign_ops.rs index 5f367a7990c..1520457ffa4 100644 --- a/tests/it/bitmap/assign_ops.rs +++ b/tests/it/bitmap/assign_ops.rs @@ -1,6 +1,6 @@ use proptest::prelude::*; -use arrow2::bitmap::{binary_assign, unary_assign, Bitmap, MutableBitmap}; +use re_arrow2::bitmap::{binary_assign, unary_assign, Bitmap, MutableBitmap}; use crate::bitmap::bitmap_strategy; diff --git a/tests/it/bitmap/bitmap_ops.rs b/tests/it/bitmap/bitmap_ops.rs index 9454af49965..dc749991118 100644 --- a/tests/it/bitmap/bitmap_ops.rs +++ b/tests/it/bitmap/bitmap_ops.rs @@ -1,6 +1,6 @@ use proptest::prelude::*; -use arrow2::bitmap::{and, or, xor, Bitmap}; +use re_arrow2::bitmap::{and, or, xor, Bitmap}; use crate::bitmap::bitmap_strategy; diff --git a/tests/it/bitmap/immutable.rs b/tests/it/bitmap/immutable.rs index cc003009e06..5c732cb9de5 100644 --- a/tests/it/bitmap/immutable.rs +++ b/tests/it/bitmap/immutable.rs @@ -1,4 +1,4 @@ -use arrow2::bitmap::Bitmap; +use re_arrow2::bitmap::Bitmap; #[test] fn as_slice() { diff --git a/tests/it/bitmap/mod.rs b/tests/it/bitmap/mod.rs index 04026e4ac43..9ae3b525d52 100644 --- a/tests/it/bitmap/mod.rs +++ b/tests/it/bitmap/mod.rs @@ -6,7 +6,7 @@ mod utils; use proptest::prelude::*; -use arrow2::bitmap::Bitmap; +use re_arrow2::bitmap::Bitmap; /// Returns a strategy of an arbitrary sliced [`Bitmap`] of size up to 1000 pub(crate) fn bitmap_strategy() -> impl Strategy { diff --git a/tests/it/bitmap/mutable.rs b/tests/it/bitmap/mutable.rs index 960c6303e66..f0dff4eafab 100644 --- a/tests/it/bitmap/mutable.rs +++ b/tests/it/bitmap/mutable.rs @@ -1,4 +1,4 @@ -use arrow2::bitmap::{Bitmap, MutableBitmap}; +use re_arrow2::bitmap::{Bitmap, MutableBitmap}; #[test] fn from_slice() { diff --git a/tests/it/bitmap/utils/bit_chunks_exact.rs b/tests/it/bitmap/utils/bit_chunks_exact.rs index 56ac806f16b..dc4887bd9d5 100644 --- a/tests/it/bitmap/utils/bit_chunks_exact.rs +++ b/tests/it/bitmap/utils/bit_chunks_exact.rs @@ -1,4 +1,4 @@ -use arrow2::bitmap::utils::BitChunksExact; +use re_arrow2::bitmap::utils::BitChunksExact; #[test] fn basics() { diff --git a/tests/it/bitmap/utils/chunk_iter.rs b/tests/it/bitmap/utils/chunk_iter.rs index 3bbad3f88cf..c5e3674c728 100644 --- a/tests/it/bitmap/utils/chunk_iter.rs +++ b/tests/it/bitmap/utils/chunk_iter.rs @@ -1,5 +1,5 @@ -use arrow2::bitmap::utils::BitChunks; -use arrow2::types::BitChunkIter; +use re_arrow2::bitmap::utils::BitChunks; +use re_arrow2::types::BitChunkIter; #[test] fn basics() { diff --git a/tests/it/bitmap/utils/fmt.rs b/tests/it/bitmap/utils/fmt.rs index 36e748bfe1b..c5b46b5fcb0 100644 --- a/tests/it/bitmap/utils/fmt.rs +++ b/tests/it/bitmap/utils/fmt.rs @@ -1,4 +1,4 @@ -use arrow2::bitmap::utils::fmt; +use re_arrow2::bitmap::utils::fmt; struct A<'a>(&'a [u8], usize, usize); diff --git a/tests/it/bitmap/utils/iterator.rs b/tests/it/bitmap/utils/iterator.rs index 1f1d56d39d0..93cbdd664c5 100644 --- a/tests/it/bitmap/utils/iterator.rs +++ b/tests/it/bitmap/utils/iterator.rs @@ -1,4 +1,4 @@ -use arrow2::bitmap::utils::BitmapIter; +use re_arrow2::bitmap::utils::BitmapIter; #[test] fn basic() { diff --git a/tests/it/bitmap/utils/mod.rs b/tests/it/bitmap/utils/mod.rs index 9b138d82eb8..1c99753ffb8 100644 --- a/tests/it/bitmap/utils/mod.rs +++ b/tests/it/bitmap/utils/mod.rs @@ -1,6 +1,6 @@ use proptest::prelude::*; -use arrow2::bitmap::utils::*; +use re_arrow2::bitmap::utils::*; use crate::bitmap::bitmap_strategy; diff --git a/tests/it/bitmap/utils/slice_iterator.rs b/tests/it/bitmap/utils/slice_iterator.rs index cab565456d8..040c1c7d687 100644 --- a/tests/it/bitmap/utils/slice_iterator.rs +++ b/tests/it/bitmap/utils/slice_iterator.rs @@ -1,7 +1,7 @@ use proptest::prelude::*; -use arrow2::bitmap::utils::SlicesIterator; -use arrow2::bitmap::Bitmap; +use re_arrow2::bitmap::utils::SlicesIterator; +use re_arrow2::bitmap::Bitmap; use crate::bitmap::bitmap_strategy; diff --git a/tests/it/bitmap/utils/zip_validity.rs b/tests/it/bitmap/utils/zip_validity.rs index dc162962202..96003307d94 100644 --- a/tests/it/bitmap/utils/zip_validity.rs +++ b/tests/it/bitmap/utils/zip_validity.rs @@ -1,4 +1,4 @@ -use arrow2::bitmap::{ +use re_arrow2::bitmap::{ utils::{BitmapIter, ZipValidity}, Bitmap, }; @@ -32,8 +32,8 @@ fn complete() { fn slices() { let a = Bitmap::from([true, false]); let a = Some(a.iter()); - let offsets = vec![0, 2, 3]; - let values = vec![1, 2, 3]; + let offsets = [0, 2, 3]; + let values = [1, 2, 3]; let iter = offsets.windows(2).map(|x| { let start = x[0]; let end = x[1]; diff --git a/tests/it/buffer/immutable.rs b/tests/it/buffer/immutable.rs index 550b2ee1709..4fa3a60955e 100644 --- a/tests/it/buffer/immutable.rs +++ b/tests/it/buffer/immutable.rs @@ -1,4 +1,4 @@ -use arrow2::buffer::Buffer; +use re_arrow2::buffer::Buffer; #[test] fn new() { diff --git a/tests/it/compute/aggregate/memory.rs b/tests/it/compute/aggregate/memory.rs index be6ca35ee06..000ef45dcb4 100644 --- a/tests/it/compute/aggregate/memory.rs +++ b/tests/it/compute/aggregate/memory.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::*, compute::aggregate::estimated_bytes_size, datatypes::{DataType, Field}, @@ -24,8 +24,10 @@ fn utf8() { #[test] fn fixed_size_list() { - let data_type = - DataType::FixedSizeList(Box::new(Field::new("elem", DataType::Float32, false)), 3); + let data_type = DataType::FixedSizeList( + std::sync::Arc::new(Field::new("elem", DataType::Float32, false)), + 3, + ); let values = Box::new(Float32Array::from_slice([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])); let a = FixedSizeListArray::new(data_type, values, None); assert_eq!(6 * std::mem::size_of::(), estimated_bytes_size(&a)); diff --git a/tests/it/compute/aggregate/min_max.rs b/tests/it/compute/aggregate/min_max.rs index ba27c73adeb..2a219dc95c2 100644 --- a/tests/it/compute/aggregate/min_max.rs +++ b/tests/it/compute/aggregate/min_max.rs @@ -1,8 +1,8 @@ -use arrow2::compute::aggregate::{ +use re_arrow2::compute::aggregate::{ max_binary, max_boolean, max_primitive, max_string, min_binary, min_boolean, min_primitive, min_string, }; -use arrow2::{array::*, datatypes::DataType}; +use re_arrow2::{array::*, datatypes::DataType}; #[test] fn test_primitive_array_min_max() { diff --git a/tests/it/compute/aggregate/sum.rs b/tests/it/compute/aggregate/sum.rs index 8c12d1d38c5..53b3ff970c2 100644 --- a/tests/it/compute/aggregate/sum.rs +++ b/tests/it/compute/aggregate/sum.rs @@ -1,8 +1,8 @@ -use arrow2::array::*; -use arrow2::compute::aggregate::{sum, sum_primitive}; -use arrow2::compute::arithmetics; -use arrow2::datatypes::DataType; -use arrow2::scalar::{PrimitiveScalar, Scalar}; +use re_arrow2::array::*; +use re_arrow2::compute::aggregate::{sum, sum_primitive}; +use re_arrow2::compute::arithmetics; +use re_arrow2::datatypes::DataType; +use re_arrow2::scalar::{PrimitiveScalar, Scalar}; #[test] fn test_primitive_array_sum() { diff --git a/tests/it/compute/arithmetics/basic/add.rs b/tests/it/compute/arithmetics/basic/add.rs index 45a4945bd1b..be16619c2ba 100644 --- a/tests/it/compute/arithmetics/basic/add.rs +++ b/tests/it/compute/arithmetics/basic/add.rs @@ -1,7 +1,7 @@ -use arrow2::array::*; -use arrow2::bitmap::Bitmap; -use arrow2::compute::arithmetics::basic::*; -use arrow2::compute::arithmetics::{ +use re_arrow2::array::*; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::compute::arithmetics::basic::*; +use re_arrow2::compute::arithmetics::{ ArrayAdd, ArrayCheckedAdd, ArrayOverflowingAdd, ArraySaturatingAdd, }; diff --git a/tests/it/compute/arithmetics/basic/div.rs b/tests/it/compute/arithmetics/basic/div.rs index 6160f2eea18..3c425417883 100644 --- a/tests/it/compute/arithmetics/basic/div.rs +++ b/tests/it/compute/arithmetics/basic/div.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::compute::arithmetics::basic::*; -use arrow2::compute::arithmetics::{ArrayCheckedDiv, ArrayDiv}; +use re_arrow2::array::*; +use re_arrow2::compute::arithmetics::basic::*; +use re_arrow2::compute::arithmetics::{ArrayCheckedDiv, ArrayDiv}; #[test] #[should_panic] diff --git a/tests/it/compute/arithmetics/basic/mul.rs b/tests/it/compute/arithmetics/basic/mul.rs index b8416e3e0c2..c2f14e1ff76 100644 --- a/tests/it/compute/arithmetics/basic/mul.rs +++ b/tests/it/compute/arithmetics/basic/mul.rs @@ -1,7 +1,7 @@ -use arrow2::array::*; -use arrow2::bitmap::Bitmap; -use arrow2::compute::arithmetics::basic::*; -use arrow2::compute::arithmetics::{ +use re_arrow2::array::*; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::compute::arithmetics::basic::*; +use re_arrow2::compute::arithmetics::{ ArrayCheckedMul, ArrayMul, ArrayOverflowingMul, ArraySaturatingMul, }; diff --git a/tests/it/compute/arithmetics/basic/pow.rs b/tests/it/compute/arithmetics/basic/pow.rs index e6ce52b03fb..f0874cfa1bf 100644 --- a/tests/it/compute/arithmetics/basic/pow.rs +++ b/tests/it/compute/arithmetics/basic/pow.rs @@ -1,5 +1,5 @@ -use arrow2::array::*; -use arrow2::compute::arithmetics::basic::*; +use re_arrow2::array::*; +use re_arrow2::compute::arithmetics::basic::*; #[test] fn test_raise_power_scalar() { diff --git a/tests/it/compute/arithmetics/basic/rem.rs b/tests/it/compute/arithmetics/basic/rem.rs index 666cb3fd624..155e2c9eb60 100644 --- a/tests/it/compute/arithmetics/basic/rem.rs +++ b/tests/it/compute/arithmetics/basic/rem.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::compute::arithmetics::basic::*; -use arrow2::compute::arithmetics::{ArrayCheckedRem, ArrayRem}; +use re_arrow2::array::*; +use re_arrow2::compute::arithmetics::basic::*; +use re_arrow2::compute::arithmetics::{ArrayCheckedRem, ArrayRem}; #[test] #[should_panic] diff --git a/tests/it/compute/arithmetics/basic/sub.rs b/tests/it/compute/arithmetics/basic/sub.rs index 3822bf705b9..b1b123c1a14 100644 --- a/tests/it/compute/arithmetics/basic/sub.rs +++ b/tests/it/compute/arithmetics/basic/sub.rs @@ -1,7 +1,7 @@ -use arrow2::array::*; -use arrow2::bitmap::Bitmap; -use arrow2::compute::arithmetics::basic::*; -use arrow2::compute::arithmetics::{ +use re_arrow2::array::*; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::compute::arithmetics::basic::*; +use re_arrow2::compute::arithmetics::{ ArrayCheckedSub, ArrayOverflowingSub, ArraySaturatingSub, ArraySub, }; diff --git a/tests/it/compute/arithmetics/decimal/add.rs b/tests/it/compute/arithmetics/decimal/add.rs index 45af77b1519..9fb11e6f829 100644 --- a/tests/it/compute/arithmetics/decimal/add.rs +++ b/tests/it/compute/arithmetics/decimal/add.rs @@ -1,9 +1,9 @@ #![allow(clippy::zero_prefixed_literal, clippy::inconsistent_digit_grouping)] -use arrow2::array::*; -use arrow2::compute::arithmetics::decimal::{adaptive_add, add, checked_add, saturating_add}; -use arrow2::compute::arithmetics::{ArrayAdd, ArrayCheckedAdd, ArraySaturatingAdd}; -use arrow2::datatypes::DataType; +use re_arrow2::array::*; +use re_arrow2::compute::arithmetics::decimal::{adaptive_add, add, checked_add, saturating_add}; +use re_arrow2::compute::arithmetics::{ArrayAdd, ArrayCheckedAdd, ArraySaturatingAdd}; +use re_arrow2::datatypes::DataType; #[test] fn test_add_normal() { diff --git a/tests/it/compute/arithmetics/decimal/div.rs b/tests/it/compute/arithmetics/decimal/div.rs index 39138d05dda..43d40463483 100644 --- a/tests/it/compute/arithmetics/decimal/div.rs +++ b/tests/it/compute/arithmetics/decimal/div.rs @@ -1,12 +1,12 @@ #![allow(clippy::zero_prefixed_literal, clippy::inconsistent_digit_grouping)] -use arrow2::array::*; -use arrow2::compute::arithmetics::decimal::{ +use re_arrow2::array::*; +use re_arrow2::compute::arithmetics::decimal::{ adaptive_div, checked_div, div, div_scalar, saturating_div, }; -use arrow2::compute::arithmetics::{ArrayCheckedDiv, ArrayDiv}; -use arrow2::datatypes::DataType; -use arrow2::scalar::PrimitiveScalar; +use re_arrow2::compute::arithmetics::{ArrayCheckedDiv, ArrayDiv}; +use re_arrow2::datatypes::DataType; +use re_arrow2::scalar::PrimitiveScalar; #[test] fn test_divide_normal() { diff --git a/tests/it/compute/arithmetics/decimal/mul.rs b/tests/it/compute/arithmetics/decimal/mul.rs index a4b4a71b257..cfd35155abb 100644 --- a/tests/it/compute/arithmetics/decimal/mul.rs +++ b/tests/it/compute/arithmetics/decimal/mul.rs @@ -1,9 +1,9 @@ #![allow(clippy::zero_prefixed_literal, clippy::inconsistent_digit_grouping)] -use arrow2::array::*; -use arrow2::compute::arithmetics::decimal::{adaptive_mul, checked_mul, mul, saturating_mul}; -use arrow2::compute::arithmetics::{ArrayCheckedMul, ArrayMul, ArraySaturatingMul}; -use arrow2::datatypes::DataType; +use re_arrow2::array::*; +use re_arrow2::compute::arithmetics::decimal::{adaptive_mul, checked_mul, mul, saturating_mul}; +use re_arrow2::compute::arithmetics::{ArrayCheckedMul, ArrayMul, ArraySaturatingMul}; +use re_arrow2::datatypes::DataType; #[test] fn test_multiply_normal() { diff --git a/tests/it/compute/arithmetics/decimal/sub.rs b/tests/it/compute/arithmetics/decimal/sub.rs index 343149a5646..315dab499f4 100644 --- a/tests/it/compute/arithmetics/decimal/sub.rs +++ b/tests/it/compute/arithmetics/decimal/sub.rs @@ -1,9 +1,9 @@ #![allow(clippy::zero_prefixed_literal, clippy::inconsistent_digit_grouping)] -use arrow2::array::*; -use arrow2::compute::arithmetics::decimal::{adaptive_sub, checked_sub, saturating_sub, sub}; -use arrow2::compute::arithmetics::{ArrayCheckedSub, ArraySaturatingSub, ArraySub}; -use arrow2::datatypes::DataType; +use re_arrow2::array::*; +use re_arrow2::compute::arithmetics::decimal::{adaptive_sub, checked_sub, saturating_sub, sub}; +use re_arrow2::compute::arithmetics::{ArrayCheckedSub, ArraySaturatingSub, ArraySub}; +use re_arrow2::datatypes::DataType; #[test] fn test_subtract_normal() { diff --git a/tests/it/compute/arithmetics/mod.rs b/tests/it/compute/arithmetics/mod.rs index a7060ab3281..f1345755280 100644 --- a/tests/it/compute/arithmetics/mod.rs +++ b/tests/it/compute/arithmetics/mod.rs @@ -2,11 +2,11 @@ mod basic; mod decimal; mod time; -use arrow2::array::*; -use arrow2::compute::arithmetics::*; -use arrow2::datatypes::DataType::*; -use arrow2::datatypes::{IntervalUnit, TimeUnit}; -use arrow2::scalar::PrimitiveScalar; +use re_arrow2::array::*; +use re_arrow2::compute::arithmetics::*; +use re_arrow2::datatypes::DataType::*; +use re_arrow2::datatypes::{IntervalUnit, TimeUnit}; +use re_arrow2::scalar::PrimitiveScalar; #[test] fn test_add() { @@ -62,7 +62,7 @@ fn consistency() { Interval(IntervalUnit::MonthDayNano), ]; - let cases = datatypes.clone().into_iter().zip(datatypes.into_iter()); + let cases = datatypes.clone().into_iter().zip(datatypes); cases.for_each(|(lhs, rhs)| { let lhs_a = new_empty_array(lhs.clone()); diff --git a/tests/it/compute/arithmetics/time.rs b/tests/it/compute/arithmetics/time.rs index 9f31dd60a97..6b4716952d7 100644 --- a/tests/it/compute/arithmetics/time.rs +++ b/tests/it/compute/arithmetics/time.rs @@ -1,47 +1,52 @@ -use arrow2::array::*; -use arrow2::compute::arithmetics::time::*; -use arrow2::datatypes::{DataType, TimeUnit}; -use arrow2::scalar::*; -use arrow2::types::months_days_ns; +use re_arrow2::array::*; +use re_arrow2::compute::arithmetics::time::*; +use re_arrow2::datatypes::{DataType, TimeUnit}; +use re_arrow2::scalar::*; +use re_arrow2::types::months_days_ns; #[test] fn test_adding_timestamp() { - let timestamp = - PrimitiveArray::from([Some(100000i64), Some(200000i64), None, Some(300000i64)]).to( - DataType::Timestamp(TimeUnit::Second, Some("America/New_York".to_string())), - ); + let timestamp = PrimitiveArray::from([Some(100000i64), Some(200000i64), None, Some(300000i64)]) + .to(DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("America/New_york".to_string())), + )); let duration = PrimitiveArray::from([Some(10i64), Some(20i64), None, Some(30i64)]) .to(DataType::Duration(TimeUnit::Second)); let result = add_duration(×tamp, &duration); - let expected = - PrimitiveArray::from([Some(100010i64), Some(200020i64), None, Some(300030i64)]).to( - DataType::Timestamp(TimeUnit::Second, Some("America/New_York".to_string())), - ); + let expected = PrimitiveArray::from([Some(100010i64), Some(200020i64), None, Some(300030i64)]) + .to(DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("America/New_york".to_string())), + )); assert_eq!(result, expected); let duration = PrimitiveScalar::from(Some(10i64)).to(DataType::Duration(TimeUnit::Second)); let result = add_duration_scalar(×tamp, &duration); - let expected = - PrimitiveArray::from([Some(100010i64), Some(200010i64), None, Some(300010i64)]).to( - DataType::Timestamp(TimeUnit::Second, Some("America/New_York".to_string())), - ); + let expected = PrimitiveArray::from([Some(100010i64), Some(200010i64), None, Some(300010i64)]) + .to(DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("America/New_york".to_string())), + )); assert_eq!(result, expected); } #[test] fn test_adding_duration_different_scale() { - let timestamp = - PrimitiveArray::from([Some(100000i64), Some(200000i64), None, Some(300000i64)]).to( - DataType::Timestamp(TimeUnit::Second, Some("America/New_York".to_string())), - ); - let expected = - PrimitiveArray::from([Some(100010i64), Some(200020i64), None, Some(300030i64)]).to( - DataType::Timestamp(TimeUnit::Second, Some("America/New_York".to_string())), - ); + let timestamp = PrimitiveArray::from([Some(100000i64), Some(200000i64), None, Some(300000i64)]) + .to(DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("America/New_york".to_string())), + )); + let expected = PrimitiveArray::from([Some(100010i64), Some(200020i64), None, Some(300030i64)]) + .to(DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("America/New_york".to_string())), + )); // Testing duration in milliseconds let duration = PrimitiveArray::from([Some(10_000i64), Some(20_000i64), None, Some(30_000i64)]) @@ -69,20 +74,29 @@ fn test_adding_duration_different_scale() { #[test] fn test_adding_subtract_timestamps_scale() { let timestamp = PrimitiveArray::from([Some(10i64), Some(20i64), None, Some(30i64)]).to( - DataType::Timestamp(TimeUnit::Millisecond, Some("America/New_York".to_string())), + DataType::Timestamp( + TimeUnit::Millisecond, + Some(std::sync::Arc::new("America/New_york".to_string())), + ), ); let duration = PrimitiveArray::from([Some(1i64), Some(2i64), None, Some(3i64)]) .to(DataType::Duration(TimeUnit::Second)); let expected = PrimitiveArray::from([Some(1_010i64), Some(2_020i64), None, Some(3_030i64)]).to( - DataType::Timestamp(TimeUnit::Millisecond, Some("America/New_York".to_string())), + DataType::Timestamp( + TimeUnit::Millisecond, + Some(std::sync::Arc::new("America/New_york".to_string())), + ), ); let result = add_duration(×tamp, &duration); assert_eq!(result, expected); let timestamp = PrimitiveArray::from([Some(10i64), Some(20i64), None, Some(30i64)]).to( - DataType::Timestamp(TimeUnit::Nanosecond, Some("America/New_York".to_string())), + DataType::Timestamp( + TimeUnit::Nanosecond, + Some(std::sync::Arc::new("America/New_york".to_string())), + ), ); let duration = PrimitiveArray::from([Some(1i64), Some(2i64), None, Some(3i64)]) .to(DataType::Duration(TimeUnit::Second)); @@ -95,7 +109,7 @@ fn test_adding_subtract_timestamps_scale() { ]) .to(DataType::Timestamp( TimeUnit::Nanosecond, - Some("America/New_York".to_string()), + Some(std::sync::Arc::new("America/New_york".to_string())), )); let result = add_duration(×tamp, &duration); @@ -104,33 +118,37 @@ fn test_adding_subtract_timestamps_scale() { #[test] fn test_subtract_timestamp() { - let timestamp = - PrimitiveArray::from([Some(100000i64), Some(200000i64), None, Some(300000i64)]).to( - DataType::Timestamp(TimeUnit::Second, Some("America/New_York".to_string())), - ); + let timestamp = PrimitiveArray::from([Some(100000i64), Some(200000i64), None, Some(300000i64)]) + .to(DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("America/New_york".to_string())), + )); let duration = PrimitiveArray::from([Some(10i64), Some(20i64), None, Some(30i64)]) .to(DataType::Duration(TimeUnit::Second)); let result = subtract_duration(×tamp, &duration); - let expected = - PrimitiveArray::from([Some(99990i64), Some(199980i64), None, Some(299970i64)]).to( - DataType::Timestamp(TimeUnit::Second, Some("America/New_York".to_string())), - ); + let expected = PrimitiveArray::from([Some(99990i64), Some(199980i64), None, Some(299970i64)]) + .to(DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("America/New_york".to_string())), + )); assert_eq!(result, expected); } #[test] fn test_subtracting_duration_different_scale() { - let timestamp = - PrimitiveArray::from([Some(100000i64), Some(200000i64), None, Some(300000i64)]).to( - DataType::Timestamp(TimeUnit::Second, Some("America/New_York".to_string())), - ); - let expected = - PrimitiveArray::from([Some(99990i64), Some(199980i64), None, Some(299970i64)]).to( - DataType::Timestamp(TimeUnit::Second, Some("America/New_York".to_string())), - ); + let timestamp = PrimitiveArray::from([Some(100000i64), Some(200000i64), None, Some(300000i64)]) + .to(DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("America/New_york".to_string())), + )); + let expected = PrimitiveArray::from([Some(99990i64), Some(199980i64), None, Some(299970i64)]) + .to(DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("America/New_york".to_string())), + )); // Testing duration in milliseconds let duration = PrimitiveArray::from([Some(10_000i64), Some(20_000i64), None, Some(30_000i64)]) @@ -158,21 +176,28 @@ fn test_subtracting_duration_different_scale() { #[test] fn test_subtracting_subtract_timestamps_scale() { let timestamp = PrimitiveArray::from([Some(10i64), Some(20i64), None, Some(30i64)]).to( - DataType::Timestamp(TimeUnit::Millisecond, Some("America/New_York".to_string())), + DataType::Timestamp( + TimeUnit::Millisecond, + Some(std::sync::Arc::new("America/New_york".to_string())), + ), ); let duration = PrimitiveArray::from([Some(1i64), Some(2i64), None, Some(3i64)]) .to(DataType::Duration(TimeUnit::Second)); - let expected = - PrimitiveArray::from([Some(-990i64), Some(-1_980i64), None, Some(-2_970i64)]).to( - DataType::Timestamp(TimeUnit::Millisecond, Some("America/New_York".to_string())), - ); + let expected = PrimitiveArray::from([Some(-990i64), Some(-1_980i64), None, Some(-2_970i64)]) + .to(DataType::Timestamp( + TimeUnit::Millisecond, + Some(std::sync::Arc::new("America/New_york".to_string())), + )); let result = subtract_duration(×tamp, &duration); assert_eq!(result, expected); let timestamp = PrimitiveArray::from([Some(10i64), Some(20i64), None, Some(30i64)]).to( - DataType::Timestamp(TimeUnit::Nanosecond, Some("America/New_York".to_string())), + DataType::Timestamp( + TimeUnit::Nanosecond, + Some(std::sync::Arc::new("America/New_york".to_string())), + ), ); let duration = PrimitiveArray::from([Some(1i64), Some(2i64), None, Some(3i64)]) .to(DataType::Duration(TimeUnit::Second)); @@ -185,7 +210,7 @@ fn test_subtracting_subtract_timestamps_scale() { ]) .to(DataType::Timestamp( TimeUnit::Nanosecond, - Some("America/New_York".to_string()), + Some(std::sync::Arc::new("America/New_york".to_string())), )); let result = subtract_duration(×tamp, &duration); @@ -342,7 +367,7 @@ fn test_add_interval() { fn test_add_interval_offset() { let timestamp = PrimitiveArray::from_slice([1i64]).to(DataType::Timestamp( TimeUnit::Second, - Some("+01:00".to_string()), + Some(std::sync::Arc::new("+01:00".to_string())), )); let interval = months_days_ns::new(0, 1, 0); @@ -351,7 +376,7 @@ fn test_add_interval_offset() { let expected = PrimitiveArray::from_slice([1i64 + 24 * 60 * 60]).to(DataType::Timestamp( TimeUnit::Second, - Some("+01:00".to_string()), + Some(std::sync::Arc::new("+01:00".to_string())), )); let result = add_interval(×tamp, &intervals).unwrap(); @@ -366,7 +391,7 @@ fn test_add_interval_offset() { fn test_add_interval_tz() { let timestamp = PrimitiveArray::from_slice([1i64]).to(DataType::Timestamp( TimeUnit::Second, - Some("GMT".to_string()), + Some(std::sync::Arc::new("GMT".to_string())), )); let interval = months_days_ns::new(0, 1, 0); @@ -374,7 +399,7 @@ fn test_add_interval_tz() { let expected = PrimitiveArray::from_slice([1i64 + 24 * 60 * 60]).to(DataType::Timestamp( TimeUnit::Second, - Some("GMT".to_string()), + Some(std::sync::Arc::new("GMT".to_string())), )); let result = add_interval(×tamp, &intervals).unwrap(); diff --git a/tests/it/compute/arity_assign.rs b/tests/it/compute/arity_assign.rs index b3581e2fa3e..b4826f6971d 100644 --- a/tests/it/compute/arity_assign.rs +++ b/tests/it/compute/arity_assign.rs @@ -1,5 +1,5 @@ -use arrow2::array::Int32Array; -use arrow2::compute::arity_assign::{binary, unary}; +use re_arrow2::array::Int32Array; +use re_arrow2::compute::arity_assign::{binary, unary}; #[test] fn test_unary_assign() { diff --git a/tests/it/compute/bitwise.rs b/tests/it/compute/bitwise.rs index 3d44e58f219..5edb1ad9b3a 100644 --- a/tests/it/compute/bitwise.rs +++ b/tests/it/compute/bitwise.rs @@ -1,5 +1,5 @@ -use arrow2::array::*; -use arrow2::compute::bitwise::*; +use re_arrow2::array::*; +use re_arrow2::compute::bitwise::*; #[test] fn test_xor() { diff --git a/tests/it/compute/boolean.rs b/tests/it/compute/boolean.rs index ae4c0fde85b..f1278139192 100644 --- a/tests/it/compute/boolean.rs +++ b/tests/it/compute/boolean.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::compute::boolean::*; -use arrow2::scalar::BooleanScalar; +use re_arrow2::array::*; +use re_arrow2::compute::boolean::*; +use re_arrow2::scalar::BooleanScalar; use std::iter::FromIterator; #[test] diff --git a/tests/it/compute/boolean_kleene.rs b/tests/it/compute/boolean_kleene.rs index 902e5b425ac..79a3004059f 100644 --- a/tests/it/compute/boolean_kleene.rs +++ b/tests/it/compute/boolean_kleene.rs @@ -1,6 +1,6 @@ -use arrow2::array::BooleanArray; -use arrow2::compute::boolean_kleene::*; -use arrow2::scalar::BooleanScalar; +use re_arrow2::array::BooleanArray; +use re_arrow2::compute::boolean_kleene::*; +use re_arrow2::scalar::BooleanScalar; #[test] fn and_generic() { diff --git a/tests/it/compute/cast.rs b/tests/it/compute/cast.rs index d8a9ecfce10..415f09eacb2 100644 --- a/tests/it/compute/cast.rs +++ b/tests/it/compute/cast.rs @@ -1,8 +1,10 @@ -use arrow2::array::*; -use arrow2::compute::cast::{can_cast_types, cast, CastOptions}; -use arrow2::datatypes::DataType::LargeList; -use arrow2::datatypes::*; -use arrow2::types::{days_ms, months_days_ns, NativeType}; +use std::sync::Arc; + +use re_arrow2::array::*; +use re_arrow2::compute::cast::{can_cast_types, cast, CastOptions}; +use re_arrow2::datatypes::DataType::LargeList; +use re_arrow2::datatypes::*; +use re_arrow2::types::{days_ms, months_days_ns, NativeType}; #[test] fn i32_to_f64() { @@ -148,7 +150,11 @@ fn i32_to_list_i32() { let array = Int32Array::from_slice([5, 6, 7, 8, 9]); let b = cast( &array, - &DataType::List(Box::new(Field::new("item", DataType::Int32, true))), + &DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + true, + ))), CastOptions::default(), ) .unwrap(); @@ -172,7 +178,11 @@ fn i32_to_list_i32_nullable() { let array = Int32Array::from(input); let b = cast( &array, - &DataType::List(Box::new(Field::new("item", DataType::Int32, true))), + &DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + true, + ))), CastOptions::default(), ) .unwrap(); @@ -196,7 +206,11 @@ fn i32_to_list_f64_nullable_sliced() { let array = array.sliced(2, 4); let b = cast( &array, - &DataType::List(Box::new(Field::new("item", DataType::Float64, true))), + &DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float64, + true, + ))), CastOptions::default(), ) .unwrap(); @@ -506,7 +520,10 @@ fn consistency() { Float64, Timestamp(TimeUnit::Second, None), Timestamp(TimeUnit::Millisecond, None), - Timestamp(TimeUnit::Millisecond, Some("+01:00".to_string())), + Timestamp( + TimeUnit::Millisecond, + Some(std::sync::Arc::new("+01:00".to_string())), + ), Timestamp(TimeUnit::Microsecond, None), Timestamp(TimeUnit::Nanosecond, None), Time64(TimeUnit::Microsecond), @@ -525,8 +542,8 @@ fn consistency() { Duration(TimeUnit::Millisecond), Duration(TimeUnit::Microsecond), Duration(TimeUnit::Nanosecond), - List(Box::new(Field::new("a", Utf8, true))), - LargeList(Box::new(Field::new("a", Utf8, true))), + List(std::sync::Arc::new(Field::new("a", Utf8, true))), + LargeList(std::sync::Arc::new(Field::new("a", Utf8, true))), ]; for d1 in &datatypes { for d2 in &datatypes { @@ -645,7 +662,10 @@ fn int32_to_date32() { fn timestamp_to_date32() { test_primitive_to_primitive( &[864000000005i64, 1545696000001], - DataType::Timestamp(TimeUnit::Millisecond, Some(String::from("UTC"))), + DataType::Timestamp( + TimeUnit::Millisecond, + Some(std::sync::Arc::new("UTC".to_string())), + ), &[10000i32, 17890], DataType::Date32, ); @@ -655,7 +675,10 @@ fn timestamp_to_date32() { fn timestamp_to_date64() { test_primitive_to_primitive( &[864000000005i64, 1545696000001], - DataType::Timestamp(TimeUnit::Millisecond, Some(String::from("UTC"))), + DataType::Timestamp( + TimeUnit::Millisecond, + Some(std::sync::Arc::new("UTC".to_string())), + ), &[864000000005i64, 1545696000001i64], DataType::Date64, ); @@ -665,7 +688,10 @@ fn timestamp_to_date64() { fn timestamp_to_i64() { test_primitive_to_primitive( &[864000000005i64, 1545696000001], - DataType::Timestamp(TimeUnit::Millisecond, Some(String::from("UTC"))), + DataType::Timestamp( + TimeUnit::Millisecond, + Some(std::sync::Arc::new("UTC".to_string())), + ), &[864000000005i64, 1545696000001i64], DataType::Int64, ); @@ -686,7 +712,7 @@ fn utf8_to_dict() { let array = Utf8Array::::from([Some("one"), None, Some("three"), Some("one")]); // Cast to a dictionary (same value type, Utf8) - let cast_type = DataType::Dictionary(u8::KEY_TYPE, Box::new(DataType::Utf8), false); + let cast_type = DataType::Dictionary(u8::KEY_TYPE, Arc::new(DataType::Utf8), false); let result = cast(&array, &cast_type, CastOptions::default()).expect("cast failed"); let mut expected = MutableDictionaryArray::>::new(); @@ -717,7 +743,7 @@ fn i32_to_dict() { let array = Int32Array::from(&[Some(1), None, Some(3), Some(1)]); // Cast to a dictionary (same value type, Utf8) - let cast_type = DataType::Dictionary(u8::KEY_TYPE, Box::new(DataType::Int32), false); + let cast_type = DataType::Dictionary(u8::KEY_TYPE, Arc::new(DataType::Int32), false); let result = cast(&array, &cast_type, CastOptions::default()).expect("cast failed"); let mut expected = MutableDictionaryArray::>::new(); @@ -782,7 +808,7 @@ fn list_to_from_fixed_size_list() { #[test] fn timestamp_with_tz_to_utf8() { - let tz = "-02:00".to_string(); + let tz = Arc::new("-02:00".to_string()); let expected = Utf8Array::::from_slice(["1996-12-19T16:39:57-02:00", "1996-12-19T17:39:57-02:00"]); let array = Int64Array::from_slice([851020797000000000, 851024397000000000]) @@ -794,7 +820,7 @@ fn timestamp_with_tz_to_utf8() { #[test] fn utf8_to_timestamp_with_tz() { - let tz = "-02:00".to_string(); + let tz = Arc::new("-02:00".to_string()); let array = Utf8Array::::from_slice(["1996-12-19T16:39:57-02:00", "1996-12-19T17:39:57-02:00"]); // the timezone is used to map the time to UTC. @@ -899,7 +925,7 @@ fn dict_keys() { let result = cast( &array, - &DataType::Dictionary(IntegerType::Int64, Box::new(DataType::Utf8), false), + &DataType::Dictionary(IntegerType::Int64, Arc::new(DataType::Utf8), false), CastOptions::default(), ) .expect("cast failed"); diff --git a/tests/it/compute/comparison.rs b/tests/it/compute/comparison.rs index a63bb39ce01..d20ca74e252 100644 --- a/tests/it/compute/comparison.rs +++ b/tests/it/compute/comparison.rs @@ -1,12 +1,14 @@ -use arrow2::array::*; -use arrow2::bitmap::Bitmap; -use arrow2::compute::comparison::{self, boolean::*, primitive, utf8}; -use arrow2::datatypes::{DataType, DataType::*, IntegerType, IntervalUnit, TimeUnit}; -use arrow2::scalar::new_scalar; +use std::sync::Arc; + +use re_arrow2::array::*; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::compute::comparison::{self, boolean::*, primitive, utf8}; +use re_arrow2::datatypes::{DataType, DataType::*, IntegerType, IntervalUnit, TimeUnit}; +use re_arrow2::scalar::new_scalar; #[test] fn consistency() { - use arrow2::compute::comparison::*; + use re_arrow2::compute::comparison::*; let datatypes = vec![ Null, Boolean, @@ -42,7 +44,7 @@ fn consistency() { Duration(TimeUnit::Millisecond), Duration(TimeUnit::Microsecond), Duration(TimeUnit::Nanosecond), - Dictionary(IntegerType::Int32, Box::new(LargeBinary), false), + Dictionary(IntegerType::Int32, Arc::new(LargeBinary), false), ]; // array <> array @@ -385,12 +387,12 @@ fn primitive_gt_eq() { #[test] #[cfg(all(feature = "compute_cast", feature = "compute_boolean_kleene"))] fn utf8_and_validity() { - use arrow2::compute::cast::CastOptions; + use re_arrow2::compute::cast::CastOptions; let a1 = Utf8Array::::from([Some("0"), Some("1"), None, Some("2")]); let a2 = Int32Array::from([Some(0), Some(1), None, Some(2)]); // due to the cast the values underneath the validity bits differ - let a2 = arrow2::compute::cast::cast(&a2, &DataType::Utf8, CastOptions::default()).unwrap(); + let a2 = re_arrow2::compute::cast::cast(&a2, &DataType::Utf8, CastOptions::default()).unwrap(); let a2 = a2.as_any().downcast_ref::>().unwrap(); let expected = BooleanArray::from_slice([true, true, true, true]); diff --git a/tests/it/compute/concatenate.rs b/tests/it/compute/concatenate.rs index b5c5e52905f..f8f437edb32 100644 --- a/tests/it/compute/concatenate.rs +++ b/tests/it/compute/concatenate.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::compute::concatenate::concatenate; -use arrow2::error::Result; +use re_arrow2::array::*; +use re_arrow2::compute::concatenate::concatenate; +use re_arrow2::error::Result; #[test] fn empty_vec() { diff --git a/tests/it/compute/contains.rs b/tests/it/compute/contains.rs index cfbdd9946e0..475ad7565b1 100644 --- a/tests/it/compute/contains.rs +++ b/tests/it/compute/contains.rs @@ -1,5 +1,5 @@ -use arrow2::array::*; -use arrow2::compute::contains::contains; +use re_arrow2::array::*; +use re_arrow2::compute::contains::contains; // disable wrapping inside literal vectors used for test data and assertions #[rustfmt::skip::macros(vec)] diff --git a/tests/it/compute/filter.rs b/tests/it/compute/filter.rs index 08a7f6cbcad..2ff734b3ba5 100644 --- a/tests/it/compute/filter.rs +++ b/tests/it/compute/filter.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::bitmap::Bitmap; -use arrow2::compute::filter::*; +use re_arrow2::array::*; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::compute::filter::*; #[test] fn array_slice() { @@ -180,7 +180,7 @@ fn list_array() { let value_offsets = Buffer::from_slice_ref(&[0i64, 3, 6, 8, 8]); let list_data_type = - DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false))); + DataType::LargeList(std::sync::Arc::new(Field::new("item", DataType::Int32, false))); let list_data = ArrayData::builder(list_data_type) .len(4) .add_buffer(value_offsets) @@ -202,7 +202,7 @@ fn list_array() { let value_offsets = Buffer::from_slice_ref(&[0i64, 3, 3]); let list_data_type = - DataType::LargeList(Box::new(Field::new("item", DataType::Int32, false))); + DataType::LargeList(std::sync::Arc::new(Field::new("item", DataType::Int32, false))); let expected = ArrayData::builder(list_data_type) .len(2) .add_buffer(value_offsets) diff --git a/tests/it/compute/hash.rs b/tests/it/compute/hash.rs index 92e263f235d..890c8e31e12 100644 --- a/tests/it/compute/hash.rs +++ b/tests/it/compute/hash.rs @@ -1,7 +1,7 @@ -use arrow2::array::new_null_array; -use arrow2::compute::hash::*; -use arrow2::datatypes::DataType::*; -use arrow2::datatypes::TimeUnit; +use re_arrow2::array::new_null_array; +use re_arrow2::compute::hash::*; +use re_arrow2::datatypes::DataType::*; +use re_arrow2::datatypes::TimeUnit; #[test] fn consistency() { diff --git a/tests/it/compute/if_then_else.rs b/tests/it/compute/if_then_else.rs index 842ac7374b6..fe42bf81d28 100644 --- a/tests/it/compute/if_then_else.rs +++ b/tests/it/compute/if_then_else.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::compute::if_then_else::if_then_else; -use arrow2::error::Result; +use re_arrow2::array::*; +use re_arrow2::compute::if_then_else::if_then_else; +use re_arrow2::error::Result; #[test] fn basics() -> Result<()> { diff --git a/tests/it/compute/length.rs b/tests/it/compute/length.rs index ed4ab83c9f2..cfdf83ed5a9 100644 --- a/tests/it/compute/length.rs +++ b/tests/it/compute/length.rs @@ -1,7 +1,7 @@ -use arrow2::array::*; -use arrow2::compute::length::*; -use arrow2::datatypes::*; -use arrow2::offset::Offset; +use re_arrow2::array::*; +use re_arrow2::compute::length::*; +use re_arrow2::datatypes::*; +use re_arrow2::offset::Offset; fn length_test_string() { vec![ @@ -43,7 +43,7 @@ fn utf8() { #[test] fn consistency() { - use arrow2::datatypes::DataType::*; + use re_arrow2::datatypes::DataType::*; let datatypes = vec![ Null, diff --git a/tests/it/compute/like.rs b/tests/it/compute/like.rs index 8b99beb0818..88a267dc3a9 100644 --- a/tests/it/compute/like.rs +++ b/tests/it/compute/like.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::compute::like::*; -use arrow2::error::Result; +use re_arrow2::array::*; +use re_arrow2::compute::like::*; +use re_arrow2::error::Result; #[test] fn test_like_binary() -> Result<()> { diff --git a/tests/it/compute/limit.rs b/tests/it/compute/limit.rs index 545d546fe3e..81652562a64 100644 --- a/tests/it/compute/limit.rs +++ b/tests/it/compute/limit.rs @@ -1,5 +1,5 @@ -use arrow2::array::*; -use arrow2::compute::limit::limit; +use re_arrow2::array::*; +use re_arrow2::compute::limit::limit; #[test] fn limit_array() { diff --git a/tests/it/compute/merge_sort.rs b/tests/it/compute/merge_sort.rs index 54b90233e53..ed167415a34 100644 --- a/tests/it/compute/merge_sort.rs +++ b/tests/it/compute/merge_sort.rs @@ -1,9 +1,9 @@ use std::iter::once; -use arrow2::array::*; -use arrow2::compute::merge_sort::*; -use arrow2::compute::sort::sort; -use arrow2::error::Result; +use re_arrow2::array::*; +use re_arrow2::compute::merge_sort::*; +use re_arrow2::compute::sort::sort; +use re_arrow2::error::Result; #[test] fn merge_u32() -> Result<()> { diff --git a/tests/it/compute/partition.rs b/tests/it/compute/partition.rs index 1e064633197..f51eb73f585 100644 --- a/tests/it/compute/partition.rs +++ b/tests/it/compute/partition.rs @@ -1,8 +1,8 @@ -use arrow2::array::*; -use arrow2::compute::partition::*; -use arrow2::compute::sort::{SortColumn, SortOptions}; -use arrow2::datatypes::DataType; -use arrow2::error::Result; +use re_arrow2::array::*; +use re_arrow2::compute::partition::*; +use re_arrow2::compute::sort::{SortColumn, SortOptions}; +use re_arrow2::datatypes::DataType; +use re_arrow2::error::Result; #[test] fn lexicographical_partition_ranges_empty() { diff --git a/tests/it/compute/regex_match.rs b/tests/it/compute/regex_match.rs index 66f28d03b9b..2f968416689 100644 --- a/tests/it/compute/regex_match.rs +++ b/tests/it/compute/regex_match.rs @@ -1,7 +1,7 @@ -use arrow2::array::{BooleanArray, Utf8Array}; -use arrow2::compute::regex_match::*; -use arrow2::error::Result; -use arrow2::offset::Offset; +use re_arrow2::array::{BooleanArray, Utf8Array}; +use re_arrow2::compute::regex_match::*; +use re_arrow2::error::Result; +use re_arrow2::offset::Offset; fn test_generic, &Utf8Array) -> Result>( lhs: Vec<&str>, diff --git a/tests/it/compute/sort/lex_sort.rs b/tests/it/compute/sort/lex_sort.rs index 8cefae1dd87..9ee7aa63d60 100644 --- a/tests/it/compute/sort/lex_sort.rs +++ b/tests/it/compute/sort/lex_sort.rs @@ -1,5 +1,5 @@ -use arrow2::array::*; -use arrow2::compute::sort::{lexsort, SortColumn, SortOptions}; +use re_arrow2::array::*; +use re_arrow2::compute::sort::{lexsort, SortColumn, SortOptions}; fn test_lex_sort_arrays(input: Vec, expected: Vec>) { let sorted = lexsort::(&input, None).unwrap(); diff --git a/tests/it/compute/sort/mod.rs b/tests/it/compute/sort/mod.rs index 736cfbadba2..4798a780e28 100644 --- a/tests/it/compute/sort/mod.rs +++ b/tests/it/compute/sort/mod.rs @@ -1,10 +1,10 @@ mod lex_sort; mod row; -use arrow2::array::*; -use arrow2::compute::sort::*; -use arrow2::datatypes::*; -use arrow2::types::NativeType; +use re_arrow2::array::*; +use re_arrow2::compute::sort::*; +use re_arrow2::datatypes::*; +use re_arrow2::types::NativeType; fn to_indices_boolean_arrays(data: &[Option], options: SortOptions, expected_data: &[i32]) { let output = BooleanArray::from(data); @@ -550,9 +550,9 @@ fn test_lex_sort_unaligned_rows() { #[test] fn consistency() { - use arrow2::array::new_null_array; - use arrow2::datatypes::DataType::*; - use arrow2::datatypes::TimeUnit; + use re_arrow2::array::new_null_array; + use re_arrow2::datatypes::DataType::*; + use re_arrow2::datatypes::TimeUnit; let datatypes = vec![ Null, diff --git a/tests/it/compute/sort/row/mod.rs b/tests/it/compute/sort/row/mod.rs index 4931689a192..7d5824e7e88 100644 --- a/tests/it/compute/sort/row/mod.rs +++ b/tests/it/compute/sort/row/mod.rs @@ -1,4 +1,6 @@ -use arrow2::{ +use std::sync::Arc; + +use re_arrow2::{ array::{ Array, BinaryArray, BooleanArray, DictionaryArray, Float32Array, Int128Array, Int16Array, Int256Array, Int32Array, MutableDictionaryArray, MutablePrimitiveArray, MutableUtf8Array, @@ -265,7 +267,7 @@ fn test_dictionary_nulls() { let values = Int32Array::from_iter([Some(1), Some(-1), None, Some(4), None]); let keys = Int32Array::from_iter([Some(0), Some(0), Some(1), Some(2), Some(4), None]); - let data_type = DataType::Dictionary(IntegerType::Int32, Box::new(DataType::Int32), false); + let data_type = DataType::Dictionary(IntegerType::Int32, Arc::new(DataType::Int32), false); let data = DictionaryArray::try_from_keys(keys, values.to_boxed()).unwrap(); let mut converter = RowConverter::new(vec![SortField::new(data_type)]); diff --git a/tests/it/compute/substring.rs b/tests/it/compute/substring.rs index 13ae035d23b..932f1920455 100644 --- a/tests/it/compute/substring.rs +++ b/tests/it/compute/substring.rs @@ -1,4 +1,4 @@ -use arrow2::{array::*, compute::substring::*, error::Result, offset::Offset}; +use re_arrow2::{array::*, compute::substring::*, error::Result, offset::Offset}; fn with_nulls_utf8() -> Result<()> { let cases = vec![ @@ -298,8 +298,8 @@ fn without_nulls_large_binary() -> Result<()> { #[test] fn consistency() { - use arrow2::datatypes::DataType::*; - use arrow2::datatypes::TimeUnit; + use re_arrow2::datatypes::DataType::*; + use re_arrow2::datatypes::TimeUnit; let datatypes = vec![ Null, Boolean, diff --git a/tests/it/compute/take.rs b/tests/it/compute/take.rs index feaa0d82081..79e4cea4b37 100644 --- a/tests/it/compute/take.rs +++ b/tests/it/compute/take.rs @@ -1,8 +1,8 @@ -use arrow2::compute::take::{can_take, take}; -use arrow2::datatypes::{DataType, Field, IntervalUnit}; -use arrow2::error::Result; -use arrow2::{array::*, bitmap::MutableBitmap, types::NativeType}; -use arrow2::{bitmap::Bitmap, buffer::Buffer}; +use re_arrow2::compute::take::{can_take, take}; +use re_arrow2::datatypes::{DataType, Field, IntervalUnit}; +use re_arrow2::error::Result; +use re_arrow2::{array::*, bitmap::MutableBitmap, types::NativeType}; +use re_arrow2::{bitmap::Bitmap, buffer::Buffer}; fn test_take_primitive( data: &[Option], @@ -72,7 +72,7 @@ fn create_test_struct() -> StructArray { Field::new("b", DataType::Int32, true), ]; StructArray::new( - DataType::Struct(fields), + DataType::Struct(std::sync::Arc::new(fields)), vec![boolean.boxed(), int.boxed()], validity, ) @@ -102,9 +102,9 @@ fn test_struct_with_nulls() { #[test] fn consistency() { - use arrow2::array::new_null_array; - use arrow2::datatypes::DataType::*; - use arrow2::datatypes::TimeUnit; + use re_arrow2::array::new_null_array; + use re_arrow2::datatypes::DataType::*; + use re_arrow2::datatypes::TimeUnit; let datatypes = vec![ Null, diff --git a/tests/it/compute/temporal.rs b/tests/it/compute/temporal.rs index 748a4dbfe60..16f6bbff572 100644 --- a/tests/it/compute/temporal.rs +++ b/tests/it/compute/temporal.rs @@ -1,6 +1,8 @@ -use arrow2::array::*; -use arrow2::compute::temporal::*; -use arrow2::datatypes::*; +use std::sync::Arc; + +use re_arrow2::array::*; +use re_arrow2::compute::temporal::*; +use re_arrow2::datatypes::*; macro_rules! temporal_test { ($func:ident, $extract:ident, $data_types:path) => { @@ -198,7 +200,7 @@ fn test_data_tz() -> Vec { // Mon May 24 2021 17:25:30 GMT+0000 Int64Array::from(&[Some(1621877130000000), None]).to(DataType::Timestamp( TimeUnit::Microsecond, - Some("GMT".to_string()), + Some(std::sync::Arc::new("GMT".to_string())), )), ), year: Some(Int32Array::from(&[Some(2021), None])), @@ -213,7 +215,10 @@ fn test_data_tz() -> Vec { }, TestData { input: Box::new(Int64Array::from(&[Some(1621877130000000), None]).to( - DataType::Timestamp(TimeUnit::Microsecond, Some("+01:00".to_string())), + DataType::Timestamp( + TimeUnit::Microsecond, + Some(std::sync::Arc::new("+01:00".to_string())), + ), )), year: Some(Int32Array::from(&[Some(2021), None])), month: Some(UInt32Array::from(&[Some(5), None])), @@ -227,7 +232,10 @@ fn test_data_tz() -> Vec { }, TestData { input: Box::new(Int64Array::from(&[Some(1621877130000000), None]).to( - DataType::Timestamp(TimeUnit::Microsecond, Some("Europe/Lisbon".to_string())), + DataType::Timestamp( + TimeUnit::Microsecond, + Some(std::sync::Arc::new("Europe/Lisbon".to_string())), + ), )), year: Some(Int32Array::from(&[Some(2021), None])), month: Some(UInt32Array::from(&[Some(5), None])), @@ -244,7 +252,7 @@ fn test_data_tz() -> Vec { // Sun Mar 29 2020 00:00:00 GMT+0000 (Western European Standard Time) Int64Array::from(&[Some(1585440000), None]).to(DataType::Timestamp( TimeUnit::Second, - Some("Europe/Lisbon".to_string()), + Some(std::sync::Arc::new("Europe/Lisbon".to_string())), )), ), year: Some(Int32Array::from(&[Some(2020), None])), @@ -262,7 +270,7 @@ fn test_data_tz() -> Vec { // Sun Mar 29 2020 02:00:00 GMT+0100 (Western European Summer Time) Int64Array::from(&[Some(1585443600), None]).to(DataType::Timestamp( TimeUnit::Second, - Some("Europe/Lisbon".to_string()), + Some(std::sync::Arc::new("Europe/Lisbon".to_string())), )), ), year: Some(Int32Array::from(&[Some(2020), None])), @@ -323,11 +331,11 @@ fn consistency_iso_week() { consistency_check(can_iso_week, iso_week); } -fn consistency_check( +fn consistency_check( can_extract: fn(&DataType) -> bool, - extract: fn(&dyn Array) -> arrow2::error::Result>, + extract: fn(&dyn Array) -> re_arrow2::error::Result>, ) { - use arrow2::datatypes::DataType::*; + use re_arrow2::datatypes::DataType::*; let datatypes = vec![ Null, @@ -346,7 +354,7 @@ fn consistency_check( Timestamp(TimeUnit::Millisecond, None), Timestamp(TimeUnit::Microsecond, None), Timestamp(TimeUnit::Nanosecond, None), - Timestamp(TimeUnit::Nanosecond, Some("+00:00".to_string())), + Timestamp(TimeUnit::Nanosecond, Some(Arc::new("+00:00".to_string()))), Time64(TimeUnit::Microsecond), Time64(TimeUnit::Nanosecond), Date32, diff --git a/tests/it/compute/utf8.rs b/tests/it/compute/utf8.rs index 21b89aac3ba..ee192ab5e57 100644 --- a/tests/it/compute/utf8.rs +++ b/tests/it/compute/utf8.rs @@ -1,4 +1,4 @@ -use arrow2::{array::*, compute::utf8::*, error::Result, offset::Offset}; +use re_arrow2::{array::*, compute::utf8::*, error::Result, offset::Offset}; fn with_nulls_utf8_lower() -> Result<()> { let cases = vec![ @@ -140,8 +140,8 @@ fn without_nulls_large_string_lower() -> Result<()> { #[test] fn consistency_lower() { - use arrow2::datatypes::DataType::*; - use arrow2::datatypes::TimeUnit; + use re_arrow2::datatypes::DataType::*; + use re_arrow2::datatypes::TimeUnit; let datatypes = vec![ Null, Boolean, @@ -325,8 +325,8 @@ fn without_nulls_large_string() -> Result<()> { #[test] fn consistency_upper() { - use arrow2::datatypes::DataType::*; - use arrow2::datatypes::TimeUnit; + use re_arrow2::datatypes::DataType::*; + use re_arrow2::datatypes::TimeUnit; let datatypes = vec![ Null, Boolean, diff --git a/tests/it/compute/window.rs b/tests/it/compute/window.rs index c89706a05f9..6b3a4b082bd 100644 --- a/tests/it/compute/window.rs +++ b/tests/it/compute/window.rs @@ -1,6 +1,6 @@ -use arrow2::array::{new_null_array, Int32Array}; -use arrow2::compute::window::*; -use arrow2::datatypes::DataType; +use re_arrow2::array::{new_null_array, Int32Array}; +use re_arrow2::compute::window::*; +use re_arrow2::datatypes::DataType; #[test] fn shift_pos() { diff --git a/tests/it/ffi/data.rs b/tests/it/ffi/data.rs index e5675ac60fe..31bacb83878 100644 --- a/tests/it/ffi/data.rs +++ b/tests/it/ffi/data.rs @@ -1,8 +1,9 @@ -use arrow2::array::*; -use arrow2::bitmap::Bitmap; -use arrow2::datatypes::{DataType, Field, TimeUnit}; -use arrow2::{error::Result, ffi}; +use re_arrow2::array::*; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::datatypes::{DataType, Field, TimeUnit}; +use re_arrow2::{error::Result, ffi}; use std::collections::BTreeMap; +use std::sync::Arc; fn _test_round_trip(array: Box, expected: Box) -> Result<()> { let field = Field::new("a", array.data_type().clone(), true); @@ -93,7 +94,7 @@ fn decimal_nullable() -> Result<()> { fn timestamp_tz() -> Result<()> { let data = Int64Array::from(&vec![Some(2), None, None]).to(DataType::Timestamp( TimeUnit::Second, - Some("UTC".to_string()), + Some(Arc::new("UTC".to_string())), )); test_round_trip(data) } @@ -210,7 +211,7 @@ fn list_sliced() -> Result<()> { let bitmap = Bitmap::from([true, false, false, true]).sliced(1, 3); let array = ListArray::::try_new( - DataType::List(Box::new(Field::new("a", DataType::Int32, true))), + DataType::List(std::sync::Arc::new(Field::new("a", DataType::Int32, true))), vec![0, 1, 1, 2].try_into().unwrap(), Box::new(PrimitiveArray::::from_slice([1, 2])), Some(bitmap), @@ -256,7 +257,10 @@ fn fixed_size_list_sliced() -> Result<()> { let bitmap = Bitmap::from([true, false, false, true]).sliced(1, 3); let array = FixedSizeListArray::try_new( - DataType::FixedSizeList(Box::new(Field::new("a", DataType::Int32, true)), 2), + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("a", DataType::Int32, true)), + 2, + ), Box::new(PrimitiveArray::::from_vec(vec![1, 2, 3, 4, 5, 6])), Some(bitmap), )?; @@ -287,7 +291,7 @@ fn list_list() -> Result<()> { #[test] fn struct_() -> Result<()> { - let data_type = DataType::Struct(vec![Field::new("a", DataType::Int32, true)]); + let data_type = DataType::Struct(Arc::new(vec![Field::new("a", DataType::Int32, true)])); let values = vec![Int32Array::from([Some(1), None, Some(3)]).boxed()]; let validity = Bitmap::from([true, false, true]); @@ -312,14 +316,14 @@ fn dict() -> Result<()> { fn schema() -> Result<()> { let field = Field::new( "a", - DataType::List(Box::new(Field::new("a", DataType::UInt32, true))), + DataType::List(std::sync::Arc::new(Field::new("a", DataType::UInt32, true))), true, ); test_round_trip_schema(field)?; let field = Field::new( "a", - DataType::Dictionary(u32::KEY_TYPE, Box::new(DataType::Utf8), false), + DataType::Dictionary(u32::KEY_TYPE, Arc::new(DataType::Utf8), false), true, ); test_round_trip_schema(field)?; @@ -337,8 +341,8 @@ fn extension() -> Result<()> { "a", DataType::Extension( "a".to_string(), - Box::new(DataType::Int32), - Some("bla".to_string()), + Arc::new(DataType::Int32), + Some("bla".to_string()).map(Arc::new), ), true, ); @@ -351,11 +355,11 @@ fn extension_children() -> Result<()> { "a", DataType::Extension( "b".to_string(), - Box::new(DataType::Struct(vec![Field::new( + Arc::new(DataType::Struct(Arc::new(vec![Field::new( "c", DataType::Int32, true, - )])), + )]))), None, ), true, diff --git a/tests/it/ffi/mod.rs b/tests/it/ffi/mod.rs index 5a06722bb83..7e9cbb62da9 100644 --- a/tests/it/ffi/mod.rs +++ b/tests/it/ffi/mod.rs @@ -4,7 +4,7 @@ mod stream; #[test] fn mmap_slice() { let slice = &[1, 2, 3]; - let array = unsafe { arrow2::ffi::mmap::slice(slice) }; + let array = unsafe { re_arrow2::ffi::mmap::slice(slice) }; assert_eq!(array.values().as_ref(), &[1, 2, 3]); // note: when `slice` is dropped, array must be dropped as-well since by construction of `slice` they share their lifetimes. } @@ -12,7 +12,7 @@ fn mmap_slice() { #[test] fn mmap_bitmap() { let slice = &[123u8, 255]; - let array = unsafe { arrow2::ffi::mmap::bitmap(slice, 2, 14) }.unwrap(); + let array = unsafe { re_arrow2::ffi::mmap::bitmap(slice, 2, 14) }.unwrap(); assert_eq!( array.values_iter().collect::>(), &[false, true, true, true, true, false, true, true, true, true, true, true, true, true] diff --git a/tests/it/ffi/stream.rs b/tests/it/ffi/stream.rs index 53887d4362f..51ab7b5f670 100644 --- a/tests/it/ffi/stream.rs +++ b/tests/it/ffi/stream.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::datatypes::Field; -use arrow2::{error::Error, error::Result, ffi}; +use re_arrow2::array::*; +use re_arrow2::datatypes::Field; +use re_arrow2::{error::Error, error::Result, ffi}; fn _test_round_trip(arrays: Vec>) -> Result<()> { let field = Field::new("a", arrays[0].data_type().clone(), true); diff --git a/tests/it/io/avro/read.rs b/tests/it/io/avro/read.rs index 90aefbf4240..f59bb041ff4 100644 --- a/tests/it/io/avro/read.rs +++ b/tests/it/io/avro/read.rs @@ -1,13 +1,15 @@ -use arrow2::chunk::Chunk; +use std::sync::Arc; + use avro_rs::types::{Record, Value}; use avro_rs::{Codec, Writer}; use avro_rs::{Days, Decimal, Duration, Millis, Months, Schema as AvroSchema}; +use re_arrow2::chunk::Chunk; -use arrow2::array::*; -use arrow2::datatypes::*; -use arrow2::error::Result; -use arrow2::io::avro::avro_schema::read::read_metadata; -use arrow2::io::avro::read; +use re_arrow2::array::*; +use re_arrow2::datatypes::*; +use re_arrow2::error::Result; +use re_arrow2::io::avro::avro_schema::read::read_metadata; +use re_arrow2::io::avro::read; pub(super) fn schema() -> (AvroSchema, Schema) { let raw_schema = r#" @@ -73,23 +75,27 @@ pub(super) fn schema() -> (AvroSchema, Schema) { Field::new("g", DataType::Utf8, true), Field::new( "h", - DataType::List(Box::new(Field::new("item", DataType::Int32, true))), + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + true, + ))), false, ), Field::new( "i", - DataType::Struct(vec![Field::new("e", DataType::Float64, false)]), + DataType::Struct(Arc::new(vec![Field::new("e", DataType::Float64, false)])), false, ), Field::new( "enum", - DataType::Dictionary(i32::KEY_TYPE, Box::new(DataType::Utf8), false), + DataType::Dictionary(i32::KEY_TYPE, Arc::new(DataType::Utf8), false), false, ), Field::new("decimal", DataType::Decimal(18, 5), false), Field::new( "nullable_struct", - DataType::Struct(vec![Field::new("e", DataType::Float64, false)]), + DataType::Struct(Arc::new(vec![Field::new("e", DataType::Float64, false)])), true, ), ]); @@ -117,7 +123,7 @@ pub(super) fn data() -> Chunk> { Utf8Array::::from([Some("foo"), None]).boxed(), array.into_box(), StructArray::new( - DataType::Struct(vec![Field::new("e", DataType::Float64, false)]), + DataType::Struct(Arc::new(vec![Field::new("e", DataType::Float64, false)])), vec![PrimitiveArray::::from_slice([1.0, 2.0]).boxed()], None, ) @@ -132,7 +138,7 @@ pub(super) fn data() -> Chunk> { .to(DataType::Decimal(18, 5)) .boxed(), StructArray::new( - DataType::Struct(vec![Field::new("e", DataType::Float64, false)]), + DataType::Struct(Arc::new(vec![Field::new("e", DataType::Float64, false)])), vec![PrimitiveArray::::from_slice([1.0, 0.0]).boxed()], Some([true, false].into()), ) @@ -331,7 +337,11 @@ fn schema_list() -> (AvroSchema, Schema) { let schema = Schema::from(vec![Field::new( "h", - DataType::List(Box::new(Field::new("item", DataType::Int32, false))), + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + false, + ))), false, )]); @@ -343,7 +353,11 @@ pub(super) fn data_list() -> Chunk> { let mut array = MutableListArray::>::new_from( Default::default(), - DataType::List(Box::new(Field::new("item", DataType::Int32, false))), + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + false, + ))), 0, ); array.try_extend(data).unwrap(); diff --git a/tests/it/io/avro/read_async.rs b/tests/it/io/avro/read_async.rs index 6a6b09ac9f6..4d12ef6f48a 100644 --- a/tests/it/io/avro/read_async.rs +++ b/tests/it/io/avro/read_async.rs @@ -3,9 +3,9 @@ use avro_rs::Codec; use futures::pin_mut; use futures::StreamExt; -use arrow2::error::Result; -use arrow2::io::avro::avro_schema::read_async::{block_stream, read_metadata}; -use arrow2::io::avro::read; +use re_arrow2::error::Result; +use re_arrow2::io::avro::avro_schema::read_async::{block_stream, read_metadata}; +use re_arrow2::io::avro::read; use super::read::{schema, write_avro}; diff --git a/tests/it/io/avro/write.rs b/tests/it/io/avro/write.rs index 7cff7740fbb..9c1552bfe71 100644 --- a/tests/it/io/avro/write.rs +++ b/tests/it/io/avro/write.rs @@ -1,12 +1,14 @@ -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::datatypes::*; -use arrow2::error::Result; -use arrow2::io::avro::avro_schema::file::{Block, CompressedBlock, Compression}; -use arrow2::io::avro::avro_schema::write::{compress, write_block, write_metadata}; -use arrow2::io::avro::write; -use arrow2::types::months_days_ns; +use std::sync::Arc; + use avro_schema::schema::{Field as AvroField, Record, Schema as AvroSchema}; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::*; +use re_arrow2::error::Result; +use re_arrow2::io::avro::avro_schema::file::{Block, CompressedBlock, Compression}; +use re_arrow2::io::avro::avro_schema::write::{compress, write_block, write_metadata}; +use re_arrow2::io::avro::write; +use re_arrow2::types::months_days_ns; use super::read::read_avro; @@ -42,20 +44,36 @@ pub(super) fn schema() -> Schema { ), Field::new( "list", - DataType::List(Box::new(Field::new("item", DataType::Int32, true))), + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + true, + ))), false, ), Field::new( "list nullable", - DataType::List(Box::new(Field::new("item", DataType::Int32, true))), + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + true, + ))), true, ), ]) } pub(super) fn data() -> Chunk> { - let list_dt = DataType::List(Box::new(Field::new("item", DataType::Int32, true))); - let list_dt1 = DataType::List(Box::new(Field::new("item", DataType::Int32, true))); + let list_dt = DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + true, + ))); + let list_dt1 = DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + true, + ))); let columns = vec![ Box::new(Int64Array::from_slice([27, 47])) as Box, @@ -242,28 +260,28 @@ fn struct_schema() -> Schema { Schema::from(vec![ Field::new( "struct", - DataType::Struct(vec![ + DataType::Struct(Arc::new(vec![ Field::new("item1", DataType::Int32, false), Field::new("item2", DataType::Int32, true), - ]), + ])), false, ), Field::new( "struct nullable", - DataType::Struct(vec![ + DataType::Struct(Arc::new(vec![ Field::new("item1", DataType::Int32, false), Field::new("item2", DataType::Int32, true), - ]), + ])), true, ), ]) } fn struct_data() -> Chunk> { - let struct_dt = DataType::Struct(vec![ + let struct_dt = DataType::Struct(Arc::new(vec![ Field::new("item1", DataType::Int32, false), Field::new("item2", DataType::Int32, true), - ]); + ])); Chunk::new(vec![ Box::new(StructArray::new( diff --git a/tests/it/io/avro/write_async.rs b/tests/it/io/avro/write_async.rs index b98071accbc..054e21ea7d1 100644 --- a/tests/it/io/avro/write_async.rs +++ b/tests/it/io/avro/write_async.rs @@ -1,10 +1,10 @@ -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::datatypes::*; -use arrow2::error::Result; -use arrow2::io::avro::avro_schema::file::Compression; -use arrow2::io::avro::avro_schema::write_async::{write_block, write_metadata}; -use arrow2::io::avro::write; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::*; +use re_arrow2::error::Result; +use re_arrow2::io::avro::avro_schema::file::Compression; +use re_arrow2::io::avro::avro_schema::write_async::{write_block, write_metadata}; +use re_arrow2::io::avro::write; use super::read::read_avro; use super::write::{data, schema, serialize_to_block}; diff --git a/tests/it/io/csv/read.rs b/tests/it/io/csv/read.rs index ca9d56b1912..7bebc8ec955 100644 --- a/tests/it/io/csv/read.rs +++ b/tests/it/io/csv/read.rs @@ -2,10 +2,10 @@ use proptest::prelude::*; use std::io::Cursor; -use arrow2::array::*; -use arrow2::datatypes::*; -use arrow2::error::Result; -use arrow2::io::csv::read::*; +use re_arrow2::array::*; +use re_arrow2::datatypes::*; +use re_arrow2::error::Result; +use re_arrow2::io::csv::read::*; #[test] fn read() -> Result<()> { @@ -379,7 +379,7 @@ fn decimal_only_integer() -> Result<()> { #[test] fn boolean() -> Result<()> { - let input = vec!["true", "True", "False", "F", "t"]; + let input = ["true", "True", "False", "F", "t"]; let input = input.join("\n"); let expected = BooleanArray::from(&[Some(true), Some(true), Some(false), None, None]); @@ -392,7 +392,7 @@ fn boolean() -> Result<()> { #[test] fn float32() -> Result<()> { - let input = vec!["12.34", "12", "0.0", "inf", "-inf", "dd"]; + let input = ["12.34", "12", "0.0", "inf", "-inf", "dd"]; let input = input.join("\n"); let expected = Float32Array::from(&[ @@ -411,7 +411,7 @@ fn float32() -> Result<()> { #[test] fn deserialize_binary() -> Result<()> { - let input = vec!["aa", "bb"]; + let input = ["aa", "bb"]; let input = input.join("\n"); let expected = BinaryArray::::from([Some(b"aa"), Some(b"bb")]); @@ -423,10 +423,13 @@ fn deserialize_binary() -> Result<()> { #[test] fn deserialize_timestamp() -> Result<()> { - let input = vec!["1996-12-19T16:34:57-02:00", "1996-12-19T16:34:58-02:00"]; + let input = ["1996-12-19T16:34:57-02:00", "1996-12-19T16:34:58-02:00"]; let input = input.join("\n"); - let data_type = DataType::Timestamp(TimeUnit::Millisecond, Some("-01:00".to_string())); + let data_type = DataType::Timestamp( + TimeUnit::Millisecond, + Some(std::sync::Arc::new("-01:00".to_string())), + ); let expected = Int64Array::from([Some(851020497000), Some(851020498000)]).to(data_type.clone()); @@ -455,6 +458,6 @@ proptest! { #[test] #[cfg_attr(miri, ignore)] // miri and proptest do not work well :( fn dates(v in "1996-12-19T16:3[0-9]:57-02:00") { - assert_eq!(infer(v.as_bytes()), DataType::Timestamp(TimeUnit::Millisecond, Some("-02:00".to_string()))); + assert_eq!(infer(v.as_bytes()), DataType::Timestamp(TimeUnit::Millisecond, Some(std::sync::Arc::new("-02:00".to_string())))); } } diff --git a/tests/it/io/csv/read_async.rs b/tests/it/io/csv/read_async.rs index a31319cc117..f761668b997 100644 --- a/tests/it/io/csv/read_async.rs +++ b/tests/it/io/csv/read_async.rs @@ -1,8 +1,8 @@ use futures::io::Cursor; -use arrow2::array::*; -use arrow2::error::Result; -use arrow2::io::csv::read_async::*; +use re_arrow2::array::*; +use re_arrow2::error::Result; +use re_arrow2::io::csv::read_async::*; #[tokio::test] async fn read() -> Result<()> { diff --git a/tests/it/io/csv/write.rs b/tests/it/io/csv/write.rs index 2800c3bd77c..aa4858c3414 100644 --- a/tests/it/io/csv/write.rs +++ b/tests/it/io/csv/write.rs @@ -1,10 +1,10 @@ use std::io::Cursor; -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::datatypes::*; -use arrow2::error::Result; -use arrow2::io::csv::write::*; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::*; +use re_arrow2::error::Result; +use re_arrow2::io::csv::write::*; fn data() -> Chunk> { let c1 = Utf8Array::::from_slice(["a b", "c", "d"]); @@ -226,7 +226,7 @@ fn data_array(column: &str) -> (Chunk>, Vec<&'static str>) { ]) .to(DataType::Timestamp( TimeUnit::Nanosecond, - Some("+01:00".to_string()), + Some(std::sync::Arc::new("+01:00".to_string())), )); ( array.boxed(), @@ -243,7 +243,7 @@ fn data_array(column: &str) -> (Chunk>, Vec<&'static str>) { ]) .to(DataType::Timestamp( TimeUnit::Nanosecond, - Some("Europe/Lisbon".to_string()), + Some(std::sync::Arc::new("Europe/Lisbon".to_string())), )); ( array.boxed(), @@ -344,7 +344,7 @@ fn write_tz_timezone_formatted_offset() -> Result<()> { PrimitiveArray::::from_slice([1_555_584_887_378_000_001, 1_555_555_555_555_000_001]) .to(DataType::Timestamp( TimeUnit::Nanosecond, - Some("+01:00".to_string()), + Some(std::sync::Arc::new("+01:00".to_string())), )); let columns = Chunk::new(vec![array.boxed()]); @@ -369,7 +369,7 @@ fn write_tz_timezone_formatted_tz() -> Result<()> { PrimitiveArray::::from_slice([1_555_584_887_378_000_001, 1_555_555_555_555_000_001]) .to(DataType::Timestamp( TimeUnit::Nanosecond, - Some("Europe/Lisbon".to_string()), + Some(std::sync::Arc::new("Europe/Lisbon".to_string())), )); let columns = Chunk::new(vec![array.boxed()]); diff --git a/tests/it/io/flight/mod.rs b/tests/it/io/flight/mod.rs index 9718b749027..306c0f8c1af 100644 --- a/tests/it/io/flight/mod.rs +++ b/tests/it/io/flight/mod.rs @@ -1,10 +1,10 @@ -use arrow2::array::Array; -use arrow2::chunk::Chunk; -use arrow2::datatypes::Schema; -use arrow2::error::Error; +use re_arrow2::array::Array; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::Schema; +use re_arrow2::error::Error; -use arrow2::io::flight::*; -use arrow2::io::ipc::write::{default_ipc_fields, WriteOptions}; +use re_arrow2::io::flight::*; +use re_arrow2::io::ipc::write::{default_ipc_fields, WriteOptions}; use super::ipc::read_gzip_json; diff --git a/tests/it/io/ipc/common.rs b/tests/it/io/ipc/common.rs index a0889670f3e..9d2e6e9828e 100644 --- a/tests/it/io/ipc/common.rs +++ b/tests/it/io/ipc/common.rs @@ -1,7 +1,7 @@ use ahash::AHashMap; use std::{fs::File, io::Read}; -use arrow2::{ +use re_arrow2::{ array::Array, chunk::Chunk, datatypes::Schema, error::Result, io::ipc::read::read_stream_metadata, io::ipc::read::StreamReader, io::ipc::IpcField, io::json_integration::read, io::json_integration::ArrowJson, diff --git a/tests/it/io/ipc/mmap.rs b/tests/it/io/ipc/mmap.rs index 11c89ae02fd..573bd2a0365 100644 --- a/tests/it/io/ipc/mmap.rs +++ b/tests/it/io/ipc/mmap.rs @@ -1,8 +1,8 @@ -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::datatypes::{DataType, Field, Schema}; -use arrow2::error::Result; -use arrow2::io::ipc::read::read_file_metadata; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::{DataType, Field, Schema}; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read::read_file_metadata; use std::sync::Arc; use super::write::file::write; @@ -16,9 +16,9 @@ fn round_trip(array: Box) -> Result<()> { let metadata = read_file_metadata(&mut std::io::Cursor::new(data.as_ref()))?; let dictionaries = - unsafe { arrow2::mmap::mmap_dictionaries_unchecked(&metadata, data.clone())? }; + unsafe { re_arrow2::mmap::mmap_dictionaries_unchecked(&metadata, data.clone())? }; - let new_array = unsafe { arrow2::mmap::mmap_unchecked(&metadata, &dictionaries, data, 0)? }; + let new_array = unsafe { re_arrow2::mmap::mmap_unchecked(&metadata, &dictionaries, data, 0)? }; assert_eq!(new_array.into_arrays()[0], array); Ok(()) } @@ -98,7 +98,11 @@ fn struct_() -> Result<()> { let array = PrimitiveArray::::from([None, None, None, Some(3), Some(4)]).boxed(); let array = StructArray::new( - DataType::Struct(vec![Field::new("f1", array.data_type().clone(), true)]), + DataType::Struct(Arc::new(vec![Field::new( + "f1", + array.data_type().clone(), + true, + )])), vec![array], Some([true, true, false, true, false].into()), ) diff --git a/tests/it/io/ipc/read/file.rs b/tests/it/io/ipc/read/file.rs index 663bae0dcc1..5b786c838b2 100644 --- a/tests/it/io/ipc/read/file.rs +++ b/tests/it/io/ipc/read/file.rs @@ -1,8 +1,8 @@ use std::fs::File; -use arrow2::chunk::Chunk; -use arrow2::error::Result; -use arrow2::io::ipc::read::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read::*; use super::super::common::read_gzip_json; diff --git a/tests/it/io/ipc/read/stream.rs b/tests/it/io/ipc/read/stream.rs index 404954c3e48..66debd43018 100644 --- a/tests/it/io/ipc/read/stream.rs +++ b/tests/it/io/ipc/read/stream.rs @@ -1,8 +1,8 @@ -use arrow2::chunk::Chunk; +use re_arrow2::chunk::Chunk; use std::fs::File; -use arrow2::error::Result; -use arrow2::io::ipc::read::*; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read::*; use crate::io::ipc::common::read_gzip_json; diff --git a/tests/it/io/ipc/read_file_async.rs b/tests/it/io/ipc/read_file_async.rs index 8b74e6592c4..602cc47ad83 100644 --- a/tests/it/io/ipc/read_file_async.rs +++ b/tests/it/io/ipc/read_file_async.rs @@ -2,8 +2,8 @@ use futures::StreamExt; use tokio::fs::File; use tokio_util::compat::*; -use arrow2::error::Result; -use arrow2::io::ipc::read::file_async::*; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read::file_async::*; use crate::io::ipc::common::read_gzip_json; diff --git a/tests/it/io/ipc/read_stream_async.rs b/tests/it/io/ipc/read_stream_async.rs index bb248932e9a..fa5a4cbc45b 100644 --- a/tests/it/io/ipc/read_stream_async.rs +++ b/tests/it/io/ipc/read_stream_async.rs @@ -2,8 +2,8 @@ use futures::StreamExt; use tokio::fs::File; use tokio_util::compat::*; -use arrow2::error::Result; -use arrow2::io::ipc::read::stream_async::*; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read::stream_async::*; use crate::io::ipc::common::read_gzip_json; diff --git a/tests/it/io/ipc/write/file.rs b/tests/it/io/ipc/write/file.rs index 5562f803c50..b1dee02b122 100644 --- a/tests/it/io/ipc/write/file.rs +++ b/tests/it/io/ipc/write/file.rs @@ -1,12 +1,12 @@ use std::io::Cursor; -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::datatypes::{Field, Schema}; -use arrow2::error::Result; -use arrow2::io::ipc::read::{read_file_metadata, FileReader}; -use arrow2::io::ipc::{write::*, IpcField}; -use arrow2::types::{i256, months_days_ns}; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::{Field, Schema}; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read::{read_file_metadata, FileReader}; +use re_arrow2::io::ipc::{write::*, IpcField}; +use re_arrow2::types::{i256, months_days_ns}; use crate::io::ipc::common::read_gzip_json; diff --git a/tests/it/io/ipc/write/file_append.rs b/tests/it/io/ipc/write/file_append.rs index 52edde06d24..3bf71c8a5e9 100644 --- a/tests/it/io/ipc/write/file_append.rs +++ b/tests/it/io/ipc/write/file_append.rs @@ -1,9 +1,9 @@ -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::datatypes::*; -use arrow2::error::Result; -use arrow2::io::ipc::read; -use arrow2::io::ipc::write::{FileWriter, WriteOptions}; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::*; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read; +use re_arrow2::io::ipc::write::{FileWriter, WriteOptions}; use super::file::write; diff --git a/tests/it/io/ipc/write/stream.rs b/tests/it/io/ipc/write/stream.rs index 092e4402b7d..539495fc1da 100644 --- a/tests/it/io/ipc/write/stream.rs +++ b/tests/it/io/ipc/write/stream.rs @@ -1,13 +1,13 @@ use std::io::Cursor; -use arrow2::array::Array; -use arrow2::chunk::Chunk; -use arrow2::datatypes::Schema; -use arrow2::error::Result; -use arrow2::io::ipc::read::read_stream_metadata; -use arrow2::io::ipc::read::StreamReader; -use arrow2::io::ipc::write::{StreamWriter, WriteOptions}; -use arrow2::io::ipc::IpcField; +use re_arrow2::array::Array; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::Schema; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read::read_stream_metadata; +use re_arrow2::io::ipc::read::StreamReader; +use re_arrow2::io::ipc::write::{StreamWriter, WriteOptions}; +use re_arrow2::io::ipc::IpcField; use crate::io::ipc::common::read_arrow_stream; use crate::io::ipc::common::read_gzip_json; diff --git a/tests/it/io/ipc/write_file_async.rs b/tests/it/io/ipc/write_file_async.rs index d510f20ba25..7f27dde6cb9 100644 --- a/tests/it/io/ipc/write_file_async.rs +++ b/tests/it/io/ipc/write_file_async.rs @@ -1,15 +1,15 @@ use std::io::Cursor; -use arrow2::array::Array; -use arrow2::chunk::Chunk; -use arrow2::datatypes::Schema; -use arrow2::error::Result; -use arrow2::io::ipc::read; -use arrow2::io::ipc::write::file_async::FileSink; -use arrow2::io::ipc::write::WriteOptions; -use arrow2::io::ipc::IpcField; use futures::io::Cursor as AsyncCursor; use futures::SinkExt; +use re_arrow2::array::Array; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::Schema; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read; +use re_arrow2::io::ipc::write::file_async::FileSink; +use re_arrow2::io::ipc::write::WriteOptions; +use re_arrow2::io::ipc::IpcField; use crate::io::ipc::common::read_arrow_stream; use crate::io::ipc::common::read_gzip_json; diff --git a/tests/it/io/ipc/write_stream_async.rs b/tests/it/io/ipc/write_stream_async.rs index 12669bdd59e..051c46772a7 100644 --- a/tests/it/io/ipc/write_stream_async.rs +++ b/tests/it/io/ipc/write_stream_async.rs @@ -1,15 +1,15 @@ use std::io::Cursor; -use arrow2::array::Array; -use arrow2::chunk::Chunk; -use arrow2::datatypes::Schema; -use arrow2::error::Result; -use arrow2::io::ipc::read; -use arrow2::io::ipc::write::stream_async; -use arrow2::io::ipc::write::stream_async::StreamSink; -use arrow2::io::ipc::IpcField; use futures::io::Cursor as AsyncCursor; use futures::SinkExt; +use re_arrow2::array::Array; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::Schema; +use re_arrow2::error::Result; +use re_arrow2::io::ipc::read; +use re_arrow2::io::ipc::write::stream_async; +use re_arrow2::io::ipc::write::stream_async::StreamSink; +use re_arrow2::io::ipc::IpcField; use crate::io::ipc::common::read_arrow_stream; use crate::io::ipc::common::read_gzip_json; diff --git a/tests/it/io/json/mod.rs b/tests/it/io/json/mod.rs index 59a68f8dd86..a79620337f2 100644 --- a/tests/it/io/json/mod.rs +++ b/tests/it/io/json/mod.rs @@ -1,11 +1,11 @@ mod read; mod write; -use arrow2::array::*; -use arrow2::chunk::Chunk; -use arrow2::datatypes::Schema; -use arrow2::error::Result; -use arrow2::io::json::write as json_write; +use re_arrow2::array::*; +use re_arrow2::chunk::Chunk; +use re_arrow2::datatypes::Schema; +use re_arrow2::error::Result; +use re_arrow2::io::json::write as json_write; fn write_batch(array: Box) -> Result> { let mut serializer = json_write::Serializer::new(vec![Ok(array)].into_iter(), vec![]); diff --git a/tests/it/io/json/read.rs b/tests/it/io/json/read.rs index 411564190b9..5929eb6b719 100644 --- a/tests/it/io/json/read.rs +++ b/tests/it/io/json/read.rs @@ -1,7 +1,9 @@ -use arrow2::array::*; -use arrow2::datatypes::*; -use arrow2::error::Result; -use arrow2::io::json::read; +use std::sync::Arc; + +use re_arrow2::array::*; +use re_arrow2::datatypes::*; +use re_arrow2::error::Result; +use re_arrow2::io::json::read; #[test] fn read_json() -> Result<()> { @@ -24,7 +26,7 @@ fn read_json() -> Result<()> { let result = read::deserialize(&json, data_type)?; let expected = StructArray::new( - DataType::Struct(vec![Field::new("a", DataType::Int64, true)]), + DataType::Struct(Arc::new(vec![Field::new("a", DataType::Int64, true)])), vec![Box::new(Int64Array::from_slice([1, 2, 3])) as _], None, ); @@ -231,7 +233,7 @@ fn deserialize_timestamp_string_ns() -> Result<()> { let json = json_deserializer::parse(data)?; - let data_type = DataType::List(Box::new(Field::new( + let data_type = DataType::List(std::sync::Arc::new(Field::new( "item", DataType::Timestamp(TimeUnit::Nanosecond, None), false, @@ -253,7 +255,7 @@ fn deserialize_timestamp_string_us() -> Result<()> { let json = json_deserializer::parse(data)?; - let data_type = DataType::List(Box::new(Field::new( + let data_type = DataType::List(std::sync::Arc::new(Field::new( "item", DataType::Timestamp(TimeUnit::Microsecond, None), false, @@ -275,7 +277,7 @@ fn deserialize_timestamp_string_ms() -> Result<()> { let json = json_deserializer::parse(data)?; - let data_type = DataType::List(Box::new(Field::new( + let data_type = DataType::List(std::sync::Arc::new(Field::new( "item", DataType::Timestamp(TimeUnit::Millisecond, None), false, @@ -297,7 +299,7 @@ fn deserialize_timestamp_string_s() -> Result<()> { let json = json_deserializer::parse(data)?; - let data_type = DataType::List(Box::new(Field::new( + let data_type = DataType::List(std::sync::Arc::new(Field::new( "item", DataType::Timestamp(TimeUnit::Second, None), false, @@ -319,9 +321,12 @@ fn deserialize_timestamp_string_tz_s() -> Result<()> { let json = json_deserializer::parse(data)?; - let data_type = DataType::List(Box::new(Field::new( + let data_type = DataType::List(std::sync::Arc::new(Field::new( "item", - DataType::Timestamp(TimeUnit::Second, Some("+01:00".to_string())), + DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("+01:00".to_string())), + ), false, ))); @@ -329,7 +334,7 @@ fn deserialize_timestamp_string_tz_s() -> Result<()> { let expected = Int64Array::from([Some(1680870214)]).to(DataType::Timestamp( TimeUnit::Second, - Some("+01:00".to_string()), + Some(std::sync::Arc::new("+01:00".to_string())), )); assert_eq!(expected, result.as_ref()); @@ -343,7 +348,7 @@ fn deserialize_timestamp_int_ns() -> Result<()> { let json = json_deserializer::parse(data)?; - let data_type = DataType::List(Box::new(Field::new( + let data_type = DataType::List(std::sync::Arc::new(Field::new( "item", DataType::Timestamp(TimeUnit::Nanosecond, None), false, diff --git a/tests/it/io/json/write.rs b/tests/it/io/json/write.rs index 1e6ede9fec0..9b252100f1f 100644 --- a/tests/it/io/json/write.rs +++ b/tests/it/io/json/write.rs @@ -1,5 +1,7 @@ -use arrow2::datatypes::IntegerType; -use arrow2::{ +use std::sync::Arc; + +use re_arrow2::datatypes::IntegerType; +use re_arrow2::{ array::*, bitmap::Bitmap, buffer::Buffer, @@ -83,7 +85,7 @@ fn dictionary_utf8() -> Result<()> { let values = Utf8Array::::from([Some("a"), Some("b"), Some("c"), Some("d")]); let keys = PrimitiveArray::from_slice([0u32, 1, 2, 3, 1]); let array = DictionaryArray::try_new( - DataType::Dictionary(IntegerType::UInt32, Box::new(DataType::LargeUtf8), false), + DataType::Dictionary(IntegerType::UInt32, Arc::new(DataType::LargeUtf8), false), keys, Box::new(values), ) @@ -99,10 +101,10 @@ fn struct_() -> Result<()> { let c1 = Int32Array::from([Some(1), Some(2), Some(3), None, Some(5)]); let c2 = Utf8Array::::from([Some("a"), Some("b"), Some("c"), Some("d"), None]); - let data_type = DataType::Struct(vec![ + let data_type = DataType::Struct(Arc::new(vec![ Field::new("c1", c1.data_type().clone(), true), Field::new("c2", c2.data_type().clone(), true), - ]); + ])); let array = StructArray::new(data_type, vec![Box::new(c1) as _, Box::new(c2)], None); let expected = r#"[{"c1":1,"c2":"a"},{"c1":2,"c2":"b"},{"c1":3,"c2":"c"},{"c1":null,"c2":"d"},{"c1":5,"c2":null}]"#; @@ -112,14 +114,14 @@ fn struct_() -> Result<()> { #[test] fn nested_struct_with_validity() -> Result<()> { - let inner = vec![ + let inner = Arc::new(vec![ Field::new("c121", DataType::Utf8, false), Field::new("c122", DataType::Int32, false), - ]; - let fields = vec![ + ]); + let fields = Arc::new(vec![ Field::new("c11", DataType::Int32, false), Field::new("c12", DataType::Struct(inner.clone()), false), - ]; + ]); let c1 = StructArray::new( DataType::Struct(fields), @@ -139,10 +141,10 @@ fn nested_struct_with_validity() -> Result<()> { ); let c2 = Utf8Array::::from([Some("a"), Some("b"), Some("c")]); - let data_type = DataType::Struct(vec![ + let data_type = DataType::Struct(Arc::new(vec![ Field::new("c1", c1.data_type().clone(), true), Field::new("c2", c2.data_type().clone(), true), - ]); + ])); let array = StructArray::new(data_type, vec![c1.boxed(), c2.boxed()], None); let expected = r#"[{"c1":{"c11":1,"c12":null},"c2":"a"},{"c1":{"c11":null,"c12":{"c121":"f","c122":null}},"c2":"b"},{"c1":null,"c2":"c"}]"#; @@ -153,17 +155,17 @@ fn nested_struct_with_validity() -> Result<()> { #[test] fn nested_struct() -> Result<()> { let c121 = Field::new("c121", DataType::Utf8, false); - let fields = vec![ + let fields = Arc::new(vec![ Field::new("c11", DataType::Int32, false), - Field::new("c12", DataType::Struct(vec![c121.clone()]), false), - ]; + Field::new("c12", DataType::Struct(Arc::new(vec![c121.clone()])), false), + ]); let c1 = StructArray::new( DataType::Struct(fields), vec![ Int32Array::from(&[Some(1), None, Some(5)]).boxed(), StructArray::new( - DataType::Struct(vec![c121]), + DataType::Struct(Arc::new(vec![c121])), vec![Box::new(Utf8Array::::from([ Some("e"), Some("f"), @@ -178,10 +180,10 @@ fn nested_struct() -> Result<()> { let c2 = Utf8Array::::from([Some("a"), Some("b"), Some("c")]); - let data_type = DataType::Struct(vec![ + let data_type = DataType::Struct(Arc::new(vec![ Field::new("c1", c1.data_type().clone(), true), Field::new("c2", c2.data_type().clone(), true), - ]); + ])); let array = StructArray::new(data_type, vec![c1.boxed(), c2.boxed()], None); let expected = r#"[{"c1":{"c11":1,"c12":{"c121":"e"}},"c2":"a"},{"c1":{"c11":null,"c12":{"c121":"f"}},"c2":"b"},{"c1":{"c11":5,"c12":{"c121":"g"}},"c2":"c"}]"#; @@ -207,10 +209,10 @@ fn struct_with_list_field() -> Result<()> { let c2 = PrimitiveArray::from_slice([1, 2, 3, 4, 5]); - let data_type = DataType::Struct(vec![ + let data_type = DataType::Struct(Arc::new(vec![ Field::new("c1", c1.data_type().clone(), true), Field::new("c2", c2.data_type().clone(), true), - ]); + ])); let array = StructArray::new(data_type, vec![c1.boxed(), c2.boxed()], None); let expected = r#"[{"c1":["a","a1"],"c2":1},{"c1":["b"],"c2":2},{"c1":["c"],"c2":3},{"c1":["d"],"c2":4},{"c1":["e"],"c2":5}]"#; @@ -242,10 +244,10 @@ fn nested_list() -> Result<()> { let c2 = Utf8Array::::from([Some("foo"), Some("bar"), None]); - let data_type = DataType::Struct(vec![ + let data_type = DataType::Struct(Arc::new(vec![ Field::new("c1", c1.data_type().clone(), true), Field::new("c2", c2.data_type().clone(), true), - ]); + ])); let array = StructArray::new(data_type, vec![c1.boxed(), c2.boxed()], None); let expected = @@ -330,12 +332,12 @@ fn fixed_size_list_records() -> Result<()> { #[test] fn list_of_struct() -> Result<()> { - let inner = vec![Field::new("c121", DataType::Utf8, false)]; - let fields = vec![ + let inner = Arc::new(vec![Field::new("c121", DataType::Utf8, false)]); + let fields = Arc::new(vec![ Field::new("c11", DataType::Int32, false), Field::new("c12", DataType::Struct(inner.clone()), false), - ]; - let c1_datatype = DataType::List(Box::new(Field::new( + ]); + let c1_datatype = DataType::List(std::sync::Arc::new(Field::new( "s", DataType::Struct(fields.clone()), false, @@ -372,10 +374,10 @@ fn list_of_struct() -> Result<()> { let c2 = Int32Array::from_slice([1, 2, 3]); - let data_type = DataType::Struct(vec![ + let data_type = DataType::Struct(Arc::new(vec![ Field::new("c1", c1.data_type().clone(), true), Field::new("c2", c2.data_type().clone(), true), - ]); + ])); let array = StructArray::new(data_type, vec![c1.boxed(), c2.boxed()], None); let expected = r#"[{"c1":[{"c11":1,"c12":null},{"c11":null,"c12":{"c121":"f"}}],"c2":1},{"c1":null,"c2":2},{"c1":[null],"c2":3}]"#; diff --git a/tests/it/io/mod.rs b/tests/it/io/mod.rs index bf228251df3..597277abca2 100644 --- a/tests/it/io/mod.rs +++ b/tests/it/io/mod.rs @@ -7,8 +7,8 @@ mod json; #[cfg(feature = "io_json")] mod ndjson; -#[cfg(feature = "io_json_integration")] -mod ipc; +// #[cfg(feature = "io_json_integration")] // disabled: requiers test data +// mod ipc; // disabled: requiers test data #[cfg(feature = "io_parquet")] mod parquet; @@ -16,8 +16,8 @@ mod parquet; #[cfg(feature = "io_avro")] mod avro; -#[cfg(feature = "io_orc")] -mod orc; +// #[cfg(feature = "io_orc")] // disabled: requiers test data +// mod orc; // disabled: requiers test data #[cfg(any( feature = "io_csv_read", @@ -26,5 +26,5 @@ mod orc; ))] mod csv; -#[cfg(feature = "io_flight")] -mod flight; +// #[cfg(feature = "io_flight")] // disabled: requiers test data +// mod flight; // disabled: requiers test data diff --git a/tests/it/io/ndjson/mod.rs b/tests/it/io/ndjson/mod.rs index bd5626b1cbd..353b4a2f5ec 100644 --- a/tests/it/io/ndjson/mod.rs +++ b/tests/it/io/ndjson/mod.rs @@ -1,10 +1,12 @@ mod read; -use arrow2::array::*; -use arrow2::bitmap::Bitmap; -use arrow2::datatypes::*; -use arrow2::error::Result; -use arrow2::io::ndjson::write as ndjson_write; +use std::sync::Arc; + +use re_arrow2::array::*; +use re_arrow2::bitmap::Bitmap; +use re_arrow2::datatypes::*; +use re_arrow2::error::Result; +use re_arrow2::io::ndjson::write as ndjson_write; use read::{infer, read_and_deserialize}; @@ -46,20 +48,28 @@ fn case_list() -> (String, Box) { "# .to_string(); - let data_type = DataType::Struct(vec![ + let data_type = DataType::Struct(Arc::new(vec![ Field::new("a", DataType::Int64, true), Field::new( "b", - DataType::List(Box::new(Field::new("item", DataType::Float64, true))), + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float64, + true, + ))), true, ), Field::new( "c", - DataType::List(Box::new(Field::new("item", DataType::Boolean, true))), + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Boolean, + true, + ))), true, ), Field::new("d", DataType::Utf8, true), - ]); + ])); let a = Int64Array::from(&[Some(1), Some(-10), None]); let mut b = MutableListArray::>::new(); @@ -100,9 +110,9 @@ fn case_dict() -> (String, Box) { "# .to_string(); - let data_type = DataType::List(Box::new(Field::new( + let data_type = DataType::List(std::sync::Arc::new(Field::new( "item", - DataType::Dictionary(u64::KEY_TYPE, Box::new(DataType::Utf8), false), + DataType::Dictionary(u64::KEY_TYPE, Arc::new(DataType::Utf8), false), true, ))); @@ -130,7 +140,12 @@ fn case_dict() -> (String, Box) { ( data, - StructArray::new(DataType::Struct(fields), vec![array.boxed()], None).boxed(), + StructArray::new( + DataType::Struct(std::sync::Arc::new(fields)), + vec![array.boxed()], + None, + ) + .boxed(), ) } @@ -139,12 +154,12 @@ fn case_basics() -> (String, Box) { {"a":-10, "b":-3.5, "c":true, "d":null} {"a":100000000, "b":0.6, "d":"text"}"# .to_string(); - let data_type = DataType::Struct(vec![ + let data_type = DataType::Struct(Arc::new(vec![ Field::new("a", DataType::Int64, true), Field::new("b", DataType::Float64, true), Field::new("c", DataType::Boolean, true), Field::new("d", DataType::Utf8, true), - ]); + ])); let array = StructArray::new( data_type, vec![ @@ -163,13 +178,13 @@ fn case_projection() -> (String, Box) { {"a":10, "b":-3.5, "c":true, "d":null, "e":"text"} {"a":100000000, "b":0.6, "d":"text"}"# .to_string(); - let data_type = DataType::Struct(vec![ + let data_type = DataType::Struct(Arc::new(vec![ Field::new("a", DataType::UInt32, true), Field::new("b", DataType::Float32, true), Field::new("c", DataType::Boolean, true), // note how "d" is not here Field::new("e", DataType::Binary, true), - ]); + ])); let array = StructArray::new( data_type, vec![ @@ -191,27 +206,30 @@ fn case_struct() -> (String, Box) { .to_string(); let d_field = Field::new("d", DataType::Utf8, true); - let c_field = Field::new("c", DataType::Struct(vec![d_field.clone()]), true); + let c_field = Field::new("c", DataType::Struct(Arc::new(vec![d_field.clone()])), true); let a_field = Field::new( "a", - DataType::Struct(vec![ + DataType::Struct(Arc::new(vec![ Field::new("b", DataType::Boolean, true), c_field.clone(), - ]), + ])), true, ); - let fields = vec![a_field]; + let fields = Arc::new(vec![a_field]); // build expected output let d = Utf8Array::::from([Some("text"), None, Some("text"), None]); let c = StructArray::new( - DataType::Struct(vec![d_field]), + DataType::Struct(Arc::new(vec![d_field])), vec![d.boxed()], Some([true, false, true, true].into()), ); let b = BooleanArray::from(vec![Some(true), Some(false), Some(true), None]); - let inner = DataType::Struct(vec![Field::new("b", DataType::Boolean, true), c_field]); + let inner = DataType::Struct(Arc::new(vec![ + Field::new("b", DataType::Boolean, true), + c_field, + ])); let expected = StructArray::new( inner, vec![b.boxed(), c.boxed()], @@ -228,14 +246,14 @@ fn case_struct() -> (String, Box) { fn case_nested_list() -> (String, Box) { let d_field = Field::new("d", DataType::Utf8, true); - let c_field = Field::new("c", DataType::Struct(vec![d_field.clone()]), true); + let c_field = Field::new("c", DataType::Struct(Arc::new(vec![d_field.clone()])), true); let b_field = Field::new("b", DataType::Boolean, true); let a_struct_field = Field::new( "a", - DataType::Struct(vec![b_field.clone(), c_field.clone()]), + DataType::Struct(Arc::new(vec![b_field.clone(), c_field.clone()])), true, ); - let a_list_data_type = DataType::List(Box::new(a_struct_field)); + let a_list_data_type = DataType::List(std::sync::Arc::new(a_struct_field)); let a_field = Field::new("a", a_list_data_type.clone(), true); let data = r#" @@ -257,7 +275,7 @@ fn case_nested_list() -> (String, Box) { ]); let c = StructArray::new( - DataType::Struct(vec![d_field]), + DataType::Struct(Arc::new(vec![d_field])), vec![d.boxed()], Some(Bitmap::from_u8_slice([0b11111011], 6)), ); @@ -271,7 +289,7 @@ fn case_nested_list() -> (String, Box) { Some(true), ]); let a_struct = StructArray::new( - DataType::Struct(vec![b_field, c_field]), + DataType::Struct(Arc::new(vec![b_field, c_field])), vec![b.boxed(), c.boxed()], None, ); @@ -283,7 +301,7 @@ fn case_nested_list() -> (String, Box) { ); let array = StructArray::new( - DataType::Struct(vec![a_field]), + DataType::Struct(Arc::new(vec![a_field])), vec![expected.boxed()], None, ) @@ -316,7 +334,7 @@ fn infer_object() -> Result<()> { let utf8_fld = Field::new("utf8", DataType::Utf8, true); let bools_fld = Field::new("bools", DataType::Boolean, true); - let expected = DataType::Struct(vec![u64_fld, f64_fld, utf8_fld, bools_fld]); + let expected = DataType::Struct(Arc::new(vec![u64_fld, f64_fld, utf8_fld, bools_fld])); let actual = infer(data)?; assert_eq!(expected, actual); diff --git a/tests/it/io/ndjson/read.rs b/tests/it/io/ndjson/read.rs index 2c8872ce1a0..92cec5c1331 100644 --- a/tests/it/io/ndjson/read.rs +++ b/tests/it/io/ndjson/read.rs @@ -1,10 +1,11 @@ use std::io::Cursor; +use std::sync::Arc; -use arrow2::array::*; -use arrow2::datatypes::{DataType, Field}; -use arrow2::error::{Error, Result}; -use arrow2::io::ndjson::read as ndjson_read; -use arrow2::io::ndjson::read::FallibleStreamingIterator; +use re_arrow2::array::*; +use re_arrow2::datatypes::{DataType, Field}; +use re_arrow2::error::{Error, Result}; +use re_arrow2::io::ndjson::read as ndjson_read; +use re_arrow2::io::ndjson::read::FallibleStreamingIterator; use super::*; @@ -89,13 +90,13 @@ fn case_nested_struct() -> (String, Box) { {"a": {"a": 2.0, "b": 2}} "#; - let inner = DataType::Struct(vec![ + let inner = DataType::Struct(Arc::new(vec![ Field::new("a", DataType::Float64, true), Field::new("b", DataType::Int64, true), Field::new("c", DataType::Boolean, true), - ]); + ])); - let data_type = DataType::Struct(vec![Field::new("a", inner.clone(), true)]); + let data_type = DataType::Struct(Arc::new(vec![Field::new("a", inner.clone(), true)])); let values = vec![ Float64Array::from([Some(2.0), None, Some(2.0), Some(2.0)]).boxed(), @@ -168,20 +169,28 @@ fn infer_schema_mixed_list() -> Result<()> { {"a":3, "b":4, "c": true, "d":[1, false, "array", 2.4]} "#; - let expected = DataType::Struct(vec![ + let expected = DataType::Struct(Arc::new(vec![ Field::new("a", DataType::Int64, true), Field::new( "b", - DataType::List(Box::new(Field::new("item", DataType::Float64, true))), + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float64, + true, + ))), true, ), Field::new( "c", - DataType::List(Box::new(Field::new("item", DataType::Boolean, true))), + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Boolean, + true, + ))), true, ), Field::new("d", DataType::Utf8, true), - ]); + ])); let result = infer(ndjson)?; @@ -240,10 +249,10 @@ fn line_break_in_values() -> Result<()> { fn invalid_read_record() -> Result<()> { let fields = vec![Field::new( "a", - DataType::Struct(vec![Field::new("a", DataType::Utf8, true)]), + DataType::Struct(Arc::new(vec![Field::new("a", DataType::Utf8, true)])), true, )]; - let data_type = DataType::Struct(fields); + let data_type = DataType::Struct(std::sync::Arc::new(fields)); let arrays = read_and_deserialize("city,lat,lng", &data_type, 1000); assert_eq!( @@ -262,7 +271,7 @@ fn skip_empty_lines() -> Result<()> { {\"a\": 3}"; - let data_type = DataType::Struct(vec![Field::new("a", DataType::Int64, true)]); + let data_type = DataType::Struct(Arc::new(vec![Field::new("a", DataType::Int64, true)])); let arrays = read_and_deserialize(ndjson, &data_type, 1000)?; assert_eq!(1, arrays.len()); diff --git a/tests/it/io/orc/read.rs b/tests/it/io/orc/read.rs index a35c54d34e6..21cfc9771fd 100644 --- a/tests/it/io/orc/read.rs +++ b/tests/it/io/orc/read.rs @@ -1,6 +1,6 @@ -use arrow2::array::*; -use arrow2::error::Error; -use arrow2::io::orc::{format, read}; +use re_arrow2::array::*; +use re_arrow2::error::Error; +use re_arrow2::io::orc::{format, read}; #[test] fn infer() -> Result<(), Error> { diff --git a/tests/it/io/parquet/deserialize.rs b/tests/it/io/parquet/deserialize.rs index 3ea1c2846e2..0f846cfa971 100644 --- a/tests/it/io/parquet/deserialize.rs +++ b/tests/it/io/parquet/deserialize.rs @@ -1,6 +1,6 @@ use std::fs::File; -use arrow2::{ +use re_arrow2::{ array::StructArray, datatypes::DataType, error::Result, diff --git a/tests/it/io/parquet/integration.rs b/tests/it/io/parquet/integration.rs index 7f84c433b0d..de37a7a8b92 100644 --- a/tests/it/io/parquet/integration.rs +++ b/tests/it/io/parquet/integration.rs @@ -1,4 +1,4 @@ -use arrow2::error::Result; +use re_arrow2::error::Result; use super::{integration_read, integration_write}; use crate::io::ipc::read_gzip_json; diff --git a/tests/it/io/parquet/mod.rs b/tests/it/io/parquet/mod.rs index 1b38c61c998..73e45a3860b 100644 --- a/tests/it/io/parquet/mod.rs +++ b/tests/it/io/parquet/mod.rs @@ -1,8 +1,9 @@ use ethnum::AsI256; use std::io::{Cursor, Read, Seek}; +use std::sync::Arc; -use arrow2::types::i256; -use arrow2::{ +use re_arrow2::types::i256; +use re_arrow2::{ array::*, bitmap::Bitmap, chunk::Chunk, @@ -37,7 +38,11 @@ fn new_struct( .zip(arrays.iter()) .map(|(n, a)| Field::new(n, a.data_type().clone(), true)) .collect(); - StructArray::new(DataType::Struct(fields), arrays, validity) + StructArray::new( + DataType::Struct(std::sync::Arc::new(fields)), + arrays, + validity, + ) } pub fn read_column(mut reader: R, column: &str) -> Result { @@ -100,13 +105,21 @@ pub fn pyarrow_nested_edge(column: &str) -> Box { // {"f1": ["a", "b", None, "c"]} // ] let a = ListArray::::new( - DataType::List(Box::new(Field::new("item", DataType::Utf8, true))), + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Utf8, + true, + ))), vec![0, 4].try_into().unwrap(), Utf8Array::::from([Some("a"), Some("b"), None, Some("c")]).boxed(), None, ); StructArray::new( - DataType::Struct(vec![Field::new("f1", a.data_type().clone(), true)]), + DataType::Struct(Arc::new(vec![Field::new( + "f1", + a.data_type().clone(), + true, + )])), vec![a.boxed()], None, ) @@ -115,7 +128,7 @@ pub fn pyarrow_nested_edge(column: &str) -> Box { "list_struct_list_nullable" => { let values = pyarrow_nested_edge("struct_list_nullable"); ListArray::::new( - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new( "item", values.data_type().clone(), true, @@ -371,7 +384,7 @@ pub fn pyarrow_nested_nullable(column: &str) -> Box { .boxed(); let array = ListArray::::new( - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new( "item", array.data_type().clone(), true, @@ -411,12 +424,20 @@ pub fn pyarrow_nested_nullable(column: &str) -> Box { match column { "list_int64_required_required" => { // [[0, 1], [], [2, 0, 3], [4, 5, 6], [], [7, 8, 9], [], [10]] - let data_type = DataType::List(Box::new(Field::new("item", DataType::Int64, false))); + let data_type = DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int64, + false, + ))); ListArray::::new(data_type, offsets, values, None).boxed() } "list_int64_optional_required" => { // [[0, 1], [], [2, 0, 3], [4, 5, 6], [], [7, 8, 9], [], [10]] - let data_type = DataType::List(Box::new(Field::new("item", DataType::Int64, true))); + let data_type = DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int64, + true, + ))); ListArray::::new(data_type, offsets, values, None).boxed() } "list_nested_i64" => { @@ -544,7 +565,7 @@ pub fn pyarrow_nested_nullable(column: &str) -> Box { ])); // [0, 2, 2, 5, 8, 8, 11, 11, 12] // [[a1, a2], None, [a3, a4, a5], [a6, a7, a8], [], [a9, a10, a11], None, [a12]] - let data_type = DataType::List(Box::new(field)); + let data_type = DataType::List(std::sync::Arc::new(field)); ListArray::::new(data_type, offsets, values, validity).boxed() } } @@ -688,7 +709,10 @@ pub fn pyarrow_nullable(column: &str) -> Box { PrimitiveArray::::from(i64_values).to(DataType::Timestamp(TimeUnit::Second, None)), ), "timestamp_s_utc" => Box::new(PrimitiveArray::::from(i64_values).to( - DataType::Timestamp(TimeUnit::Second, Some("UTC".to_string())), + DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("UTC".to_string())), + ), )), _ => unreachable!(), } @@ -841,11 +865,11 @@ pub fn pyarrow_nullable_statistics(column: &str) -> Statistics { null_count: UInt64Array::from([Some(3)]).boxed(), min_value: Box::new(Int64Array::from_slice([-256]).to(DataType::Timestamp( TimeUnit::Second, - Some("UTC".to_string()), + Some(std::sync::Arc::new("UTC".to_string())), ))), max_value: Box::new(Int64Array::from_slice([9]).to(DataType::Timestamp( TimeUnit::Second, - Some("UTC".to_string()), + Some(std::sync::Arc::new("UTC".to_string())), ))), }, _ => unreachable!(), @@ -944,7 +968,7 @@ pub fn pyarrow_required_statistics(column: &str) -> Statistics { pub fn pyarrow_nested_nullable_statistics(column: &str) -> Statistics { let new_list = |array: Box, nullable: bool| { ListArray::::new( - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new( "item", array.data_type().clone(), nullable, @@ -1319,7 +1343,7 @@ pub fn pyarrow_nested_nullable_statistics(column: &str) -> Statistics { pub fn pyarrow_nested_edge_statistics(column: &str) -> Statistics { let new_list = |array: Box| { ListArray::::new( - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new( "item", array.data_type().clone(), true, @@ -1336,7 +1360,7 @@ pub fn pyarrow_nested_edge_statistics(column: &str) -> Statistics { .zip(arrays.iter()) .map(|(n, a)| Field::new(n, a.data_type().clone(), true)) .collect(); - StructArray::new(DataType::Struct(fields), arrays, None) + StructArray::new(DataType::Struct(std::sync::Arc::new(fields)), arrays, None) }; let names = vec!["f1".to_string()]; @@ -1442,18 +1466,28 @@ pub fn pyarrow_struct(column: &str) -> Box { Field::new("f2", DataType::Boolean, true), ]; match column { - "struct" => StructArray::new(DataType::Struct(fields), vec![string, boolean], None).boxed(), + "struct" => StructArray::new( + DataType::Struct(std::sync::Arc::new(fields)), + vec![string, boolean], + None, + ) + .boxed(), "struct_nullable" => { let values = vec![string, boolean]; - StructArray::new(DataType::Struct(fields), values, Some(mask.into())).boxed() + StructArray::new( + DataType::Struct(std::sync::Arc::new(fields)), + values, + Some(mask.into()), + ) + .boxed() } "struct_struct" => { let struct_ = pyarrow_struct("struct"); Box::new(StructArray::new( - DataType::Struct(vec![ - Field::new("f1", DataType::Struct(fields), true), + DataType::Struct(Arc::new(vec![ + Field::new("f1", DataType::Struct(std::sync::Arc::new(fields)), true), Field::new("f2", DataType::Boolean, true), - ]), + ])), vec![struct_, boolean], None, )) @@ -1461,10 +1495,10 @@ pub fn pyarrow_struct(column: &str) -> Box { "struct_struct_nullable" => { let struct_ = pyarrow_struct("struct"); Box::new(StructArray::new( - DataType::Struct(vec![ - Field::new("f1", DataType::Struct(fields), true), + DataType::Struct(Arc::new(vec![ + Field::new("f1", DataType::Struct(std::sync::Arc::new(fields)), true), Field::new("f2", DataType::Boolean, true), - ]), + ])), vec![struct_, boolean], Some(mask.into()), )) @@ -1645,12 +1679,15 @@ pub fn pyarrow_map(column: &str) -> Box { "map" => { let s1 = [Some("a1"), Some("a2")]; let s2 = [Some("b1"), Some("b2")]; - let dt = DataType::Struct(vec![ + let dt = DataType::Struct(Arc::new(vec![ Field::new("key", DataType::Utf8, false), Field::new("value", DataType::Utf8, true), - ]); + ])); MapArray::try_new( - DataType::Map(Box::new(Field::new("entries", dt.clone(), false)), false), + DataType::Map( + std::sync::Arc::new(Field::new("entries", dt.clone(), false)), + false, + ), vec![0, 2].try_into().unwrap(), StructArray::try_new( dt, @@ -1670,12 +1707,15 @@ pub fn pyarrow_map(column: &str) -> Box { "map_nullable" => { let s1 = [Some("a1"), Some("a2")]; let s2 = [Some("b1"), None]; - let dt = DataType::Struct(vec![ + let dt = DataType::Struct(Arc::new(vec![ Field::new("key", DataType::Utf8, false), Field::new("value", DataType::Utf8, true), - ]); + ])); MapArray::try_new( - DataType::Map(Box::new(Field::new("entries", dt.clone(), false)), false), + DataType::Map( + std::sync::Arc::new(Field::new("entries", dt.clone(), false)), + false, + ), vec![0, 2].try_into().unwrap(), StructArray::try_new( dt, @@ -1698,14 +1738,16 @@ pub fn pyarrow_map(column: &str) -> Box { pub fn pyarrow_map_statistics(column: &str) -> Statistics { let new_map = |arrays: Vec>, fields: Vec| { - let fields = fields - .into_iter() - .zip(arrays.iter()) - .map(|(f, a)| Field::new(f.name, a.data_type().clone(), f.is_nullable)) - .collect::>(); + let fields = Arc::new( + fields + .into_iter() + .zip(arrays.iter()) + .map(|(f, a)| Field::new(f.name, a.data_type().clone(), f.is_nullable)) + .collect::>(), + ); MapArray::new( DataType::Map( - Box::new(Field::new( + Arc::new(Field::new( "entries", DataType::Struct(fields.clone()), false, @@ -1880,7 +1922,7 @@ fn generic_data() -> Result<(Schema, Chunk>)> { let values = PrimitiveArray::from_slice([1i64, 3]) .to(DataType::Timestamp( TimeUnit::Millisecond, - Some("UTC".to_string()), + Some(std::sync::Arc::new("UTC".to_string())), )) .boxed(); let array7 = DictionaryArray::try_from_keys(indices.clone(), values).unwrap(); @@ -2208,7 +2250,7 @@ fn nested_dict_data(data_type: DataType) -> Result<(Schema, Chunk let indices = PrimitiveArray::from_values((0..3u64).map(|x| x % 2)); let values = DictionaryArray::try_from_keys(indices, values).unwrap(); let values = ListArray::try_new( - DataType::List(Box::new(Field::new( + DataType::List(std::sync::Arc::new(Field::new( "item", values.data_type().clone(), false, diff --git a/tests/it/io/parquet/read.rs b/tests/it/io/parquet/read.rs index 12512116f41..51f89a75f13 100644 --- a/tests/it/io/parquet/read.rs +++ b/tests/it/io/parquet/read.rs @@ -1,8 +1,8 @@ use std::fs::File; -use arrow2::array::*; -use arrow2::error::*; -use arrow2::io::parquet::read::*; +use re_arrow2::array::*; +use re_arrow2::error::*; +use re_arrow2::io::parquet::read::*; use super::*; @@ -894,10 +894,10 @@ fn read_int96_timestamps() -> Result<()> { let parse = |time_unit: TimeUnit| { let mut reader = Cursor::new(timestamp_data); let metadata = read_metadata(&mut reader)?; - let schema = arrow2::datatypes::Schema { - fields: vec![arrow2::datatypes::Field::new( + let schema = re_arrow2::datatypes::Schema { + fields: vec![re_arrow2::datatypes::Field::new( "timestamps", - arrow2::datatypes::DataType::Timestamp(time_unit, None), + re_arrow2::datatypes::DataType::Timestamp(time_unit, None), false, )], metadata: BTreeMap::new(), @@ -910,13 +910,13 @@ fn read_int96_timestamps() -> Result<()> { // Timestamp(TimeUnit::Nanoseconds) and will cause a panic in dev builds/overflow in release builds // However, the code should work for the Microsecond/Millisecond time units for time_unit in [ - arrow2::datatypes::TimeUnit::Microsecond, - arrow2::datatypes::TimeUnit::Millisecond, - arrow2::datatypes::TimeUnit::Second, + re_arrow2::datatypes::TimeUnit::Microsecond, + re_arrow2::datatypes::TimeUnit::Millisecond, + re_arrow2::datatypes::TimeUnit::Second, ] { parse(time_unit).expect("Should not error"); } - std::panic::catch_unwind(|| parse(arrow2::datatypes::TimeUnit::Nanosecond)) + std::panic::catch_unwind(|| parse(re_arrow2::datatypes::TimeUnit::Nanosecond)) .expect_err("Should be a panic error"); Ok(()) diff --git a/tests/it/io/parquet/read_indexes.rs b/tests/it/io/parquet/read_indexes.rs index 4e41bb2baf6..c60342c8388 100644 --- a/tests/it/io/parquet/read_indexes.rs +++ b/tests/it/io/parquet/read_indexes.rs @@ -1,9 +1,11 @@ use std::io::Cursor; -use arrow2::chunk::Chunk; -use arrow2::error::Error; -use arrow2::io::parquet::read::indexes; -use arrow2::{array::*, datatypes::*, error::Result, io::parquet::read::*, io::parquet::write::*}; +use re_arrow2::chunk::Chunk; +use re_arrow2::error::Error; +use re_arrow2::io::parquet::read::indexes; +use re_arrow2::{ + array::*, datatypes::*, error::Result, io::parquet::read::*, io::parquet::write::*, +}; /// Returns 2 sets of pages with different the same number of rows distributed un-evenly fn pages(arrays: &[&dyn Array], encoding: Encoding) -> Result<(Vec, Vec, Schema)> { diff --git a/tests/it/io/parquet/sample_tests.rs b/tests/it/io/parquet/sample_tests.rs index 959f1201283..60b037fb885 100644 --- a/tests/it/io/parquet/sample_tests.rs +++ b/tests/it/io/parquet/sample_tests.rs @@ -1,5 +1,5 @@ -use arrow2::io::parquet::write::*; -use arrow2::{ +use re_arrow2::io::parquet::write::*; +use re_arrow2::{ chunk::Chunk, datatypes::{Field, Metadata, Schema}, error::Result, diff --git a/tests/it/io/parquet/write.rs b/tests/it/io/parquet/write.rs index dee5b8e2536..8023284f436 100644 --- a/tests/it/io/parquet/write.rs +++ b/tests/it/io/parquet/write.rs @@ -1,7 +1,7 @@ use std::io::Cursor; -use arrow2::error::Result; -use arrow2::io::parquet::write::*; +use re_arrow2::error::Result; +use re_arrow2::io::parquet::write::*; use super::*; diff --git a/tests/it/io/parquet/write_async.rs b/tests/it/io/parquet/write_async.rs index 1197e31c0da..7f2f3479535 100644 --- a/tests/it/io/parquet/write_async.rs +++ b/tests/it/io/parquet/write_async.rs @@ -1,5 +1,6 @@ use ahash::AHashMap; -use arrow2::{ +use futures::{future::BoxFuture, io::Cursor, SinkExt}; +use re_arrow2::{ array::{Float32Array, Int32Array}, chunk::Chunk, datatypes::{DataType, Field, Schema}, @@ -9,7 +10,6 @@ use arrow2::{ write::{CompressionOptions, Encoding, Version, WriteOptions}, }, }; -use futures::{future::BoxFuture, io::Cursor, SinkExt}; use super::FileSink; diff --git a/tests/it/io/print.rs b/tests/it/io/print.rs index 8cbc15a95f8..f733a09a744 100644 --- a/tests/it/io/print.rs +++ b/tests/it/io/print.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::*, bitmap::Bitmap, buffer::Buffer, @@ -161,7 +161,10 @@ fn write_timestamp_second_with_tz() { ]; check_datetime!( i64, - DataType::Timestamp(TimeUnit::Second, Some("UTC".to_string())), + DataType::Timestamp( + TimeUnit::Second, + Some(std::sync::Arc::new("UTC".to_string())) + ), 11111111, expected ); @@ -327,7 +330,11 @@ fn write_struct() -> Result<()> { let validity = Some(Bitmap::from(&[true, false, true])); - let array = StructArray::new(DataType::Struct(fields), values, validity); + let array = StructArray::new( + DataType::Struct(std::sync::Arc::new(fields)), + values, + validity, + ); let columns = Chunk::new(vec![&array as &dyn Array]); @@ -356,7 +363,7 @@ fn write_union() -> Result<()> { Field::new("a", DataType::Int32, true), Field::new("b", DataType::Utf8, true), ]; - let data_type = DataType::Union(fields, None, UnionMode::Sparse); + let data_type = DataType::Union(std::sync::Arc::new(fields), None, UnionMode::Sparse); let types = Buffer::from(vec![0, 0, 1]); let fields = vec![ Int32Array::from(&[Some(1), None, Some(2)]).boxed(), diff --git a/tests/it/scalar/binary.rs b/tests/it/scalar/binary.rs index ee71b20ba5b..e488db41412 100644 --- a/tests/it/scalar/binary.rs +++ b/tests/it/scalar/binary.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ datatypes::DataType, scalar::{BinaryScalar, Scalar}, }; diff --git a/tests/it/scalar/boolean.rs b/tests/it/scalar/boolean.rs index 9882e5a77ee..4e70748760b 100644 --- a/tests/it/scalar/boolean.rs +++ b/tests/it/scalar/boolean.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ datatypes::DataType, scalar::{BooleanScalar, Scalar}, }; diff --git a/tests/it/scalar/fixed_size_binary.rs b/tests/it/scalar/fixed_size_binary.rs index 3962c390180..b9d2cf04be6 100644 --- a/tests/it/scalar/fixed_size_binary.rs +++ b/tests/it/scalar/fixed_size_binary.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ datatypes::DataType, scalar::{FixedSizeBinaryScalar, Scalar}, }; diff --git a/tests/it/scalar/fixed_size_list.rs b/tests/it/scalar/fixed_size_list.rs index 89809d343a2..d06c22529b4 100644 --- a/tests/it/scalar/fixed_size_list.rs +++ b/tests/it/scalar/fixed_size_list.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::BooleanArray, datatypes::{DataType, Field}, scalar::{FixedSizeListScalar, Scalar}, @@ -7,7 +7,10 @@ use arrow2::{ #[allow(clippy::eq_op)] #[test] fn equal() { - let dt = DataType::FixedSizeList(Box::new(Field::new("a", DataType::Boolean, true)), 2); + let dt = DataType::FixedSizeList( + std::sync::Arc::new(Field::new("a", DataType::Boolean, true)), + 2, + ); let a = FixedSizeListScalar::new( dt.clone(), Some(BooleanArray::from_slice([true, false]).boxed()), @@ -26,7 +29,10 @@ fn equal() { #[test] fn basics() { - let dt = DataType::FixedSizeList(Box::new(Field::new("a", DataType::Boolean, true)), 2); + let dt = DataType::FixedSizeList( + std::sync::Arc::new(Field::new("a", DataType::Boolean, true)), + 2, + ); let a = FixedSizeListScalar::new( dt.clone(), Some(BooleanArray::from_slice([true, false]).boxed()), diff --git a/tests/it/scalar/list.rs b/tests/it/scalar/list.rs index d8954e6bba0..47aef678c24 100644 --- a/tests/it/scalar/list.rs +++ b/tests/it/scalar/list.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ array::BooleanArray, datatypes::{DataType, Field}, scalar::{ListScalar, Scalar}, @@ -7,7 +7,11 @@ use arrow2::{ #[allow(clippy::eq_op)] #[test] fn equal() { - let dt = DataType::List(Box::new(Field::new("a", DataType::Boolean, true))); + let dt = DataType::List(std::sync::Arc::new(Field::new( + "a", + DataType::Boolean, + true, + ))); let a = ListScalar::::new( dt.clone(), Some(BooleanArray::from_slice([true, false]).boxed()), @@ -23,7 +27,11 @@ fn equal() { #[test] fn basics() { - let dt = DataType::List(Box::new(Field::new("a", DataType::Boolean, true))); + let dt = DataType::List(std::sync::Arc::new(Field::new( + "a", + DataType::Boolean, + true, + ))); let a = ListScalar::::new( dt.clone(), Some(BooleanArray::from_slice([true, false]).boxed()), diff --git a/tests/it/scalar/map.rs b/tests/it/scalar/map.rs index 1a232a5049c..c4a4b8453f8 100644 --- a/tests/it/scalar/map.rs +++ b/tests/it/scalar/map.rs @@ -1,4 +1,6 @@ -use arrow2::{ +use std::sync::Arc; + +use re_arrow2::{ array::{BooleanArray, StructArray, Utf8Array}, datatypes::{DataType, Field}, scalar::{MapScalar, Scalar}, @@ -7,10 +9,10 @@ use arrow2::{ #[allow(clippy::eq_op)] #[test] fn equal() { - let kv_dt = DataType::Struct(vec![ + let kv_dt = DataType::Struct(Arc::new(vec![ Field::new("key", DataType::Utf8, false), Field::new("value", DataType::Boolean, true), - ]); + ])); let kv_array1 = StructArray::try_new( kv_dt.clone(), vec![ @@ -30,7 +32,10 @@ fn equal() { ) .unwrap(); - let dt = DataType::Map(Box::new(Field::new("entries", kv_dt, true)), false); + let dt = DataType::Map( + std::sync::Arc::new(Field::new("entries", kv_dt, true)), + false, + ); let a = MapScalar::new(dt.clone(), Some(Box::new(kv_array1))); let b = MapScalar::new(dt.clone(), None); assert_eq!(a, a); @@ -43,10 +48,10 @@ fn equal() { #[test] fn basics() { - let kv_dt = DataType::Struct(vec![ + let kv_dt = DataType::Struct(Arc::new(vec![ Field::new("key", DataType::Utf8, false), Field::new("value", DataType::Boolean, true), - ]); + ])); let kv_array = StructArray::try_new( kv_dt.clone(), vec![ @@ -57,7 +62,10 @@ fn basics() { ) .unwrap(); - let dt = DataType::Map(Box::new(Field::new("entries", kv_dt, true)), false); + let dt = DataType::Map( + std::sync::Arc::new(Field::new("entries", kv_dt, true)), + false, + ); let a = MapScalar::new(dt.clone(), Some(Box::new(kv_array.clone()))); assert_eq!(kv_array, a.values().as_ref()); diff --git a/tests/it/scalar/mod.rs b/tests/it/scalar/mod.rs index 5dd1568f6d1..66608c453ed 100644 --- a/tests/it/scalar/mod.rs +++ b/tests/it/scalar/mod.rs @@ -12,5 +12,5 @@ mod utf8; // check that `PartialEq` can be derived #[derive(PartialEq)] struct A { - array: Box, + array: Box, } diff --git a/tests/it/scalar/null.rs b/tests/it/scalar/null.rs index 685b237b803..4a593327322 100644 --- a/tests/it/scalar/null.rs +++ b/tests/it/scalar/null.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ datatypes::DataType, scalar::{NullScalar, Scalar}, }; diff --git a/tests/it/scalar/primitive.rs b/tests/it/scalar/primitive.rs index 8769af7cd75..bba069efd56 100644 --- a/tests/it/scalar/primitive.rs +++ b/tests/it/scalar/primitive.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ datatypes::DataType, scalar::{PrimitiveScalar, Scalar}, }; diff --git a/tests/it/scalar/struct_.rs b/tests/it/scalar/struct_.rs index 2785ecb7b41..f90d97211e9 100644 --- a/tests/it/scalar/struct_.rs +++ b/tests/it/scalar/struct_.rs @@ -1,4 +1,6 @@ -use arrow2::{ +use std::sync::Arc; + +use re_arrow2::{ datatypes::{DataType, Field}, scalar::{BooleanScalar, Scalar, StructScalar}, }; @@ -6,7 +8,7 @@ use arrow2::{ #[allow(clippy::eq_op)] #[test] fn equal() { - let dt = DataType::Struct(vec![Field::new("a", DataType::Boolean, true)]); + let dt = DataType::Struct(Arc::new(vec![Field::new("a", DataType::Boolean, true)])); let a = StructScalar::new( dt.clone(), Some(vec![ @@ -29,7 +31,7 @@ fn equal() { #[test] fn basics() { - let dt = DataType::Struct(vec![Field::new("a", DataType::Boolean, true)]); + let dt = DataType::Struct(Arc::new(vec![Field::new("a", DataType::Boolean, true)])); let values = vec![Box::new(BooleanScalar::from(Some(true))) as Box]; diff --git a/tests/it/scalar/utf8.rs b/tests/it/scalar/utf8.rs index 6c844e01f0d..d74c9a0e5fe 100644 --- a/tests/it/scalar/utf8.rs +++ b/tests/it/scalar/utf8.rs @@ -1,4 +1,4 @@ -use arrow2::{ +use re_arrow2::{ datatypes::DataType, scalar::{Scalar, Utf8Scalar}, }; diff --git a/tests/it/temporal_conversions.rs b/tests/it/temporal_conversions.rs index 1bb206de5ad..408775ae85b 100644 --- a/tests/it/temporal_conversions.rs +++ b/tests/it/temporal_conversions.rs @@ -1,7 +1,9 @@ -use arrow2::array::*; -use arrow2::datatypes::TimeUnit; -use arrow2::temporal_conversions; -use arrow2::types::months_days_ns; +use std::sync::Arc; + +use re_arrow2::array::*; +use re_arrow2::datatypes::TimeUnit; +use re_arrow2::temporal_conversions; +use re_arrow2::types::months_days_ns; use chrono::NaiveDateTime; @@ -193,7 +195,7 @@ fn timestamp_to_negative_datetime() { #[test] fn tz_aware() { - let tz = "-02:00".to_string(); + let tz = Arc::new("-02:00".to_string()); let expected = "Timestamp(Nanosecond, Some(\"-02:00\"))[1996-12-19 16:39:57 -02:00, 1996-12-19 17:39:57 -02:00, None]"; let fmt = "%Y-%m-%dT%H:%M:%S%.f%:z"; @@ -208,7 +210,7 @@ fn tz_aware() { #[test] fn tz_aware_no_timezone() { - let tz = "-02:00".to_string(); + let tz = Arc::new("-02:00".to_string()); let expected = "Timestamp(Nanosecond, Some(\"-02:00\"))[None, None, None]"; let fmt = "%Y-%m-%dT%H:%M:%S%.f"; let array = Utf8Array::::from_slice([ diff --git a/tests/it/types.rs b/tests/it/types.rs index ee337e901fb..a7e60b32b2f 100644 --- a/tests/it/types.rs +++ b/tests/it/types.rs @@ -1,4 +1,4 @@ -use arrow2::types::{days_ms, months_days_ns, BitChunkIter, BitChunkOnes, NativeType}; +use re_arrow2::types::{days_ms, months_days_ns, BitChunkIter, BitChunkOnes, NativeType}; #[test] fn test_basic1() {