diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml index 13a3008b74bc..0157caf8c296 100644 --- a/.github/actions/setup-builder/action.yaml +++ b/.github/actions/setup-builder/action.yaml @@ -25,6 +25,31 @@ inputs: runs: using: "composite" steps: + - name: Cache Cargo + uses: actions/cache@v3 + with: + # these represent dependencies downloaded by cargo + # and thus do not depend on the OS, arch nor rust version. + # + # source https://github.com/actions/cache/blob/main/examples.md#rust---cargo + path: | + /usr/local/cargo/bin/ + /usr/local/cargo/registry/index/ + /usr/local/cargo/registry/cache/ + /usr/local/cargo/git/db/ + key: cargo-cache3-${{ hashFiles('**/Cargo.toml') }} + restore-keys: cargo-cache3- + - name: Generate lockfile + shell: bash + run: cargo fetch + - name: Cache Rust dependencies + uses: actions/cache@v3 + with: + # these represent compiled steps of both dependencies and arrow + # and thus are specific for a particular OS, arch and rust version. + path: /github/home/target + key: ${{ runner.os }}-${{ runner.arch }}-target-cache3-${{ inputs.rust-version }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-target-cache3-${{ inputs.rust-version }}- - name: Install Build Dependencies shell: bash run: | @@ -36,4 +61,4 @@ runs: echo "Installing ${{ inputs.rust-version }}" rustup toolchain install ${{ inputs.rust-version }} rustup default ${{ inputs.rust-version }} - rustup component add rustfmt + echo "CARGO_TARGET_DIR=/github/home/target" >> $GITHUB_ENV diff --git a/.github/workflows/miri.yaml b/.github/workflows/miri.yaml index 2167ed519cd4..7feacc07dd73 100644 --- a/.github/workflows/miri.yaml +++ b/.github/workflows/miri.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: Rust +name: MIRI on: # always trigger diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7f5996290577..9331db745659 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -30,8 +30,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - arch: [amd64] - rust: [stable] + arch: [ amd64 ] + rust: [ stable ] container: image: ${{ matrix.arch }}/rust env: @@ -40,39 +40,23 @@ jobs: RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v2 - - name: Cache Cargo - uses: actions/cache@v3 - with: - # these represent dependencies downloaded by cargo - # and thus do not depend on the OS, arch nor rust version. - path: /github/home/.cargo - key: cargo-cache3- - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - # these represent compiled steps of both dependencies and arrow - # and thus are specific for a particular OS, arch and rust version. - path: /github/home/target - key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}- - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ matrix.rust }} - name: Build Workspace run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" cargo build # test the crate linux-test: name: Test Workspace on AMD64 Rust ${{ matrix.rust }} - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-latest strategy: matrix: - arch: [amd64] - rust: [stable] + arch: [ amd64 ] + rust: [ stable ] container: image: ${{ matrix.arch }}/rust env: @@ -85,54 +69,52 @@ jobs: - uses: actions/checkout@v2 with: submodules: true - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /github/home/.cargo - # this key equals the ones on `linux-build-lib` for re-use - key: cargo-cache3- - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: /github/home/target - # this key equals the ones on `linux-build-lib` for re-use - key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }} - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ matrix.rust }} - name: Run tests run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - # run tests on all workspace members with default feature list cargo test - - # Switch to arrow crate - cd arrow - # re-run tests on arrow crate to ensure - # all arrays are created correctly - cargo test --features=force_validate - cargo test --features=prettyprint - # run test on arrow crate with minimal set of features - cargo test --no-default-features + - name: Re-run tests with all supported features + run: | + cargo test -p arrow --features=force_validate,prettyprint + - name: Run examples + run: | + # Test arrow examples cargo run --example builders cargo run --example dynamic_types cargo run --example read_csv cargo run --example read_csv_infer_schema - cargo check --no-default-features - - # Switch to parquet crate - cd ../parquet - # re-run tests on parquet crate with async feature enabled - cargo test --features=async - cargo check --no-default-features - - # Switch to arrow-flight - cd ../arrow-flight - cargo test --features=flight-sql-experimental - cargo check --no-default-features + - name: Test compilation of arrow library crate with different feature combinations + run: | + cargo check -p arrow + cargo check -p arrow --no-default-features + - name: Test compilation of arrow targets with different feature combinations + run: | + cargo check -p arrow --all-targets + cargo check -p arrow --no-default-features --all-targets + cargo check -p arrow --no-default-features --all-targets --features test_utils + - name: Re-run tests on arrow-flight with all features + run: | + cargo test -p arrow-flight --all-features + - name: Re-run tests on parquet crate with all features + run: | + cargo test -p parquet --all-features + - name: Test compilation of parquet library crate with different feature combinations + run: | + cargo check -p parquet + cargo check -p parquet --no-default-features + cargo check -p parquet --no-default-features --features arrow + - name: Test compilation of parquet targets with different feature combinations + run: | + cargo check -p parquet --all-targets + cargo check -p parquet --no-default-features --all-targets + cargo check -p parquet --no-default-features --features arrow --all-targets + - name: Test compilation of parquet_derive macro with different feature combinations + run: | + cargo check -p parquet_derive # test the --features "simd" of the arrow crate. This requires nightly. linux-test-simd: @@ -140,8 +122,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - arch: [amd64] - rust: [nightly] + arch: [ amd64 ] + rust: [ nightly ] container: image: ${{ matrix.arch }}/rust env: @@ -153,40 +135,25 @@ jobs: - uses: actions/checkout@v2 with: submodules: true - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /github/home/.cargo - key: cargo-nightly-cache3- - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: /github/home/target - key: ${{ runner.os }}-${{ matrix.arch }}-target-nightly-cache3-${{ matrix.rust }} - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ matrix.rust }} - name: Run tests run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cd arrow - cargo test --features "simd" - - name: Check new project build with simd features + cargo test -p arrow --features "simd" + - name: Check compilation with simd features run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cd arrow/test/dependency/simd - cargo check + cargo check -p arrow --features simd + cargo check -p arrow --features simd --all-targets windows-and-macos: name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }} runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, macos-latest] - rust: [stable] + os: [ windows-latest, macos-latest ] + rust: [ stable ] steps: - uses: actions/checkout@v2 with: @@ -197,7 +164,6 @@ jobs: run: | rustup toolchain install ${{ matrix.rust }} rustup default ${{ matrix.rust }} - rustup component add rustfmt - name: Run tests shell: bash run: | @@ -209,12 +175,12 @@ jobs: clippy: name: Clippy - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-latest strategy: matrix: - arch: [amd64] - rust: [stable] + arch: [ amd64 ] + rust: [ stable ] container: image: ${{ matrix.arch }}/rust env: @@ -225,29 +191,15 @@ jobs: - uses: actions/checkout@v2 with: submodules: true - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /github/home/.cargo - # this key equals the ones on `linux-build-lib` for re-use - key: cargo-cache3- - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: /github/home/target - # this key equals the ones on `linux-build-lib` for re-use - key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }} - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ matrix.rust }} - name: Setup Clippy run: | - rustup component add rustfmt clippy + rustup component add clippy - name: Run clippy run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" cargo clippy --features test_common --features prettyprint --features=async --all-targets --workspace -- -D warnings check_benches: @@ -255,8 +207,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - arch: [amd64] - rust: [stable] + arch: [ amd64 ] + rust: [ stable ] container: image: ${{ matrix.arch }}/rust env: @@ -267,27 +219,13 @@ jobs: - uses: actions/checkout@v2 with: submodules: true - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /github/home/.cargo - # this key equals the ones on `linux-build-lib` for re-use - key: cargo-cache3- - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: /github/home/target - # this key equals the ones on `linux-build-lib` for re-use - key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }} - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ matrix.rust }} - name: Check benchmarks run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cargo check --benches --workspace --features test_common,prettyprint,async,experimental + cargo check --benches --workspace --features test_common,prettyprint,async,experimental lint: name: Lint (cargo fmt) @@ -309,8 +247,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - arch: [amd64] - rust: [stable] + arch: [ amd64 ] + rust: [ stable ] steps: - uses: actions/checkout@v2 with: @@ -319,7 +257,6 @@ jobs: run: | rustup toolchain install ${{ matrix.rust }} rustup default ${{ matrix.rust }} - rustup component add rustfmt clippy - name: Cache Cargo uses: actions/cache@v3 with: @@ -354,8 +291,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - arch: [amd64] - rust: [nightly] + arch: [ amd64 ] + rust: [ nightly ] container: image: ${{ matrix.arch }}/rust env: @@ -382,13 +319,10 @@ jobs: run: | rustup toolchain install ${{ matrix.rust }} rustup override set ${{ matrix.rust }} - rustup component add rustfmt rustup target add wasm32-unknown-unknown rustup target add wasm32-wasi - name: Build arrow crate run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" cd arrow cargo build --no-default-features --features=csv,ipc,simd --target wasm32-unknown-unknown cargo build --no-default-features --features=csv,ipc,simd --target wasm32-wasi @@ -399,14 +333,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - arch: [amd64] - rust: [nightly] + arch: [ amd64 ] + rust: [ nightly ] container: image: ${{ matrix.arch }}/rust env: # Disable full debug symbol generation to speed up CI build and keep memory down # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" + RUSTDOCFLAGS: "-Dwarnings" steps: - uses: actions/checkout@v2 with: @@ -415,74 +350,10 @@ jobs: run: | apt update apt install -y libpython3.9-dev - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /github/home/.cargo - key: cargo-nightly-cache3- - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: /github/home/target - key: ${{ runner.os }}-${{ matrix.arch }}-target-nightly-cache3-${{ matrix.rust }} - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ matrix.rust }} - name: Run cargo doc run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - export RUSTDOCFLAGS="-Dwarnings" cargo doc --document-private-items --no-deps --workspace --all-features - - - # test builds with various feature flag combinations outside the main workspace - default-build: - name: Feature Flag Builds ${{ matrix.rust }} - runs-on: ubuntu-latest - strategy: - matrix: - arch: [amd64] - rust: [stable] - container: - image: ${{ matrix.arch }}/rust - env: - # Disable debug symbol generation to speed up CI build and keep memory down - RUSTFLAGS: "-C debuginfo=0" - steps: - - uses: actions/checkout@v2 - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /github/home/.cargo - # this key equals the ones on `linux-build-lib` for re-use - key: cargo-cache3- - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: /github/home/target - # this key equals the ones on `linux-build-lib` for re-use - key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }} - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{ matrix.rust }} - - name: Arrow Build with default features - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cd arrow/test/dependency/default-features - cargo check - - name: Arrow Build with default-features=false - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cd arrow/test/dependency/no-default-features - cargo check - - name: Parquet Derive build with default-features - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cd parquet_derive/test/dependency/default-features - cargo check diff --git a/Cargo.toml b/Cargo.toml index de7d36f34814..2837f028e8c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,15 @@ members = [ "arrow-flight", "integration-testing", ] +# Enable the version 2 feature resolver, which avoids unifying features for targets that are not being built +# +# Critically this prevents dev-dependencies from enabling features even when not building a target that +# uses dev-dependencies, e.g. the library crate. This in turn ensures that we can catch invalid feature +# flag combinations that would otherwise only surface in dependent crates +# +# Reference - https://doc.rust-lang.org/nightly/cargo/reference/features.html#feature-resolver-version-2 +# +resolver = "2" # this package is excluded because it requires different compilation flags, thereby significantly changing # how it is compiled within the workspace, causing the whole workspace to be compiled from scratch diff --git a/arrow/Cargo.toml b/arrow/Cargo.toml index edc04ca5f2d9..fd60078f1d92 100644 --- a/arrow/Cargo.toml +++ b/arrow/Cargo.toml @@ -88,6 +88,7 @@ tempfile = "3" [[bench]] name = "aggregate_kernels" harness = false +required-features = ["test_utils"] [[bench]] name = "array_from_vec" @@ -96,6 +97,7 @@ harness = false [[bench]] name = "builder" harness = false +required-features = ["test_utils"] [[bench]] name = "buffer_bit_ops" @@ -104,6 +106,7 @@ harness = false [[bench]] name = "boolean_kernels" harness = false +required-features = ["test_utils"] [[bench]] name = "boolean_append_packed" @@ -112,22 +115,27 @@ harness = false [[bench]] name = "arithmetic_kernels" harness = false +required-features = ["test_utils"] [[bench]] name = "cast_kernels" harness = false +required-features = ["test_utils"] [[bench]] name = "comparison_kernels" harness = false +required-features = ["test_utils"] [[bench]] name = "filter_kernels" harness = false +required-features = ["test_utils"] [[bench]] name = "take_kernels" harness = false +required-features = ["test_utils"] [[bench]] name = "length_kernel" @@ -140,10 +148,12 @@ harness = false [[bench]] name = "sort_kernel" harness = false +required-features = ["test_utils"] [[bench]] name = "partition_kernels" harness = false +required-features = ["test_utils"] [[bench]] name = "csv_writer" @@ -156,6 +166,7 @@ harness = false [[bench]] name = "equal" harness = false +required-features = ["test_utils"] [[bench]] name = "array_slice" @@ -164,18 +175,22 @@ harness = false [[bench]] name = "concatenate_kernel" harness = false +required-features = ["test_utils"] [[bench]] name = "mutable_array" harness = false +required-features = ["test_utils"] [[bench]] name = "buffer_create" harness = false +required-features = ["test_utils"] [[bench]] name = "substring_kernels" harness = false +required-features = ["test_utils"] [[bench]] name = "array_data_validate" diff --git a/arrow/examples/read_csv.rs b/arrow/examples/read_csv.rs index 243d8d0f7ee3..5ccf0c58a797 100644 --- a/arrow/examples/read_csv.rs +++ b/arrow/examples/read_csv.rs @@ -35,7 +35,8 @@ fn main() { Field::new("lng", DataType::Float64, false), ]); - let file = File::open("test/data/uk_cities.csv").unwrap(); + let path = format!("{}/test/data/uk_cities.csv", env!("CARGO_MANIFEST_DIR")); + let file = File::open(path).unwrap(); let mut csv = csv::Reader::new(file, Arc::new(schema), false, None, 1024, None, None, None); diff --git a/arrow/examples/read_csv_infer_schema.rs b/arrow/examples/read_csv_infer_schema.rs index 11f8cfb7f7d2..e9f5ff650706 100644 --- a/arrow/examples/read_csv_infer_schema.rs +++ b/arrow/examples/read_csv_infer_schema.rs @@ -26,7 +26,11 @@ use std::fs::File; fn main() { #[cfg(feature = "csv")] { - let file = File::open("test/data/uk_cities_with_headers.csv").unwrap(); + let path = format!( + "{}/test/data/uk_cities_with_headers.csv", + env!("CARGO_MANIFEST_DIR") + ); + let file = File::open(path).unwrap(); let builder = csv::ReaderBuilder::new() .has_header(true) .infer_schema(Some(100)); diff --git a/arrow/test/dependency/README.md b/arrow/test/dependency/README.md deleted file mode 100644 index b618b4636e7c..000000000000 --- a/arrow/test/dependency/README.md +++ /dev/null @@ -1,21 +0,0 @@ - - -This directory contains projects that use arrow as a dependency with -various combinations of feature flags. diff --git a/arrow/test/dependency/default-features/Cargo.toml b/arrow/test/dependency/default-features/Cargo.toml deleted file mode 100644 index c03aef9979e5..000000000000 --- a/arrow/test/dependency/default-features/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -name = "defeault-features" -description = "Models a user application of arrow that uses default features of arrow" -version = "0.1.0" -edition = "2021" -rust-version = "1.57" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -arrow = { path = "../../../../arrow", version = "15.0.0" } - -[workspace] diff --git a/arrow/test/dependency/default-features/src/main.rs b/arrow/test/dependency/default-features/src/main.rs deleted file mode 100644 index e7a11a969c03..000000000000 --- a/arrow/test/dependency/default-features/src/main.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("Hello, world!"); -} diff --git a/arrow/test/dependency/no-default-features/Cargo.toml b/arrow/test/dependency/no-default-features/Cargo.toml deleted file mode 100644 index c637aa6e9a8b..000000000000 --- a/arrow/test/dependency/no-default-features/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -name = "no-default-features" -description = "Models a user application of arrow that specifies no-default-features=true" -version = "0.1.0" -edition = "2021" -rust-version = "1.57" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -arrow = { path = "../../../../arrow", version = "15.0.0", default-features = false } - -[workspace] diff --git a/arrow/test/dependency/no-default-features/src/main.rs b/arrow/test/dependency/no-default-features/src/main.rs deleted file mode 100644 index e7a11a969c03..000000000000 --- a/arrow/test/dependency/no-default-features/src/main.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("Hello, world!"); -} diff --git a/arrow/test/dependency/simd/Cargo.toml b/arrow/test/dependency/simd/Cargo.toml deleted file mode 100644 index ecc6a93a8be9..000000000000 --- a/arrow/test/dependency/simd/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -name = "defeault-features" -description = "Models a user application of arrow that uses the simd feature of arrow" -version = "0.1.0" -edition = "2021" -rust-version = "1.57" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -arrow = { path = "../../../../arrow", version = "15.0.0", features = ["simd"]} - -[workspace] diff --git a/arrow/test/dependency/simd/src/main.rs b/arrow/test/dependency/simd/src/main.rs deleted file mode 100644 index e7a11a969c03..000000000000 --- a/arrow/test/dependency/simd/src/main.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("Hello, world!"); -} diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml index bb7a8cd10583..a805ea8c4214 100644 --- a/parquet/Cargo.toml +++ b/parquet/Cargo.toml @@ -39,18 +39,19 @@ brotli = { version = "3.3", optional = true } flate2 = { version = "1.0", optional = true } lz4 = { version = "1.23", optional = true } zstd = { version = "0.11.1", optional = true, default-features = false } -chrono = { version = "0.4", default-features = false } +chrono = { version = "0.4", default-features = false, features = ["alloc"] } num = "0.4" num-bigint = "0.4" arrow = { path = "../arrow", version = "15.0.0", optional = true, default-features = false, features = ["ipc"] } base64 = { version = "0.13", optional = true } clap = { version = "3", optional = true, features = ["derive", "env"] } -serde_json = { version = "1.0", features = ["preserve_order"], optional = true } +serde_json = { version = "1.0", optional = true } rand = "0.8" futures = { version = "0.3", optional = true } tokio = { version = "1.0", optional = true, default-features = false, features = ["macros", "fs", "rt", "io-util"] } [dev-dependencies] +base64 = "0.13" criterion = "0.3" rand = "0.8" snap = "1.0" @@ -58,19 +59,24 @@ tempfile = "3.0" brotli = "3.3" flate2 = "1.0" lz4 = "1.23" +zstd = "0.11" serde_json = { version = "1.0", features = ["preserve_order"] } -arrow = { path = "../arrow", version = "15.0.0", default-features = false, features = ["test_utils", "prettyprint"] } +arrow = { path = "../arrow", version = "15.0.0", default-features = false, features = ["ipc", "test_utils", "prettyprint"] } [package.metadata.docs.rs] all-features = true [features] default = ["arrow", "snap", "brotli", "flate2", "lz4", "zstd", "base64"] +# Enable arrow reader/writer APIs +arrow = ["dep:arrow", "base64"] +# Enable CLI tools cli = ["serde_json", "base64", "clap"] +# Enable internal testing APIs test_common = [] # Experimental, unstable functionality primarily used for testing experimental = [] -# Enable async API +# Enable async APIs async = ["futures", "tokio"] [[bin]] @@ -87,11 +93,12 @@ required-features = ["cli"] [[bench]] name = "arrow_writer" +required-features = ["arrow"] harness = false [[bench]] name = "arrow_reader" -required-features = ["test_common", "experimental"] +required-features = ["arrow", "test_common", "experimental"] harness = false [lib] diff --git a/parquet/README.md b/parquet/README.md index ed8b5518b8f9..fbb6e3e1b5d5 100644 --- a/parquet/README.md +++ b/parquet/README.md @@ -27,7 +27,7 @@ See [crate documentation](https://docs.rs/parquet/latest/parquet/) for examples ## Rust Version Compatbility -This crate is tested with the latest stable version of Rust. We do not currrently test against other, older versions of the Rust compiler. +This crate is tested with the latest stable version of Rust. We do not currently test against other, older versions of the Rust compiler. ## Features diff --git a/parquet/src/errors.rs b/parquet/src/errors.rs index fcbb846f110f..c2fb5bd66cf9 100644 --- a/parquet/src/errors.rs +++ b/parquet/src/errors.rs @@ -17,7 +17,7 @@ //! Common Parquet errors and macros. -use std::{cell, convert, io, result, str}; +use std::{cell, io, result, str}; #[cfg(any(feature = "arrow", test))] use arrow::error::ArrowError; @@ -108,7 +108,7 @@ pub type Result = result::Result; // ---------------------------------------------------------------------- // Conversion from `ParquetError` to other types of `Error`s -impl convert::From for io::Error { +impl From for io::Error { fn from(e: ParquetError) -> Self { io::Error::new(io::ErrorKind::Other, e) } @@ -135,6 +135,7 @@ macro_rules! eof_err { ($fmt:expr, $($args:expr),*) => (ParquetError::EOF(format!($fmt, $($args),*))); } +#[cfg(any(feature = "arrow", test))] macro_rules! arrow_err { ($fmt:expr) => (ParquetError::ArrowError($fmt.to_owned())); ($fmt:expr, $($args:expr),*) => (ParquetError::ArrowError(format!($fmt, $($args),*))); diff --git a/parquet/src/record/api.rs b/parquet/src/record/api.rs index 95b97bc9546e..5df21e4b0d00 100644 --- a/parquet/src/record/api.rs +++ b/parquet/src/record/api.rs @@ -27,7 +27,7 @@ use crate::data_type::{ByteArray, Decimal, Int96}; use crate::errors::{ParquetError, Result}; use crate::schema::types::ColumnDescPtr; -#[cfg(feature = "cli")] +#[cfg(any(feature = "cli", test))] use serde_json::Value; /// Macro as a shortcut to generate 'not yet implemented' panic error. @@ -79,7 +79,7 @@ impl Row { } } - #[cfg(feature = "cli")] + #[cfg(any(feature = "cli", test))] pub fn to_json_value(&self) -> Value { Value::Object( self.fields @@ -650,7 +650,7 @@ impl Field { } } - #[cfg(feature = "cli")] + #[cfg(any(feature = "cli", test))] pub fn to_json_value(&self) -> Value { match &self { Field::Null => Value::Null, @@ -669,7 +669,7 @@ impl Field { Field::Double(n) => serde_json::Number::from_f64(*n) .map(Value::Number) .unwrap_or(Value::Null), - Field::Decimal(n) => Value::String(convert_decimal_to_string(&n)), + Field::Decimal(n) => Value::String(convert_decimal_to_string(n)), Field::Str(s) => Value::String(s.to_owned()), Field::Bytes(b) => Value::String(base64::encode(b.data())), Field::Date(d) => Value::String(convert_date_to_string(*d)), @@ -1668,7 +1668,7 @@ mod tests { } #[test] - #[cfg(feature = "cli")] + #[cfg(any(feature = "cli", test))] fn test_to_json_value() { assert_eq!(Field::Null.to_json_value(), Value::Null); assert_eq!(Field::Bool(true).to_json_value(), Value::Bool(true)); @@ -1707,21 +1707,19 @@ mod tests { ); assert_eq!( Field::Float(5.0).to_json_value(), - Value::Number(serde_json::Number::from_f64(f64::from(5.0 as f32)).unwrap()) + Value::Number(serde_json::Number::from_f64(5.0).unwrap()) ); assert_eq!( Field::Float(5.1234).to_json_value(), - Value::Number( - serde_json::Number::from_f64(f64::from(5.1234 as f32)).unwrap() - ) + Value::Number(serde_json::Number::from_f64(5.1234_f32 as f64).unwrap()) ); assert_eq!( Field::Double(6.0).to_json_value(), - Value::Number(serde_json::Number::from_f64(6.0 as f64).unwrap()) + Value::Number(serde_json::Number::from_f64(6.0).unwrap()) ); assert_eq!( Field::Double(6.1234).to_json_value(), - Value::Number(serde_json::Number::from_f64(6.1234 as f64).unwrap()) + Value::Number(serde_json::Number::from_f64(6.1234).unwrap()) ); assert_eq!( Field::Str("abc".to_string()).to_json_value(), diff --git a/parquet_derive/test/dependency/README.md b/parquet_derive/test/dependency/README.md deleted file mode 100644 index b618b4636e7c..000000000000 --- a/parquet_derive/test/dependency/README.md +++ /dev/null @@ -1,21 +0,0 @@ - - -This directory contains projects that use arrow as a dependency with -various combinations of feature flags. diff --git a/parquet_derive/test/dependency/default-features/Cargo.toml b/parquet_derive/test/dependency/default-features/Cargo.toml deleted file mode 100644 index 7434552aaec4..000000000000 --- a/parquet_derive/test/dependency/default-features/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -name = "defeault-features" -description = "Models a user application of parquet_derive that uses no additional features of arrow" -version = "0.1.0" -edition = "2021" -rust-version = "1.57" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -parquet_derive = { path = "../../../../parquet_derive", version = "15.0.0" } - -# Keep this out of the default workspace -[workspace] diff --git a/parquet_derive/test/dependency/default-features/src/main.rs b/parquet_derive/test/dependency/default-features/src/main.rs deleted file mode 100644 index e7a11a969c03..000000000000 --- a/parquet_derive/test/dependency/default-features/src/main.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("Hello, world!"); -}