diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index aa85d9a991..eab61c6dc1 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -9,7 +9,7 @@ on: paths: - csharp/** - glide-core/src/** - - submodules/** + - glide-core/redis-rs/redis/src/** - .github/workflows/csharp.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml @@ -20,7 +20,7 @@ on: paths: - csharp/** - glide-core/src/** - - submodules/** + - glide-core/redis-rs/redis/src/** - .github/workflows/csharp.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml @@ -76,8 +76,7 @@ jobs: steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Set up dotnet ${{ matrix.dotnet }} uses: actions/setup-dotnet@v4 @@ -122,8 +121,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - uses: ./.github/workflows/lint-rust with: diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 6eaf3d1d19..3290839a6a 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -8,7 +8,7 @@ on: - v* paths: - glide-core/src/** - - submodules/** + - glide-core/redis-rs/redis/src/** - go/** - .github/workflows/go.yml - .github/workflows/install-shared-dependencies/action.yml @@ -19,7 +19,7 @@ on: pull_request: paths: - glide-core/src/** - - submodules/** + - glide-core/redis-rs/redis/src/** - go/** - .github/workflows/go.yml - .github/workflows/install-shared-dependencies/action.yml @@ -73,8 +73,7 @@ jobs: steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Set up Go ${{ matrix.go }} uses: actions/setup-go@v5 @@ -204,8 +203,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - uses: ./.github/workflows/lint-rust with: diff --git a/.github/workflows/install-redis/action.yml b/.github/workflows/install-redis/action.yml index f4a7491d04..b60f0687b5 100644 --- a/.github/workflows/install-redis/action.yml +++ b/.github/workflows/install-redis/action.yml @@ -17,8 +17,7 @@ runs: shell: bash - uses: actions/checkout@v4 - with: - submodules: recursive + - uses: actions/cache@v3 id: cache-redis diff --git a/.github/workflows/java-cd.yml b/.github/workflows/java-cd.yml index e9df283c50..d3f2038313 100644 --- a/.github/workflows/java-cd.yml +++ b/.github/workflows/java-cd.yml @@ -86,8 +86,7 @@ jobs: echo "No cleaning needed" fi - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Set up JDK uses: actions/setup-java@v4 @@ -225,8 +224,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - with: - submodules: recursive + - name: Set up JDK uses: actions/setup-java@v4 diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index ebc6a06169..2c31562f78 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -8,7 +8,7 @@ on: - v* paths: - glide-core/src/** - - submodules/** + - glide-core/redis-rs/redis/src/** - java/** - .github/workflows/java.yml - .github/workflows/install-shared-dependencies/action.yml @@ -19,7 +19,7 @@ on: pull_request: paths: - glide-core/src/** - - submodules/** + - glide-core/redis-rs/redis/src/** - java/** - .github/workflows/java.yml - .github/workflows/install-shared-dependencies/action.yml @@ -74,8 +74,7 @@ jobs: steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - uses: gradle/actions/wrapper-validation@v3 @@ -191,8 +190,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - uses: ./.github/workflows/lint-rust with: @@ -210,8 +208,7 @@ jobs: run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Set up JDK uses: actions/setup-java@v4 diff --git a/.github/workflows/lint-rust/action.yml b/.github/workflows/lint-rust/action.yml index 11ca944f71..0823dda958 100644 --- a/.github/workflows/lint-rust/action.yml +++ b/.github/workflows/lint-rust/action.yml @@ -15,8 +15,7 @@ runs: steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Install Rust toolchain and protoc uses: ./.github/workflows/install-rust-and-protoc @@ -42,7 +41,7 @@ runs: - run: | cargo update - cargo install --locked --version 0.15.1 cargo-deny + cargo install --locked cargo-deny cargo deny check --config ${GITHUB_WORKSPACE}/deny.toml working-directory: ${{ inputs.cargo-toml-folder }} shell: bash diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index a9b6b4be18..c4c17a7e46 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -8,7 +8,7 @@ on: - v* paths: - glide-core/src/** - - submodules/** + - glide-core/redis-rs/redis/src/** - node/** - utils/cluster_manager.py - .github/workflows/node.yml @@ -22,7 +22,7 @@ on: pull_request: paths: - glide-core/src/** - - submodules/** + - glide-core/redis-rs/redis/src/** - node/** - utils/cluster_manager.py - .github/workflows/node.yml @@ -67,8 +67,7 @@ jobs: steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Use Node.js 16.x uses: actions/setup-node@v3 @@ -125,8 +124,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - uses: ./.github/workflows/lint-rust with: @@ -239,8 +237,7 @@ jobs: apk add git - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Setup musl on Linux uses: ./.github/workflows/setup-musl-on-linux @@ -287,8 +284,7 @@ jobs: run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Use Node.js 18.x uses: actions/setup-node@v4 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index ebb83894e6..6c45d7707c 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -9,7 +9,7 @@ on: paths: - python/** - glide-core/src/** - - submodules/** + - glide-core/redis-rs/redis/src/**/** - utils/cluster_manager.py - .github/workflows/python.yml - .github/workflows/build-python-wrapper/action.yml @@ -25,7 +25,7 @@ on: paths: - python/** - glide-core/src/** - - submodules/** + - glide-core/redis-rs/redis/src/** - utils/cluster_manager.py - .github/workflows/python.yml - .github/workflows/build-python-wrapper/action.yml @@ -84,8 +84,7 @@ jobs: steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Set up Python uses: actions/setup-python@v4 @@ -167,8 +166,7 @@ jobs: steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Set up Python uses: actions/setup-python@v4 @@ -204,8 +202,7 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - name: lint rust uses: ./.github/workflows/lint-rust @@ -299,8 +296,6 @@ jobs: run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide - uses: actions/checkout@v4 - with: - submodules: recursive - name: Build Python wrapper uses: ./.github/workflows/build-python-wrapper diff --git a/.github/workflows/redis-rs.yml b/.github/workflows/redis-rs.yml new file mode 100644 index 0000000000..cdc4967759 --- /dev/null +++ b/.github/workflows/redis-rs.yml @@ -0,0 +1,142 @@ +name: Redis-rs CI + +on: + push: + branches: + - main + - release-* + - v* + paths: + - glide-core/redis-rs/redis/** + - utils/cluster_manager.py + - deny.toml + - .github/workflows/install-shared-dependencies/action.yml + - .github/workflows/redis-rs.yml + pull_request: + paths: + - glide-core/redis-rs/redis/** + - utils/cluster_manager.py + - deny.toml + - .github/workflows/install-shared-dependencies/action.yml + - .github/workflows/redis-rs.yml + workflow_dispatch: + workflow_call: + +concurrency: + group: redis-rs-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + redis-rs-CI: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: "ubuntu" + target: "x86_64-unknown-linux-gnu" + engine-version: "7.2.5" + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Cache dependencies + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + workspaces: ./glide-core/redis-rs/redis + + + - name: Build project + run: cargo build --release + working-directory: ./glide-core/redis-rs/redis/src + + - name: Lint redis-rs + shell: bash + run: | + cargo fmt --all -- --check + cargo clippy -- -D warnings + cargo install --locked cargo-deny + cargo deny check all --config ${GITHUB_WORKSPACE}/deny.toml --exclude-dev all + working-directory: ./glide-core/redis-rs/redis + + - name: Test + run: | + cargo test --release -- --test-threads=1 | tee ../test-results.xml + echo "### Tests passed :v:" >> $GITHUB_STEP_SUMMARY + working-directory: ./glide-core/redis-rs/redis/src + + - name: Upload test reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: test-reports-redis-rs-${{ github.sha }} + path: ./glide-core/redis-rs/redis/test-results.xml + + - name: Run benchmarks + run: | + cargo bench | tee bench-results.xml + working-directory: ./glide-core/redis-rs/redis + + - name: Upload benchmark results + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-redis-rs-${{ github.sha }} + path: ./glide-core/redis-rs/redis/bench-results.xml + + - name: Test docs + run: | + cargo test --doc + working-directory: ./glide-core/redis-rs/redis/src + + - name: Security audit + run: | + cargo audit | tee audit-results.txt + if grep -q "Crate: " audit-results.txt; then + echo "## Security audit results summary: Security vulnerabilities found :exclamation: :exclamation:" >> $GITHUB_STEP_SUMMARY + echo "Security audit results summary: Security vulnerabilities found" + exit 1 + else + echo "### Security audit results summary: All good, no security vulnerabilities found :closed_lock_with_key:" >> $GITHUB_STEP_SUMMARY + echo "Security audit results summary: All good, no security vulnerabilities found" + fi + working-directory: ./glide-core/redis-rs/redis + + - name: Upload audit results + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: audit-results-redis-rs--${{ github.sha }} + path: ./glide-core/redis-rs/redis/audit-results.txt + + - name: Run cargo machete + run: | + cargo install cargo-machete + cargo machete | tee machete-results.txt + if grep -A1 "cargo-machete found the following unused dependencies in this directory:" machete-results.txt | sed -n '2p' | grep -v "^if" > /dev/null; then + echo "Machete results summary: Unused dependencies found" >> $GITHUB_STEP_SUMMARY + echo "Machete results summary: Unused dependencies found" + cat machete-results.txt | grep -A1 "cargo-machete found the following unused dependencies in this directory:" | sed -n '2p' | grep -v "^if" >> $GITHUB_STEP_SUMMARY + exit 1 + else + echo "### Machete results summary: All good, no unused dependencies found :rocket:" >> $GITHUB_STEP_SUMMARY + echo "Machete results summary: All good, no unused dependencies found" + fi + working-directory: ./glide-core/redis-rs/redis + + - name: Upload machete results + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: machete-results-redis-rs-${{ github.sha }} + path: ./glide-core/redis-rs/redis/machete-results.txt diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c632880a2b..95b47f2ce2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -9,7 +9,7 @@ on: paths: - logger_core/** - glide-core/** - - submodules/** + - glide-core/redis-rs/redis/src/** - utils/cluster_manager.py - .github/workflows/rust.yml - .github/workflows/install-shared-dependencies/action.yml @@ -21,7 +21,7 @@ on: paths: - logger_core/** - glide-core/** - - submodules/** + - glide-core/redis-rs/redis/src/** - utils/cluster_manager.py - .github/workflows/rust.yml - .github/workflows/install-shared-dependencies/action.yml @@ -63,8 +63,7 @@ jobs: steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - name: Install shared software dependencies uses: ./.github/workflows/install-shared-dependencies @@ -78,11 +77,11 @@ jobs: - name: Run tests working-directory: ./glide-core - run: cargo test --all-features -- --nocapture --test-threads=1 # TODO remove the concurrency limit after we fix test flakyness. + run: cargo test --all-features --release -- --test-threads=1 # TODO remove the concurrency limit after we fix test flakyness. - name: Run logger tests working-directory: ./logger_core - run: cargo test --all-features -- --nocapture --test-threads=1 + run: cargo test --all-features -- --test-threads=1 - name: Check features working-directory: ./glide-core @@ -99,8 +98,7 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v4 - with: - submodules: recursive + - uses: ./.github/workflows/lint-rust with: diff --git a/deny.toml b/deny.toml index f97a82f0c8..a0c30726d4 100644 --- a/deny.toml +++ b/deny.toml @@ -9,24 +9,6 @@ # The values provided in this template are the default values that will be used # when any section or field is not specified in your own configuration -# If 1 or more target triples (and optionally, target_features) are specified, -# only the specified targets will be checked when running `cargo deny check`. -# This means, if a particular package is only ever used as a target specific -# dependency, such as, for example, the `nix` crate only being used via the -# `target_family = "unix"` configuration, that only having windows targets in -# this list would mean the nix crate, as well as any of its exclusive -# dependencies not shared by any other crates, would be ignored, as the target -# list here is effectively saying which targets you are building for. -targets = [ - # The triple can be any string, but only the target triples built in to - # rustc (as of 1.40) can be checked against actual config expressions - #{ triple = "x86_64-unknown-linux-musl" }, - # You can also specify which target_features you promise are enabled for a - # particular target. target_features are currently not validated against - # the actual valid features supported by the target architecture. - #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, -] - # This section is considered when running `cargo deny check advisories` # More documentation for the advisories section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html @@ -35,22 +17,13 @@ targets = [ db-path = "~/.cargo/advisory-db" # The url(s) of the advisory databases to use db-urls = ["https://github.com/rustsec/advisory-db"] -# The lint level for security vulnerabilities -vulnerability = "deny" -# The lint level for unmaintained crates -unmaintained = "deny" # The lint level for crates that have been yanked from their source registry yanked = "deny" -# The lint level for crates with security notices. Note that as of -# 2019-12-17 there are no security notice advisories in -# https://github.com/rustsec/advisory-db -notice = "deny" -unsound = "deny" # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ # Unmaintained dependency error that needs more attention due to nested dependencies - "RUSTSEC-2024-0370" + "RUSTSEC-2024-0370", ] # Threshold for security vulnerabilities, any vulnerability with a CVSS score # lower than the range specified will be ignored. Note that ignored advisories @@ -72,8 +45,6 @@ ignore = [ # More documentation for the licenses section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html [licenses] -# The lint level for crates which do not have a detectable license -unlicensed = "deny" # List of explicitly allowed licenses # See https://spdx.org/licenses/ for list of possible licenses # [possible values: any SPDX 3.11 short identifier (+ optional exception)]. @@ -85,28 +56,9 @@ allow = [ "BSD-3-Clause", "Unicode-DFS-2016", "ISC", - "OpenSSL" -] -# List of explicitly disallowed licenses -# See https://spdx.org/licenses/ for list of possible licenses -# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. -deny = [ - #"Nokia", + "OpenSSL", + "MPL-2.0", ] -# Lint level for licenses considered copyleft -copyleft = "deny" -# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses -# * both - The license will be approved if it is both OSI-approved *AND* FSF -# * either - The license will be approved if it is either OSI-approved *OR* FSF -# * osi-only - The license will be approved if is OSI-approved *AND NOT* FSF -# * fsf-only - The license will be approved if is FSF *AND NOT* OSI-approved -# * neither - This predicate is ignored and the default lint level is used -allow-osi-fsf-free = "neither" -# Lint level used when no other predicates are matched -# 1. License isn't in the allow or deny lists -# 2. License isn't copyleft -# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither" -default = "deny" # The confidence threshold for detecting a license from license text. # The higher the value, the more closely the license text must be to the # canonical license text of a valid SPDX license file. @@ -137,7 +89,7 @@ expression = "MIT AND ISC AND OpenSSL" # depending on the rest of your configuration license-files = [ # Each entry is a crate relative path, and the (opaque) hash of its contents - { path = "LICENSE", hash = 0xbd0eed23 } + { path = "LICENSE", hash = 0xbd0eed23 }, ] [licenses.private] diff --git a/glide-core/Cargo.toml b/glide-core/Cargo.toml index 28cad8e646..150c0ff33d 100644 --- a/glide-core/Cargo.toml +++ b/glide-core/Cargo.toml @@ -32,7 +32,7 @@ integer-encoding = { version = "4.0.0", optional = true } thiserror = "1" rand = { version = "0.8.5" } futures-intrusive = "0.5.0" -directories = { version = "4.0", optional = true } +directories = { version = "5.0", optional = true } once_cell = "1.18.0" sha1_smol = "1.0.0" nanoid = "0.4.0" @@ -52,13 +52,13 @@ standalone_heartbeat = [] rsevents = "0.3.1" socket2 = "^0.5" tempfile = "3.3.0" -rstest = "^0.18" +rstest = "^0.23" serial_test = "3" criterion = { version = "^0.5", features = ["html_reports", "async_tokio"] } -which = "5" +which = "6" ctor = "0.2.2" redis = { path = "./redis-rs/redis", features = ["tls-rustls-insecure"] } -iai-callgrind = "0.9" +iai-callgrind = "0.14" tokio = { version = "1", features = ["rt-multi-thread"] } glide-core = { path = ".", features = [ "socket-layer", diff --git a/glide-core/redis-rs/redis/src/aio/multiplexed_connection.rs b/glide-core/redis-rs/redis/src/aio/multiplexed_connection.rs index c23d4dfca4..15df4e9aa8 100644 --- a/glide-core/redis-rs/redis/src/aio/multiplexed_connection.rs +++ b/glide-core/redis-rs/redis/src/aio/multiplexed_connection.rs @@ -23,7 +23,6 @@ use pin_project_lite::pin_project; use std::collections::VecDeque; use std::fmt; use std::fmt::Debug; -use std::io; use std::pin::Pin; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc;