diff --git a/.github/PULL_REQUEST_TEMPLATE/release-checklist.md b/.github/PULL_REQUEST_TEMPLATE/release-checklist.md index f18de3d5195..1f010c13a07 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release-checklist.md +++ b/.github/PULL_REQUEST_TEMPLATE/release-checklist.md @@ -23,7 +23,8 @@ Sometimes `dependabot` misses some dependency updates, or we accidentally turned Here's how we make sure we got everything: - [ ] Run `cargo update` on the latest `main` branch, and keep the output -- [ ] If needed, update [deny.toml](https://github.com/ZcashFoundation/zebra/blob/main/book/src/dev/continuous-integration.md#fixing-duplicate-dependencies-in-check-denytoml-bans) +- [ ] If needed, [add duplicate dependency exceptions to deny.toml](https://github.com/ZcashFoundation/zebra/blob/main/book/src/dev/continuous-integration.md#fixing-duplicate-dependencies-in-check-denytoml-bans) +- [ ] If needed, remove resolved duplicate dependencies from `deny.toml` - [ ] Open a separate PR with the changes - [ ] Add the output of `cargo update` to that PR as a comment @@ -157,10 +158,7 @@ The end of support height is calculated from the current blockchain height: and put the output in a comment on the PR. ## Publish Docker Images -- [ ] Wait until [the Docker images have been published](https://github.com/ZcashFoundation/zebra/actions/workflows/release-binaries.yml) -- [ ] Test the Docker image using `docker run --tty --interactive zfnd/zebra:v1.0.0`, - and put the output in a comment on the PR. - (You can use [gcloud cloud shell](https://console.cloud.google.com/home/dashboard?cloudshell=true)) +- [ ] Wait for the [the Docker images to be published successfully](https://github.com/ZcashFoundation/zebra/actions/workflows/release-binaries.yml). - [ ] Un-freeze the [`batched` queue](https://dashboard.mergify.com/github/ZcashFoundation/repo/zebra/queues) using Mergify. ## Release Failures diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 6ae6f28166d..95c187bec97 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -38,7 +38,7 @@ on: # https://github.com/ZcashFoundation/zebra/blob/main/docker/Dockerfile#L83 features: required: false - default: "sentry" + default: "default-release-binaries" type: string test_features: required: false diff --git a/.github/workflows/continous-delivery.patch.yml b/.github/workflows/continous-delivery.patch.yml new file mode 100644 index 00000000000..53ecb144a9a --- /dev/null +++ b/.github/workflows/continous-delivery.patch.yml @@ -0,0 +1,36 @@ +name: CD + +on: + # Only patch the Docker image test jobs + pull_request: + paths-ignore: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # configuration files + - '.cargo/config.toml' + - '**/clippy.toml' + # workflow definitions + - 'docker/**' + - '.dockerignore' + - '.github/workflows/continous-delivery.yml' + - '.github/workflows/find-cached-disks.yml' + + +jobs: + # Also patched by continous-integration-docker.patch.yml, which has a different paths-ignore + build: + name: Build CI Docker / Build images + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-configuration-file: + name: Test Zebra CD Docker config file + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' diff --git a/.github/workflows/continous-delivery.yml b/.github/workflows/continous-delivery.yml index 15c8c08a615..8cd334560fc 100644 --- a/.github/workflows/continous-delivery.yml +++ b/.github/workflows/continous-delivery.yml @@ -6,9 +6,12 @@ name: CD # # Since the different event types each use a different Managed Instance Group or instance, # we can run different event types concurrently. +# +# For pull requests, we only run the tests from this workflow, and don't do any deployments. +# So an in-progress pull request gets cancelled, just like other tests. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} - cancel-in-progress: false + cancel-in-progress: ${{ github.event_name == 'pull_request' }} on: workflow_dispatch: @@ -25,14 +28,52 @@ on: required: false type: boolean default: false + # Temporarily disabled to reduce network load, see #6894. #push: # branches: # - main + # paths: + # # code and tests + # - '**/*.rs' + # # hard-coded checkpoints and proptest regressions + # - '**/*.txt' + # # dependencies + # - '**/Cargo.toml' + # - '**/Cargo.lock' + # # configuration files + # - '.cargo/config.toml' + # - '**/clippy.toml' + # # workflow definitions + # - 'docker/**' + # - '.dockerignore' + # - '.github/workflows/continous-delivery.yml' + # - '.github/workflows/build-docker-image.yml' + + # Only runs the Docker image tests, doesn't deploy any instances + pull_request: + paths: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # configuration files + - '.cargo/config.toml' + - '**/clippy.toml' + # workflow definitions + - 'docker/**' + - '.dockerignore' + - '.github/workflows/continous-delivery.yml' + - '.github/workflows/find-cached-disks.yml' + release: types: - published + jobs: # If a release was made we want to extract the first part of the semver from the # tag_name @@ -82,7 +123,7 @@ jobs: # Test that Zebra works using the default config with the latest Zebra version, # and test reconfiguring the docker image for testnet. test-configuration-file: - name: Test Zebra default Docker config file + name: Test Zebra CD Docker config file timeout-minutes: 15 runs-on: ubuntu-latest needs: build diff --git a/.github/workflows/continous-integration-docker.patch.yml b/.github/workflows/continous-integration-docker.patch.yml index b3cf3a8b537..489d75b01b3 100644 --- a/.github/workflows/continous-integration-docker.patch.yml +++ b/.github/workflows/continous-integration-docker.patch.yml @@ -19,8 +19,10 @@ on: - '**/clippy.toml' # workflow definitions - 'docker/**' + - '.dockerignore' - '.github/workflows/continous-integration-docker.yml' - '.github/workflows/deploy-gcp-tests.yml' + - '.github/workflows/find-cached-disks.yml' - '.github/workflows/build-docker-image.yml' jobs: diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 40d2313a894..902218ef266 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -80,10 +80,11 @@ on: - '**/clippy.toml' # workflow definitions - 'docker/**' + - '.dockerignore' - '.github/workflows/continous-integration-docker.yml' - '.github/workflows/deploy-gcp-tests.yml' - - '.github/workflows/build-docker-image.yml' - '.github/workflows/find-cached-disks.yml' + - '.github/workflows/build-docker-image.yml' jobs: # to also run a job on Mergify head branches, diff --git a/.github/workflows/continous-integration-os.yml b/.github/workflows/continous-integration-os.yml index 93303977e95..3a189653a82 100644 --- a/.github/workflows/continous-integration-os.yml +++ b/.github/workflows/continous-integration-os.yml @@ -261,8 +261,8 @@ jobs: # We don't need to check `--no-default-features` here, because (except in very rare cases): # - disabling features isn't going to add duplicate dependencies # - disabling features isn't going to add more crate sources - features: ['', '--all-features'] - # We always want to run the --all-features job, because it gives accurate "skip tree root was not found" warnings + features: ['', '--features default-release-binaries', '--all-features'] + # Always run the --all-features job, to get accurate "skip tree root was not found" warnings fail-fast: false # Prevent sudden announcement of a new advisory from failing ci: @@ -274,12 +274,14 @@ jobs: persist-credentials: false - uses: r7kamura/rust-problem-matchers@v1.3.0 - # The --all-features job is the only job that gives accurate "skip tree root was not found" warnings. - # In other jobs, we expect some of these warnings, due to disabled features. - name: Check ${{ matrix.checks }} with features ${{ matrix.features }} uses: EmbarkStudios/cargo-deny-action@v1 with: - command: check ${{ matrix.checks }} + # --all-features spuriously activates openssl, but we want to ban that dependency in + # all of zebrad's production features for security reasons. But the --all-features job is + # the only job that gives accurate "skip tree root was not found" warnings. + # In other jobs, we expect some of these warnings, due to disabled features. + command: check ${{ matrix.checks }} ${{ matrix.features == '--all-features' && '--allow banned' || '--allow unmatched-skip-root' }} arguments: --workspace ${{ matrix.features }} unused-deps: diff --git a/.github/workflows/dockerhub-description.yml b/.github/workflows/dockerhub-description.yml index 958a23bc8e8..55a02bb920f 100644 --- a/.github/workflows/dockerhub-description.yml +++ b/.github/workflows/dockerhub-description.yml @@ -22,7 +22,7 @@ jobs: persist-credentials: false - name: Docker Hub Description - uses: peter-evans/dockerhub-description@v3.4.1 + uses: peter-evans/dockerhub-description@v3.4.2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d2f56cbd1f9..d3427fd19a3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -37,7 +37,7 @@ jobs: - name: Rust files id: changed-files-rust - uses: tj-actions/changed-files@v36.4.0 + uses: tj-actions/changed-files@v37.0.3 with: files: | **/*.rs @@ -49,7 +49,7 @@ jobs: - name: Workflow files id: changed-files-workflows - uses: tj-actions/changed-files@v36.4.0 + uses: tj-actions/changed-files@v37.0.3 with: files: | .github/workflows/*.yml diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 2ec7338b5d5..81ee18c7b18 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -44,7 +44,7 @@ jobs: tag_suffix: .experimental network: Testnet rpc_port: '18232' - features: "sentry getblocktemplate-rpcs" + features: "default-release-binaries getblocktemplate-rpcs" test_features: "" rust_backtrace: '1' zebra_skip_ipv6_tests: '1' diff --git a/Cargo.lock b/Cargo.lock index fdaf7eceba6..63195c9fd51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,7 +12,7 @@ dependencies = [ "arc-swap", "backtrace", "canonical-path", - "clap 4.3.4", + "clap 4.3.8", "color-eyre", "fs-err", "once_cell", @@ -773,9 +773,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.4" +version = "4.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80672091db20273a15cf9fdd4e47ed43b5091ec9841bf4c6145c9dfbbcae09ed" +checksum = "d9394150f5b4273a1763355bd1c2ec54cc5a2593f790587bcd6b2c947cfa9211" dependencies = [ "clap_builder", "clap_derive", @@ -784,9 +784,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.4" +version = "4.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1458a1df40e1e2afebb7ab60ce55c1fa8f431146205aa5f4887e0b111c27636" +checksum = "9a78fbdd3cc2914ddf37ba444114bc7765bbdcb55ec9cbe6fa054f0137400717" dependencies = [ "anstream", "anstyle", @@ -956,10 +956,10 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.3.4", + "clap 4.3.8", "criterion-plot", "is-terminal", - "itertools", + "itertools 0.10.5", "num-traits", "once_cell", "oorandom", @@ -980,7 +980,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -2101,9 +2101,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.29.0" +version = "1.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a28d25139df397cbca21408bb742cf6837e04cdbebf1b07b760caf971d6a972" +checksum = "28491f7753051e5704d4d0ae7860d45fae3238d7d235bc4289dcd45c48d3cec3" dependencies = [ "console", "lazy_static", @@ -2163,6 +2163,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -3296,7 +3305,7 @@ checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", "heck 0.4.1", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "multimap", @@ -3317,7 +3326,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.5", "proc-macro2 1.0.60", "quote 1.0.28", "syn 1.0.109", @@ -5669,7 +5678,7 @@ dependencies = [ "hex", "humantime", "incrementalmerkletree", - "itertools", + "itertools 0.11.0", "jubjub", "lazy_static", "num-integer", @@ -5765,7 +5774,7 @@ dependencies = [ "howudoin", "humantime-serde", "indexmap", - "itertools", + "itertools 0.11.0", "lazy_static", "metrics 0.21.0", "num-integer", @@ -5864,7 +5873,7 @@ dependencies = [ "howudoin", "indexmap", "insta", - "itertools", + "itertools 0.11.0", "jubjub", "lazy_static", "metrics 0.21.0", @@ -5923,7 +5932,7 @@ version = "1.0.0-beta.26" dependencies = [ "color-eyre", "hex", - "itertools", + "itertools 0.11.0", "regex", "reqwest", "serde_json", @@ -5945,7 +5954,7 @@ dependencies = [ "abscissa_core", "atty", "chrono", - "clap 4.3.4", + "clap 4.3.8", "color-eyre", "console-subscriber", "dirs", diff --git a/deny.toml b/deny.toml index 30c7a846928..900d08ef9e0 100644 --- a/deny.toml +++ b/deny.toml @@ -10,6 +10,13 @@ [bans] # Lint level for when multiple versions of the same crate are detected multiple-versions = "deny" + +# Don't allow wildcard ("any version") dependencies +wildcards = "deny" +# Allow private and dev wildcard dependencies. +# Switch this to `false` when #6924 is implemented. +allow-wildcard-paths = true + # The graph highlighting used when creating dotgraphs for crates # with multiple versions # * lowest-version - The path to the lowest versioned duplicate is highlighted @@ -17,12 +24,18 @@ multiple-versions = "deny" # * all - Both lowest-version and simplest-path are used highlight = "all" -# We don't use this for Zebra. -# # List of crates that are allowed. Use with care! #allow = [ #] +# List of crates that can never become Zebra dependencies. +deny = [ + # Often has memory safety vulnerabilities. + # Enabled by --all-features, use the `cargo hack` script in the deny.toml CI job instead. + { name = "openssl" }, + { name = "openssl-sys" }, +] + # We only use this for some `librustzcash` and `orchard` crates. # If we add a crate here, duplicate dependencies of that crate are still shown. # @@ -44,6 +57,12 @@ skip-tree = [ # wait for prost-build to upgrade { name = "prettyplease", version = "=0.1.25" }, + # wait for criterion to upgrade + { name = "itertools", version = "=0.10.5" }, + + # wait for backtrace and multiple dependents to upgrade + { name = "miniz_oxide", version = "=0.6.2" }, + # ZF crates # wait for zcashd and zcash_script to upgrade @@ -68,6 +87,9 @@ skip-tree = [ # wait for zcash_address to upgrade { name = "bs58", version = "=0.4.0" }, + # wait for minreq and zcash_proofs to upgrade + { name = "rustls", version = "=0.20.8" }, + # zebra-utils dependencies # wait for structopt upgrade (or upgrade to clap 4) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4f21f9aee5d..3b636339797 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -29,7 +29,7 @@ FROM chef AS deps SHELL ["/bin/bash", "-xo", "pipefail", "-c"] COPY --from=planner /opt/zebrad/recipe.json recipe.json -# Install zebra build deps +# Install zebra build deps and Dockerfile deps RUN apt-get -qq update && \ apt-get -qq install -y --no-install-recommends \ llvm \ @@ -37,6 +37,7 @@ RUN apt-get -qq update && \ clang \ ca-certificates \ protobuf-compiler \ + rsync \ ; \ rm -rf /var/lib/apt/lists/* /tmp/* @@ -80,7 +81,7 @@ ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS:-1} # Build zebrad with these features # Keep these in sync with: # https://github.com/ZcashFoundation/zebra/blob/main/.github/workflows/build-docker-image.yml#L42 -ARG FEATURES="sentry" +ARG FEATURES="default-release-binaries" ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints" # Use ENTRYPOINT_FEATURES to override the specific features used to run tests in entrypoint.sh, # separately from the test and production image builds. @@ -102,16 +103,28 @@ FROM deps AS tests COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/zcash-params /root/.zcash-params /root/.zcash-params COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /opt/lightwalletd /usr/local/bin +# cargo uses timestamps for its cache, so they need to be in this order: +# unmodified source files < previous build cache < modified source files +COPY . . + # Re-hydrate the minimum project skeleton identified by `cargo chef prepare` in the planner stage, +# over the top of the original source files, # and build it to cache all possible sentry and test dependencies. # -# This is the caching Docker layer for Rust! +# This is the caching Docker layer for Rust tests! +# It creates fake empty test binaries so dependencies are built, but Zebra is not fully built. # # TODO: add --locked when cargo-chef supports it RUN cargo chef cook --tests --release --features "${TEST_FEATURES} ${FEATURES}" --workspace --recipe-path recipe.json -COPY . . -# Test Zebra +# Undo the source file changes made by cargo-chef. +# rsync invalidates the cargo cache for the changed files only, by updating their timestamps. +# This makes sure the fake empty binaries created by cargo-chef are rebuilt. +COPY --from=planner /opt/zebrad zebra-original +RUN rsync --recursive --checksum --itemize-changes --verbose zebra-original/ . +RUN rm -r zebra-original + +# Build Zebra test binaries, but don't run them RUN cargo test --locked --release --features "${TEST_FEATURES} ${FEATURES}" --workspace --no-run RUN cp /opt/zebrad/target/release/zebrad /usr/local/bin RUN cp /opt/zebrad/target/release/zebra-checkpoints /usr/local/bin @@ -129,10 +142,19 @@ ENTRYPOINT [ "/entrypoint.sh" ] # zebrad binary from this step. FROM deps AS release +COPY . . + +# This is the caching layer for Rust zebrad builds. +# It creates a fake empty zebrad binary, see above for details. +# # TODO: add --locked when cargo-chef supports it RUN cargo chef cook --release --features "${FEATURES}" --package zebrad --bin zebrad --recipe-path recipe.json -COPY . . +# Undo the source file changes made by cargo-chef, so the fake empty zebrad binary is rebuilt. +COPY --from=planner /opt/zebrad zebra-original +RUN rsync --recursive --checksum --itemize-changes --verbose zebra-original/ . +RUN rm -r zebra-original + # Build zebrad RUN cargo build --locked --release --features "${FEATURES}" --package zebrad --bin zebrad diff --git a/docker/zcash-params/Dockerfile b/docker/zcash-params/Dockerfile index 1036a2be40b..a9bea7233a6 100644 --- a/docker/zcash-params/Dockerfile +++ b/docker/zcash-params/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get -qq update && \ ENV CARGO_HOME /opt/zebrad/.cargo/ # Build dependencies - this is the caching Docker layer! -RUN cargo chef cook --release --features sentry --package zebrad --recipe-path recipe.json +RUN cargo chef cook --release --features default-release-binaries --package zebrad --recipe-path recipe.json ARG RUST_BACKTRACE=0 ENV RUST_BACKTRACE ${RUST_BACKTRACE} @@ -36,4 +36,4 @@ ENV COLORBT_SHOW_HIDDEN ${COLORBT_SHOW_HIDDEN} COPY . . # Pre-download Zcash Sprout and Sapling parameters -RUN cargo run --locked --release --features sentry --package zebrad --bin zebrad download +RUN cargo run --locked --release --features default-release-binaries --package zebrad --bin zebrad download diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index 905b548efa2..7615d602c34 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -96,7 +96,7 @@ serde-big-array = "0.5.1" # Processing futures = "0.3.28" -itertools = "0.10.5" +itertools = "0.11.0" rayon = "1.7.0" # ZF deps diff --git a/zebra-chain/src/orchard/shielded_data.rs b/zebra-chain/src/orchard/shielded_data.rs index dc55d19a8f7..3a034c05f0f 100644 --- a/zebra-chain/src/orchard/shielded_data.rs +++ b/zebra-chain/src/orchard/shielded_data.rs @@ -269,9 +269,6 @@ impl ZcashDeserialize for Flags { // Consensus rule: "In a version 5 transaction, // the reserved bits 2..7 of the flagsOrchard field MUST be zero." // https://zips.z.cash/protocol/protocol.pdf#txnencodingandconsensus - // - // Clippy 1.64 is wrong here, this lazy evaluation is necessary, constructors are functions. This is fixed in 1.66. - #[allow(clippy::unnecessary_lazy_evaluations)] Flags::from_bits(reader.read_u8()?) .ok_or_else(|| SerializationError::Parse("invalid reserved orchard flags")) } diff --git a/zebra-chain/src/transaction/serialize.rs b/zebra-chain/src/transaction/serialize.rs index da6a3770bf1..f79244da6ea 100644 --- a/zebra-chain/src/transaction/serialize.rs +++ b/zebra-chain/src/transaction/serialize.rs @@ -884,9 +884,6 @@ impl ZcashDeserialize for Transaction { } // Denoted as `nConsensusBranchId` in the spec. // Convert it to a NetworkUpgrade - // - // Clippy 1.64 is wrong here, this lazy evaluation is necessary, constructors are functions. This is fixed in 1.66. - #[allow(clippy::unnecessary_lazy_evaluations)] let network_upgrade = NetworkUpgrade::from_branch_id(limited_reader.read_u32::()?) .ok_or_else(|| { diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index 3b694ac6773..970cf4118aa 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -280,7 +280,7 @@ where check::miner_fees_are_valid(&block, network, block_miner_fees)?; // Finally, submit the block for contextual verification. - let new_outputs = Arc::try_unwrap(known_utxos) + let new_outputs = Arc::into_inner(known_utxos) .expect("all verification tasks using known_utxos are complete"); let prepared_block = zs::SemanticallyVerifiedBlock { diff --git a/zebra-network/Cargo.toml b/zebra-network/Cargo.toml index cd1029e6325..7bb8e14f0ce 100644 --- a/zebra-network/Cargo.toml +++ b/zebra-network/Cargo.toml @@ -48,7 +48,7 @@ dirs = "5.0.1" hex = "0.4.3" humantime-serde = "1.1.1" indexmap = { version = "1.9.3", features = ["serde"] } -itertools = "0.10.5" +itertools = "0.11.0" lazy_static = "1.4.0" num-integer = "0.1.45" ordered-map = "0.4.2" diff --git a/zebra-network/src/protocol/external/codec.rs b/zebra-network/src/protocol/external/codec.rs index 7aee299dafa..6a4ae0585eb 100644 --- a/zebra-network/src/protocol/external/codec.rs +++ b/zebra-network/src/protocol/external/codec.rs @@ -500,8 +500,6 @@ impl Codec { /// Note: zcashd only requires fields up to `address_recv`, but everything up to `relay` is required in Zebra. /// see fn read_version(&self, mut reader: R) -> Result { - // Clippy 1.64 is wrong here, this lazy evaluation is necessary, constructors are functions. This is fixed in 1.66. - #[allow(clippy::unnecessary_lazy_evaluations)] Ok(VersionMessage { version: Version(reader.read_u32::()?), // Use from_bits_truncate to discard unknown service bits. diff --git a/zebra-node-services/Cargo.toml b/zebra-node-services/Cargo.toml index 70b469727ea..9d9b3885a50 100644 --- a/zebra-node-services/Cargo.toml +++ b/zebra-node-services/Cargo.toml @@ -42,7 +42,8 @@ zebra-chain = { path = "../zebra-chain" , version = "1.0.0-beta.26"} # Tool and test feature rpc-client color-eyre = { version = "0.6.2", optional = true } jsonrpc-core = { version = "18.0.0", optional = true } -reqwest = { version = "0.11.18", optional = true } +# Security: avoid default dependency on openssl +reqwest = { version = "0.11.18", default-features = false, features = ["rustls-tls"], optional = true } serde = { version = "1.0.164", optional = true } serde_json = { version = "1.0.97", optional = true } @@ -50,6 +51,6 @@ serde_json = { version = "1.0.97", optional = true } color-eyre = "0.6.2" jsonrpc-core = "18.0.0" -reqwest = "0.11.18" +reqwest = { version = "0.11.18", default-features = false, features = ["rustls-tls"] } serde = "1.0.164" serde_json = "1.0.97" diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index e9f6976e6ee..e9e4aa0e992 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -78,7 +78,7 @@ zebra-script = { path = "../zebra-script", version = "1.0.0-beta.26" } zebra-state = { path = "../zebra-state", version = "1.0.0-beta.26" } [dev-dependencies] -insta = { version = "1.29.0", features = ["redactions", "json", "ron"] } +insta = { version = "1.30.0", features = ["redactions", "json", "ron"] } proptest = "1.2.0" diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index 89ea428ab04..e4dcda077a9 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -49,7 +49,7 @@ dirs = "5.0.1" futures = "0.3.28" hex = "0.4.3" indexmap = "1.9.3" -itertools = "0.10.5" +itertools = "0.11.0" lazy_static = "1.4.0" metrics = "0.21.0" mset = "0.1.1" @@ -67,7 +67,8 @@ tower = { version = "0.4.13", features = ["buffer", "util"] } tracing = "0.1.37" # elasticsearch specific dependencies. -elasticsearch = { version = "8.5.0-alpha.1", package = "elasticsearch", optional = true } +# Security: avoid default dependency on openssl +elasticsearch = { version = "8.5.0-alpha.1", default-features = false, features = ["rustls-tls"], optional = true } serde_json = { version = "1.0.97", package = "serde_json", optional = true } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.26" } @@ -90,7 +91,7 @@ once_cell = "1.18.0" spandoc = "0.2.2" hex = { version = "0.4.3", features = ["serde"] } -insta = { version = "1.29.0", features = ["ron"] } +insta = { version = "1.30.0", features = ["ron"] } proptest = "1.2.0" proptest-derive = "0.3.0" diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index f4cd7213e47..cf495311efb 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -51,7 +51,7 @@ pub struct CommitSemanticallyVerifiedError(#[from] ValidateContextError); #[non_exhaustive] #[allow(missing_docs)] pub enum ValidateContextError { - #[error("block parent not found in any chain")] + #[error("block parent not found in any chain, or not enough blocks in chain")] #[non_exhaustive] NotReadyToBeCommitted, diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 98fff194237..d231403a729 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -262,6 +262,10 @@ pub struct SemanticallyVerifiedBlockWithTrees { pub treestate: Treestate, } +/// Contains a block ready to be committed. +/// +/// Zebra's state service passes this `enum` over to the finalized state +/// when committing a block. pub enum FinalizableBlock { Checkpoint { checkpoint_verified: CheckpointVerifiedBlock, @@ -273,7 +277,7 @@ pub enum FinalizableBlock { } impl FinalizableBlock { - /// Create a new `FinalizedBlock` given a `ContextuallyVerifiedBlock`. + /// Create a new [`FinalizableBlock`] given a [`ContextuallyVerifiedBlock`]. pub fn new(contextually_verified: ContextuallyVerifiedBlock, treestate: Treestate) -> Self { Self::Contextual { contextually_verified, @@ -282,7 +286,7 @@ impl FinalizableBlock { } #[cfg(test)] - /// Extract a `Block` from a `FinalizedBlock` variant. + /// Extract a [`Block`] from a [`FinalizableBlock`] variant. pub fn inner_block(&self) -> Arc { match self { FinalizableBlock::Checkpoint { diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 02b4e8625f9..0bb027e0279 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -269,7 +269,7 @@ impl Drop for ReadStateService { // so dropping it should check if we can shut down. if let Some(block_write_task) = self.block_write_task.take() { - if let Ok(block_write_task_handle) = Arc::try_unwrap(block_write_task) { + if let Some(block_write_task_handle) = Arc::into_inner(block_write_task) { // We're the last database user, so we can tell it to shut down (blocking): // - flushes the database to disk, and // - drops the database, which cleans up any database tasks correctly. @@ -1165,15 +1165,11 @@ impl Service for ReadStateService { if let Some(block_write_task) = block_write_task { if block_write_task.is_finished() { - match Arc::try_unwrap(block_write_task) { + if let Some(block_write_task) = Arc::into_inner(block_write_task) { // We are the last state with a reference to this task, so we can propagate any panics - Ok(block_write_task_handle) => { - if let Err(thread_panic) = block_write_task_handle.join() { - std::panic::resume_unwind(thread_panic); - } + if let Err(thread_panic) = block_write_task.join() { + std::panic::resume_unwind(thread_panic); } - // We're not the last state, so we need to put it back - Err(arc_block_write_task) => self.block_write_task = Some(arc_block_write_task), } } else { // It hasn't finished, so we need to put it back diff --git a/zebra-state/src/service/check.rs b/zebra-state/src/service/check.rs index 6e6313f85df..bd8dd8b8648 100644 --- a/zebra-state/src/service/check.rs +++ b/zebra-state/src/service/check.rs @@ -80,19 +80,25 @@ where .expect("valid blocks have a coinbase height"); check::height_one_more_than_parent_height(parent_height, semantically_verified.height)?; - // skip this check during tests if we don't have enough blocks in the chain - #[cfg(test)] if relevant_chain.len() < POW_ADJUSTMENT_BLOCK_SPAN { + // skip this check during tests if we don't have enough blocks in the chain + // process_queued also checks the chain length, so we can skip this assertion during testing + // (tests that want to check this code should use the correct number of blocks) + // + // TODO: accept a NotReadyToBeCommitted error in those tests instead + #[cfg(test)] return Ok(()); + + // In production, blocks without enough context are invalid. + // + // The BlockVerifierRouter makes sure that the first 1 million blocks (or more) are + // checkpoint verified. The state queues and block write task make sure that blocks are + // committed in strict height order. But this function is only called on semantically + // verified blocks, so there will be at least 1 million blocks in the state when it is + // called. So this error should never happen. + #[cfg(not(test))] + return Err(ValidateContextError::NotReadyToBeCommitted); } - // process_queued also checks the chain length, so we can skip this assertion during testing - // (tests that want to check this code should use the correct number of blocks) - assert_eq!( - relevant_chain.len(), - POW_ADJUSTMENT_BLOCK_SPAN, - "state must contain enough blocks to do proof of work contextual validation, \ - and validation must receive the exact number of required blocks" - ); let relevant_data = relevant_chain.iter().map(|block| { ( diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs index fccae9b094d..74ae8dd54ba 100644 --- a/zebra-state/src/service/finalized_state.rs +++ b/zebra-state/src/service/finalized_state.rs @@ -232,9 +232,11 @@ impl FinalizedState { FinalizableBlock::Checkpoint { checkpoint_verified, } => { - // Checkpoint-verified blocks don't have an associated treestate, so we get the parent - // block treestate from the database and update it for the block being committed. - // We check that the treestate is the parent block in an assert later in this function. + // Checkpoint-verified blocks don't have an associated treestate, so we retrieve the + // treestate of the finalized tip from the database and update it for the block + // being committed, assuming the retrieved treestate is the parent block's + // treestate. Later on, this function proves this assumption by asserting that the + // finalized tip is the parent block of the block being committed. let block = checkpoint_verified.block.clone(); let mut history_tree = self.db.history_tree(); diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index f9c4b174e7e..1fa2b29e347 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -399,6 +399,8 @@ impl NonFinalizedState { // Pushing a block onto a Chain can launch additional parallel batches. // TODO: should we pass _scope into Chain::push()? scope.spawn_fifo(|_scope| { + // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: + // https://github.com/rust-lang/rust/issues/93610 let new_chain = Arc::try_unwrap(new_chain) .unwrap_or_else(|shared_chain| (*shared_chain).clone()); chain_push_result = Some(new_chain.push(contextual).map(Arc::new)); diff --git a/zebra-test/Cargo.toml b/zebra-test/Cargo.toml index d6edb5c5aa0..4c712e3beb3 100644 --- a/zebra-test/Cargo.toml +++ b/zebra-test/Cargo.toml @@ -18,7 +18,7 @@ categories = ["command-line-utilities", "cryptography::cryptocurrencies"] hex = "0.4.3" indexmap = "1.9.3" lazy_static = "1.4.0" -insta = "1.29.0" +insta = "1.30.0" proptest = "1.2.0" once_cell = "1.18.0" rand = { version = "0.8.5", package = "rand" } diff --git a/zebra-utils/Cargo.toml b/zebra-utils/Cargo.toml index 98e3b042c0e..98b7fb50d20 100644 --- a/zebra-utils/Cargo.toml +++ b/zebra-utils/Cargo.toml @@ -15,6 +15,10 @@ keywords = ["zebra", "zcash"] # Must be one of categories = ["command-line-utilities", "cryptography::cryptocurrencies"] +# Zebra is only supported on the latest stable Rust version. See the README for details. +# Any Zebra release can break compatibility with older Rust versions. +rust-version = "1.70" + [[bin]] name = "zebra-checkpoints" # this setting is required for Zebra's Docker build caches @@ -77,7 +81,7 @@ zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.26" } zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.26", optional = true } # These crates are needed for the zebra-checkpoints binary -itertools = { version = "0.10.5", optional = true } +itertools = { version = "0.11.0", optional = true } # These crates are needed for the search-issue-refs binary regex = { version = "1.8.4", optional = true } diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index cfeef6df271..56c034e2658 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -19,7 +19,7 @@ edition = "2021" # Zebra is only supported on the latest stable Rust version. See the README for details. # Any Zebra release can break compatibility with older Rust versions. -rust-version = "1.66" +rust-version = "1.70" # Settings that impact runtime behaviour @@ -39,6 +39,9 @@ pre-release-replacements = [ # In release builds, don't compile debug logging code, to improve performance. default = ["release_max_level_info"] +# Default features for official ZF binary release builds +default-release-binaries = ["default", "sentry"] + # Production features that activate extra dependencies, or extra features in dependencies # Experimental mining RPC support @@ -135,7 +138,7 @@ zebra-state = { path = "../zebra-state", version = "1.0.0-beta.26" } zebra-utils = { path = "../zebra-utils", version = "1.0.0-beta.26", optional = true } abscissa_core = "0.7.0" -clap = { version = "4.3.4", features = ["cargo"] } +clap = { version = "4.3.8", features = ["cargo"] } chrono = { version = "0.4.26", default-features = false, features = ["clock", "std"] } humantime-serde = "1.1.1" indexmap = "1.9.3" diff --git a/zebrad/src/application.rs b/zebrad/src/application.rs index d7fb3356dd1..47f324a2e96 100644 --- a/zebrad/src/application.rs +++ b/zebrad/src/application.rs @@ -419,7 +419,10 @@ impl Application for ZebradApp { tracing_config.filter = Some(default_filter.to_owned()); tracing_config.flamegraph = None; } - components.push(Box::new(Tracing::new(tracing_config)?)); + components.push(Box::new(Tracing::new( + config.network.network, + tracing_config, + )?)); // Log git metadata and platform info when zebrad starts up if is_server { diff --git a/zebrad/src/components/tracing.rs b/zebrad/src/components/tracing.rs index d12491f87a7..439c5052e49 100644 --- a/zebrad/src/components/tracing.rs +++ b/zebrad/src/components/tracing.rs @@ -138,6 +138,12 @@ impl Config { self.force_use_color || (self.use_color && atty::is(atty::Stream::Stdout)) } + /// Returns `true` if standard error should use color escapes. + /// Automatically checks if Zebra is running in a terminal. + pub fn use_color_stderr(&self) -> bool { + self.force_use_color || (self.use_color && atty::is(atty::Stream::Stderr)) + } + /// Returns `true` if output that could go to standard output or standard error /// should use color escapes. Automatically checks if Zebra is running in a terminal. pub fn use_color_stdout_and_stderr(&self) -> bool { diff --git a/zebrad/src/components/tracing/component.rs b/zebrad/src/components/tracing/component.rs index 4a3a4560eb3..4498b1a1054 100644 --- a/zebrad/src/components/tracing/component.rs +++ b/zebrad/src/components/tracing/component.rs @@ -15,12 +15,26 @@ use tracing_subscriber::{ util::SubscriberInitExt, EnvFilter, }; +use zebra_chain::parameters::Network; use crate::{application::build_version, components::tracing::Config}; #[cfg(feature = "flamegraph")] use super::flame; +// Art generated with these two images. +// Zebra logo: book/theme/favicon.png +// Heart image: https://commons.wikimedia.org/wiki/File:Heart_coraz%C3%B3n.svg +// (License: CC BY-SA 3.0) +// +// How to render +// +// Convert heart image to PNG (2000px) and run: +// img2txt -W 40 -H 20 -f utf8 -d none Heart_corazón.svg.png > heart.utf8 +// img2txt -W 40 -H 20 -f utf8 -d none favicon.png > logo.utf8 +// paste favicon.utf8 heart.utf8 > zebra.utf8 +static ZEBRA_ART: [u8; include_bytes!("zebra.utf8").len()] = *include_bytes!("zebra.utf8"); + /// A type-erased boxed writer that can be sent between threads safely. pub type BoxWrite = Box; @@ -52,15 +66,36 @@ pub struct Tracing { impl Tracing { /// Try to create a new [`Tracing`] component with the given `filter`. - #[allow(clippy::print_stdout, clippy::print_stderr)] - pub fn new(config: Config) -> Result { + #[allow(clippy::print_stdout, clippy::print_stderr, clippy::unwrap_in_result)] + pub fn new(network: Network, config: Config) -> Result { // Only use color if tracing output is being sent to a terminal or if it was explicitly // forced to. let use_color = config.use_color_stdout(); + let use_color_stderr = config.use_color_stderr(); let filter = config.filter.unwrap_or_default(); let flame_root = &config.flamegraph; + // If it's a terminal and color escaping is enabled: clear screen and + // print Zebra logo (here `use_color` is being interpreted as + // "use escape codes") + if use_color_stderr { + // Clear screen + eprint!("\x1B[2J"); + eprintln!( + "{}", + std::str::from_utf8(&ZEBRA_ART) + .expect("should always work on a UTF-8 encoded constant") + ); + } + + eprintln!( + "Thank you for running a {} zebrad {} node!", + network.lowercase_name(), + build_version() + ); + eprintln!("You're helping to strengthen the network and contributing to a social good :)"); + let writer = if let Some(log_file) = config.log_file.as_ref() { println!("running zebra"); @@ -263,10 +298,6 @@ impl Tracing { howudoin::init(terminal_consumer); info!("activated progress bar"); - - if config.log_file.is_some() { - eprintln!("waiting for initial progress reports..."); - } } Ok(Self { diff --git a/zebrad/src/components/tracing/zebra.utf8 b/zebrad/src/components/tracing/zebra.utf8 new file mode 100644 index 00000000000..bf620ddea8d --- /dev/null +++ b/zebrad/src/components/tracing/zebra.utf8 @@ -0,0 +1,20 @@ + X@8:::::::@X + @::X:;SX;%8S8@@:X::8 + @:;@8@.S8 ;;t;. XX.;@@;:@ 8; %X X% ;8 + 8:@88 .%@.S@.XS: 888:8 X :: .: X + :;SX XS8;::@X@8::;8t8: 8;;:@ % SS % + @::@ 88:::8 8::88::::::X@ 8::8 S X + @:XS8 ;::::::8.88%8@S88::::8: 8S8:8 X S + ::;8 t8S8::::@St 88SX @SS@8:8S;8S @@:: 8 8 +8:XX %%@@::X% 8: X@ t:8t X ..%t8 ;8:8   +::8t: tX:X8tS 8: :@ ;.8@.X tt888: @8:: @ X +::8S: @:8Xtt% 8: S 8 8S.   .8;::. 88:: 8 8 +::X8 S@8 8:8.@8@8Xt S  .%8;S %8:: : : + ::@8 S .  @:::::::X;@ 8.88S8 @8;:X :: :; + 8:X8X .@:X88:::::::::888 @;:X %88:: t t + @::S 888@8:::::::::8X; 8 8::8 8 8 + X:;S@ tXX8;::::::::S::@8 8;S:8 X. .X + ::8S@ :S88XX@8X: @XX:: :: :: + 8:;%X8.:S t; X8%S:8X tt + X8::8SS8SX8@@X@8;8S:8X + X@8:::::::@XS