Merge branch 'main' into issue6681

.github/PULL_REQUEST_TEMPLATE/release-checklist.md

@@ -23,7 +23,8 @@ Sometimes `dependabot` misses some dependency updates, or we accidentally turned
 
 Here's how we make sure we got everything:
 - [ ] Run `cargo update` on the latest `main` branch, and keep the output
-- [ ] If needed, update [deny.toml](https://github.com/ZcashFoundation/zebra/blob/main/book/src/dev/continuous-integration.md#fixing-duplicate-dependencies-in-check-denytoml-bans)
+- [ ] If needed, [add duplicate dependency exceptions to deny.toml](https://github.com/ZcashFoundation/zebra/blob/main/book/src/dev/continuous-integration.md#fixing-duplicate-dependencies-in-check-denytoml-bans)
+- [ ] If needed, remove resolved duplicate dependencies from `deny.toml`
 - [ ] Open a separate PR with the changes
 - [ ] Add the output of `cargo update` to that PR as a comment
 
@@ -157,10 +158,7 @@ The end of support height is calculated from the current blockchain height:
       and put the output in a comment on the PR.
 
 ## Publish Docker Images
-- [ ] Wait until [the Docker images have been published](https://github.com/ZcashFoundation/zebra/actions/workflows/release-binaries.yml)
-- [ ] Test the Docker image using `docker run --tty --interactive zfnd/zebra:v1.0.0`,
-      and put the output in a comment on the PR.
-      (You can use [gcloud cloud shell](https://console.cloud.google.com/home/dashboard?cloudshell=true))
+- [ ] Wait for the [the Docker images to be published successfully](https://github.com/ZcashFoundation/zebra/actions/workflows/release-binaries.yml).
 - [ ] Un-freeze the [`batched` queue](https://dashboard.mergify.com/github/ZcashFoundation/repo/zebra/queues) using Mergify.
 
 ## Release Failures

.github/workflows/build-docker-image.yml

@@ -38,7 +38,7 @@ on:
       # https://github.com/ZcashFoundation/zebra/blob/main/docker/Dockerfile#L83
       features:
         required: false
-        default: "sentry"
+        default: "default-release-binaries"
         type: string
       test_features:
         required: false

.github/workflows/continous-delivery.patch.yml

@@ -0,0 +1,36 @@
+name: CD
+
+on:
+  # Only patch the Docker image test jobs
+  pull_request:
+    paths-ignore:
+      # code and tests
+      - '**/*.rs'
+      # hard-coded checkpoints and proptest regressions
+      - '**/*.txt'
+      # dependencies
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
+      # workflow definitions
+      - 'docker/**'
+      - '.dockerignore'
+      - '.github/workflows/continous-delivery.yml'
+      - '.github/workflows/find-cached-disks.yml'
+
+
+jobs:
+  # Also patched by continous-integration-docker.patch.yml, which has a different paths-ignore
+  build:
+    name: Build CI Docker / Build images
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'
+
+  test-configuration-file:
+    name: Test Zebra CD Docker config file
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'

.github/workflows/continous-delivery.yml

@@ -6,9 +6,12 @@ name: CD
 #
 # Since the different event types each use a different Managed Instance Group or instance,
 # we can run different event types concurrently.
+#
+# For pull requests, we only run the tests from this workflow, and don't do any deployments.
+# So an in-progress pull request gets cancelled, just like other tests.
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
-  cancel-in-progress: false
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 on:
   workflow_dispatch:
@@ -25,14 +28,52 @@ on:
         required: false
         type: boolean
         default: false
+
   # Temporarily disabled to reduce network load, see #6894.
   #push:
   #  branches:
   #    - main
+  #  paths:
+  #    # code and tests
+  #    - '**/*.rs'
+  #    # hard-coded checkpoints and proptest regressions
+  #    - '**/*.txt'
+  #    # dependencies
+  #    - '**/Cargo.toml'
+  #    - '**/Cargo.lock'
+  #    # configuration files
+  #    - '.cargo/config.toml'
+  #    - '**/clippy.toml'
+  #    # workflow definitions
+  #    - 'docker/**'
+  #    - '.dockerignore'
+  #    - '.github/workflows/continous-delivery.yml'
+  #    - '.github/workflows/build-docker-image.yml'
+
+  # Only runs the Docker image tests, doesn't deploy any instances
+  pull_request:
+    paths:
+      # code and tests
+      - '**/*.rs'
+      # hard-coded checkpoints and proptest regressions
+      - '**/*.txt'
+      # dependencies
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
+      # workflow definitions
+      - 'docker/**'
+      - '.dockerignore'
+      - '.github/workflows/continous-delivery.yml'
+      - '.github/workflows/find-cached-disks.yml'
+
   release:
     types:
       - published
 
+
 jobs:
   # If a release was made we want to extract the first part of the semver from the
   # tag_name
@@ -82,7 +123,7 @@ jobs:
   # Test that Zebra works using the default config with the latest Zebra version,
   # and test reconfiguring the docker image for testnet.
   test-configuration-file:
-    name: Test Zebra default Docker config file
+    name: Test Zebra CD Docker config file
     timeout-minutes: 15
     runs-on: ubuntu-latest
     needs: build

.github/workflows/continous-integration-docker.patch.yml

@@ -19,8 +19,10 @@ on:
       - '**/clippy.toml'
       # workflow definitions
       - 'docker/**'
+      - '.dockerignore'
       - '.github/workflows/continous-integration-docker.yml'
       - '.github/workflows/deploy-gcp-tests.yml'
+      - '.github/workflows/find-cached-disks.yml'
       - '.github/workflows/build-docker-image.yml'
 
 jobs:

.github/workflows/continous-integration-docker.yml

@@ -80,10 +80,11 @@ on:
       - '**/clippy.toml'
       # workflow definitions
       - 'docker/**'
+      - '.dockerignore'
       - '.github/workflows/continous-integration-docker.yml'
       - '.github/workflows/deploy-gcp-tests.yml'
-      - '.github/workflows/build-docker-image.yml'
       - '.github/workflows/find-cached-disks.yml'
+      - '.github/workflows/build-docker-image.yml'
 
 jobs:
   # to also run a job on Mergify head branches,

.github/workflows/continous-integration-os.yml

@@ -261,8 +261,8 @@ jobs:
         # We don't need to check `--no-default-features` here, because (except in very rare cases):
         # - disabling features isn't going to add duplicate dependencies
         # - disabling features isn't going to add more crate sources
-        features: ['', '--all-features']
-      # We always want to run the --all-features job, because it gives accurate "skip tree root was not found" warnings
+        features: ['', '--features default-release-binaries', '--all-features']
+      # Always run the --all-features job, to get accurate "skip tree root was not found" warnings
       fail-fast: false
 
     # Prevent sudden announcement of a new advisory from failing ci:
@@ -274,12 +274,14 @@ jobs:
           persist-credentials: false
       - uses: r7kamura/rust-problem-matchers@v1.3.0
 
-      # The --all-features job is the only job that gives accurate "skip tree root was not found" warnings.
-      # In other jobs, we expect some of these warnings, due to disabled features.
       - name: Check ${{ matrix.checks }} with features ${{ matrix.features }}
         uses: EmbarkStudios/cargo-deny-action@v1
         with:
-          command: check ${{ matrix.checks }}
+          # --all-features spuriously activates openssl, but we want to ban that dependency in
+          # all of zebrad's production features for security reasons. But the --all-features job is
+          # the only job that gives accurate "skip tree root was not found" warnings.
+          # In other jobs, we expect some of these warnings, due to disabled features.
+          command: check ${{ matrix.checks }} ${{ matrix.features == '--all-features' && '--allow banned' || '--allow unmatched-skip-root' }}
           arguments: --workspace ${{ matrix.features }}
 
   unused-deps:

.github/workflows/dockerhub-description.yml

@@ -22,7 +22,7 @@ jobs:
           persist-credentials: false
 
       - name: Docker Hub Description
-        uses: peter-evans/dockerhub-description@v3.4.1
+        uses: peter-evans/dockerhub-description@v3.4.2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}

.github/workflows/lint.yml

@@ -37,7 +37,7 @@ jobs:
 
       - name: Rust files
         id: changed-files-rust
-        uses: tj-actions/changed-files@v36.4.0
+        uses: tj-actions/changed-files@v37.0.3
         with:
           files: |
             **/*.rs
@@ -49,7 +49,7 @@ jobs:
 
       - name: Workflow files
         id: changed-files-workflows
-        uses: tj-actions/changed-files@v36.4.0
+        uses: tj-actions/changed-files@v37.0.3
         with:
           files: |
             .github/workflows/*.yml

.github/workflows/release-binaries.yml

@@ -44,7 +44,7 @@ jobs:
       tag_suffix: .experimental
       network: Testnet
       rpc_port: '18232'
-      features: "sentry getblocktemplate-rpcs"
+      features: "default-release-binaries getblocktemplate-rpcs"
       test_features: ""
       rust_backtrace: '1'
       zebra_skip_ipv6_tests: '1'

Cargo.lock

@@ -12,7 +12,7 @@ dependencies = [
  "arc-swap",
  "backtrace",
  "canonical-path",
- "clap 4.3.4",
+ "clap 4.3.8",
  "color-eyre",
  "fs-err",
  "once_cell",
@@ -773,9 +773,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.3.4"
+version = "4.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80672091db20273a15cf9fdd4e47ed43b5091ec9841bf4c6145c9dfbbcae09ed"
+checksum = "d9394150f5b4273a1763355bd1c2ec54cc5a2593f790587bcd6b2c947cfa9211"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -784,9 +784,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.3.4"
+version = "4.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1458a1df40e1e2afebb7ab60ce55c1fa8f431146205aa5f4887e0b111c27636"
+checksum = "9a78fbdd3cc2914ddf37ba444114bc7765bbdcb55ec9cbe6fa054f0137400717"
 dependencies = [
  "anstream",
  "anstyle",
@@ -956,10 +956,10 @@ dependencies = [
  "anes",
  "cast",
  "ciborium",
- "clap 4.3.4",
+ "clap 4.3.8",
  "criterion-plot",
  "is-terminal",
- "itertools",
+ "itertools 0.10.5",
  "num-traits",
  "once_cell",
  "oorandom",
@@ -980,7 +980,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
 dependencies = [
  "cast",
- "itertools",
+ "itertools 0.10.5",
 ]
 
 [[package]]
@@ -2101,9 +2101,9 @@ dependencies = [
 
 [[package]]
 name = "insta"
-version = "1.29.0"
+version = "1.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a28d25139df397cbca21408bb742cf6837e04cdbebf1b07b760caf971d6a972"
+checksum = "28491f7753051e5704d4d0ae7860d45fae3238d7d235bc4289dcd45c48d3cec3"
 dependencies = [
  "console",
  "lazy_static",
@@ -2163,6 +2163,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.6"
@@ -3296,7 +3305,7 @@ checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270"
 dependencies = [
  "bytes",
  "heck 0.4.1",
- "itertools",
+ "itertools 0.10.5",
  "lazy_static",
  "log",
  "multimap",
@@ -3317,7 +3326,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
 dependencies = [
  "anyhow",
- "itertools",
+ "itertools 0.10.5",
  "proc-macro2 1.0.60",
  "quote 1.0.28",
  "syn 1.0.109",
@@ -5669,7 +5678,7 @@ dependencies = [
  "hex",
  "humantime",
  "incrementalmerkletree",
- "itertools",
+ "itertools 0.11.0",
  "jubjub",
  "lazy_static",
  "num-integer",
@@ -5765,7 +5774,7 @@ dependencies = [
  "howudoin",
  "humantime-serde",
  "indexmap",
- "itertools",
+ "itertools 0.11.0",
  "lazy_static",
  "metrics 0.21.0",
  "num-integer",
@@ -5864,7 +5873,7 @@ dependencies = [
  "howudoin",
  "indexmap",
  "insta",
- "itertools",
+ "itertools 0.11.0",
  "jubjub",
  "lazy_static",
  "metrics 0.21.0",
@@ -5923,7 +5932,7 @@ version = "1.0.0-beta.26"
 dependencies = [
  "color-eyre",
  "hex",
- "itertools",
+ "itertools 0.11.0",
  "regex",
  "reqwest",
  "serde_json",
@@ -5945,7 +5954,7 @@ dependencies = [
  "abscissa_core",
  "atty",
  "chrono",
- "clap 4.3.4",
+ "clap 4.3.8",
  "color-eyre",
  "console-subscriber",
  "dirs",