merge-queue: embarking main (744aca9), #3688 and #3657 together

.github/workflows/cd.yml

@@ -67,7 +67,7 @@ jobs:
       with:
         target: runtime
         context: .
-        file: ./docker/Dockerfile.build
+        file: ./docker/Dockerfile
         tags: |
           ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}:latest
           ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}:${{ env.GITHUB_SHA_SHORT }}

.github/workflows/ci.patch.yml

@@ -8,7 +8,6 @@ on:
       - '**/Cargo.toml'
       - '**/Cargo.lock'
       - '**/deny.toml'
-      - 'docker/**'
       - '.github/workflows/ci.yml'
 
 jobs:

.github/workflows/test.yml

@@ -72,7 +72,7 @@ jobs:
         with:
           target: tester
           context: .
-          file: ./docker/Dockerfile.build
+          file: ./docker/Dockerfile
           tags: |
             ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
             ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
@@ -146,6 +146,26 @@ jobs:
           docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
           docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored
 
+  test-lightwalletd-integration:
+    name: Test integration with lightwalletd
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ github.event.inputs.regenerate-disks != 'true' }}
+    steps:
+      - uses: actions/checkout@v2.4.0
+        with:
+          persist-credentials: false
+
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Run tests with included lightwalletd binary
+        run: |
+          docker pull ${{ env.GAR_BASE }}/zebrad-test:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }}
+          docker run -e ZEBRA_SKIP_IPV6_TESTS -e ZEBRA_TEST_LIGHTWALLETD --name zebrad-tests -t ${{ env.GAR_BASE }}/zebrad-test:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance -- lightwalletd_integration 
+        env:
+          ZEBRA_TEST_LIGHTWALLETD: '1'
+
   regenerate-stateful-disks:
     name: Regenerate stateful disks
     runs-on: ubuntu-latest

.github/workflows/zcash-lightwalletd.yml

@@ -0,0 +1,73 @@
+name: zcash-lightwalletd
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'zebra-rpc/**'
+      - 'zebrad/tests/acceptance.rs'
+      - 'zebrad/src/config.rs'
+      - 'zebrad/src/commands/start.rs'
+      - 'docker/zcash-lightwalletd/Dockerfile'
+      - '.github/workflows/zcash-lightwalletd.yml'
+
+env:
+  PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+  GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra
+  GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }}
+  IMAGE_NAME: lightwalletd
+
+jobs:
+  build:
+    name: Build images
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2.4.0
+      with:
+        repository: adityapk00/lightwalletd
+        ref: 'master'
+        persist-credentials: false
+
+    - uses: actions/checkout@v2.4.0
+      with:
+        path: zebra
+        persist-credentials: false
+
+    - name: Inject slug/short variables
+      uses: rlespinasse/github-slug-action@v4
+
+    # Setup gcloud CLI
+    - name: Authenticate to Google Cloud
+      id: auth
+      uses: google-github-actions/auth@v0.6.0
+      with:
+        credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+    # Setup Docker Buildx to allow use of docker cache layers from GH
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to Google Artifact Registry
+      uses: docker/login-action@v1.12.0
+      with:
+        registry: us-docker.pkg.dev
+        username: _json_key
+        password: ${{ secrets.GOOGLE_CREDENTIALS }}
+
+    # Build and push image to Google Artifact Registry
+    - name: Build & push
+      id: docker_build
+      uses: docker/build-push-action@v2.9.0
+      with:
+        target: builder
+        context: .
+        file: ./zebra/docker/zcash-lightwalletd/Dockerfile
+        tags: |
+          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
+          ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}
+        push: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max

.github/workflows/zcash-params.yml

@@ -9,6 +9,7 @@ on:
       - 'zebra-consensus/src/primitives/groth16/params.rs'
       - 'zebra-consensus/src/chain.rs'
       - 'zebrad/src/commands/start.rs'
+      - 'docker/zcash-params/Dockerfile'
       - '.github/workflows/zcash-params.yml'
 
 env:
@@ -57,7 +58,7 @@ jobs:
       with:
         target: builder
         context: .
-        file: ./docker/Dockerfile.params
+        file: ./docker/zcash-params/Dockerfile
         tags: |
           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest
           ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }}

docker/Dockerfile.build → docker/Dockerfile

@@ -56,6 +56,7 @@ FROM builder AS tester
 # Pre-download Zcash Sprout and Sapling parameters
 # TODO: do not hardcode the user /root/ even though is a safe assumption
 COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/zcash-params /root/.zcash-params /root/.zcash-params
+COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /lightwalletd /usr/local/bin
 
 # Skip IPv6 tests by default, as some CI environment don't have IPv6 available
 ARG ZEBRA_SKIP_IPV6_TESTS=1

docker/zcash-lightwalletd/Dockerfile

@@ -0,0 +1,61 @@
+ARG ZCASHD_CONF_PATH=/etc/zcash.conf
+ARG LWD_GRPC_PORT=9067
+ARG LWD_HTTP_PORT=9068
+
+##
+## Build
+##
+FROM golang:1.17 AS build
+
+# Create and change to the app directory.
+WORKDIR /app
+
+# Retrieve application dependencies.
+# This allows the container build to reuse cached dependencies.
+# Expecting to copy go.mod and if present go.sum.
+COPY go.mod ./
+COPY go.sum ./
+
+# Do not use `go get` as it updates the requirements listed in your go.mod file.
+# `go mod download` does not add new requirements or update existing requirements.
+RUN go mod download
+
+# Copy local code to the container image.
+COPY . ./
+
+# Build and install the binary.
+RUN go build -v -o /lightwalletd
+
+ARG ZCASHD_CONF_PATH
+
+RUN set -ex; \
+  { \
+    echo "rpcuser=zcashrpc"; \
+    echo "rpcpassword=`head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13 ; echo ''`" \
+    echo "rpcbind=127.0.0.1"; \
+    echo "rpcport=8232"; \
+  } > "${ZCASHD_CONF_PATH}"
+
+ENTRYPOINT ["/lightwalletd"]
+CMD ["--no-tls-very-insecure", "--grpc-bind-addr=0.0.0.0:9067",  "--http-bind-addr=0.0.0.0:9068", "--log-file=/dev/stdout", "--log-level=7"]
+
+##
+## Deploy
+##
+FROM debian:bullseye-slim as runtime
+
+ARG ZCASHD_CONF_PATH
+# Maintain backward compatibility with mainstream repo using this ARGs in docker-compose
+ARG LWD_GRPC_PORT
+ARG LWD_HTTP_PORT
+
+WORKDIR /
+
+COPY --from=build /lightwalletd /usr/local/bin
+COPY --from=build $ZCASHD_CONF_PATH ./
+
+EXPOSE 9067
+EXPOSE 9068
+
+ENTRYPOINT ["lightwalletd"]
+CMD ["--no-tls-very-insecure", "--grpc-bind-addr=0.0.0.0:9067",  "--http-bind-addr=0.0.0.0:9068", "--log-file=/dev/stdout", "--log-level=7"]

zebrad/tests/acceptance.rs

@@ -1247,29 +1247,23 @@ fn cached_mandatory_checkpoint_test_config() -> Result<ZebradConfig> {
 ///
 /// If `check_legacy_chain` is true, make sure the logs contain the legacy chain check.
 ///
-/// Callers can supply an extra `test_child_predicate`, which is called on
-/// the `TestChild` between the startup checks, and the final
-/// `STOP_AT_HEIGHT_REGEX` check.
-///
-/// The `TestChild` is spawned with a timeout, so the predicate should use
-/// `expect_stdout_line_matches` or `expect_stderr_line_matches`.
+/// The test passes when `zebrad` logs the `stop_regex`.
+/// Typically this is `STOP_AT_HEIGHT_REGEX`,
+/// with an extra check for checkpoint or full validation.
 ///
 /// This test ignores the `ZEBRA_SKIP_NETWORK_TESTS` env var.
 ///
 /// # Test Status
 ///
 /// Returns an error if the child exits or the fixed timeout elapses
 /// before `STOP_AT_HEIGHT_REGEX` is found.
-fn create_cached_database_height<P>(
+fn create_cached_database_height(
     network: Network,
     height: Height,
     debug_skip_parameter_preload: bool,
     checkpoint_sync: bool,
-    test_child_predicate: impl Into<Option<P>>,
-) -> Result<()>
-where
-    P: FnOnce(&mut TestChild<PathBuf>) -> Result<()>,
-{
+    stop_regex: &str,
+) -> Result<()> {
     println!("Creating cached database");
     // 16 hours
     let timeout = Duration::from_secs(60 * 60 * 16);
@@ -1295,11 +1289,7 @@ where
     child.expect_stdout_line_matches("starting legacy chain check")?;
     child.expect_stdout_line_matches("no legacy chain found")?;
 
-    if let Some(test_child_predicate) = test_child_predicate.into() {
-        test_child_predicate(&mut child)?;
-    }
-
-    child.expect_stdout_line_matches(STOP_AT_HEIGHT_REGEX)?;
+    child.expect_stdout_line_matches(stop_regex)?;
 
     child.kill()?;
 
@@ -1308,38 +1298,31 @@ where
 
 fn create_cached_database(network: Network) -> Result<()> {
     let height = network.mandatory_checkpoint_height();
+    let checkpoint_stop_regex = format!("{}.*CommitFinalized request", STOP_AT_HEIGHT_REGEX);
+
     create_cached_database_height(
         network,
         height,
         true,
         // Use checkpoints to increase sync performance while caching the database
         true,
-        |test_child: &mut TestChild<PathBuf>| {
-            // make sure pre-cached databases finish before the mandatory checkpoint
-            //
-            // TODO: this check passes even if we reach the mandatory checkpoint,
-            //       because we sync finalized state, then non-finalized state.
-            //       Instead, fail if we see "best non-finalized chain root" in the logs.
-            test_child.expect_stdout_line_matches("CommitFinalized request")?;
-            Ok(())
-        },
+        // Check that we're still using checkpoints when we finish the cached sync
+        &checkpoint_stop_regex,
     )
 }
 
 fn sync_past_mandatory_checkpoint(network: Network) -> Result<()> {
     let height = network.mandatory_checkpoint_height() + 1200;
+    let full_validation_stop_regex =
+        format!("{}.*best non-finalized chain root", STOP_AT_HEIGHT_REGEX);
+
     create_cached_database_height(
         network,
         height.unwrap(),
         false,
         // Test full validation by turning checkpoints off
         false,
-        |test_child: &mut TestChild<PathBuf>| {
-            // make sure cached database tests finish after the mandatory checkpoint,
-            // using the non-finalized state (the checkpoint_sync config must be false)
-            test_child.expect_stdout_line_matches("best non-finalized chain root")?;
-            Ok(())
-        },
+        &full_validation_stop_regex,
     )
 }