diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 77bae1bef9f..85c8f867741 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -67,7 +67,7 @@ jobs: with: target: runtime context: . - file: ./docker/Dockerfile.build + file: ./docker/Dockerfile tags: | ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}:latest ${{ env.GAR_BASE }}/${{ env.GITHUB_HEAD_REF_SLUG_URL || env.GITHUB_REF_SLUG_URL }}:${{ env.GITHUB_SHA_SHORT }} diff --git a/.github/workflows/ci.patch.yml b/.github/workflows/ci.patch.yml index 6257040e116..d5525509f7f 100644 --- a/.github/workflows/ci.patch.yml +++ b/.github/workflows/ci.patch.yml @@ -8,7 +8,6 @@ on: - '**/Cargo.toml' - '**/Cargo.lock' - '**/deny.toml' - - 'docker/**' - '.github/workflows/ci.yml' jobs: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7c6b3672ed..53db50a2e95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,7 +72,7 @@ jobs: with: target: tester context: . - file: ./docker/Dockerfile.build + file: ./docker/Dockerfile tags: | ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} @@ -146,6 +146,26 @@ jobs: docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} docker run -e ZEBRA_SKIP_IPV6_TESTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance sync_large_checkpoints_ -- --ignored + test-lightwalletd-integration: + name: Test integration with lightwalletd + runs-on: ubuntu-latest + needs: build + if: ${{ github.event.inputs.regenerate-disks != 'true' }} + steps: + - uses: actions/checkout@v2.4.0 + with: + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Run tests with included lightwalletd binary + run: | + docker pull ${{ env.GAR_BASE }}/zebrad-test:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} + docker run -e ZEBRA_SKIP_IPV6_TESTS -e ZEBRA_TEST_LIGHTWALLETD --name zebrad-tests -t ${{ env.GAR_BASE }}/zebrad-test:${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT || env.GITHUB_SHA_SHORT }} cargo test --locked --release --features enable-sentry --test acceptance -- lightwalletd_integration + env: + ZEBRA_TEST_LIGHTWALLETD: '1' + regenerate-stateful-disks: name: Regenerate stateful disks runs-on: ubuntu-latest diff --git a/.github/workflows/zcash-lightwalletd.yml b/.github/workflows/zcash-lightwalletd.yml new file mode 100644 index 00000000000..bb76ee76a88 --- /dev/null +++ b/.github/workflows/zcash-lightwalletd.yml @@ -0,0 +1,73 @@ +name: zcash-lightwalletd + +on: + workflow_dispatch: + push: + branches: + - 'main' + paths: + - 'zebra-rpc/**' + - 'zebrad/tests/acceptance.rs' + - 'zebrad/src/config.rs' + - 'zebrad/src/commands/start.rs' + - 'docker/zcash-lightwalletd/Dockerfile' + - '.github/workflows/zcash-lightwalletd.yml' + +env: + PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + GAR_BASE: us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/zebra + GCR_BASE: gcr.io/${{ secrets.GCP_PROJECT_ID }} + IMAGE_NAME: lightwalletd + +jobs: + build: + name: Build images + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2.4.0 + with: + repository: adityapk00/lightwalletd + ref: 'master' + persist-credentials: false + + - uses: actions/checkout@v2.4.0 + with: + path: zebra + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.6.0 + with: + credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} + + # Setup Docker Buildx to allow use of docker cache layers from GH + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Google Artifact Registry + uses: docker/login-action@v1.12.0 + with: + registry: us-docker.pkg.dev + username: _json_key + password: ${{ secrets.GOOGLE_CREDENTIALS }} + + # Build and push image to Google Artifact Registry + - name: Build & push + id: docker_build + uses: docker/build-push-action@v2.9.0 + with: + target: builder + context: . + file: ./zebra/docker/zcash-lightwalletd/Dockerfile + tags: | + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} + push: true + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/zcash-params.yml b/.github/workflows/zcash-params.yml index 475bda82396..129cb64a289 100644 --- a/.github/workflows/zcash-params.yml +++ b/.github/workflows/zcash-params.yml @@ -9,6 +9,7 @@ on: - 'zebra-consensus/src/primitives/groth16/params.rs' - 'zebra-consensus/src/chain.rs' - 'zebrad/src/commands/start.rs' + - 'docker/zcash-params/Dockerfile' - '.github/workflows/zcash-params.yml' env: @@ -57,7 +58,7 @@ jobs: with: target: builder context: . - file: ./docker/Dockerfile.params + file: ./docker/zcash-params/Dockerfile tags: | ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:latest ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_SHA_SHORT }} diff --git a/docker/Dockerfile.build b/docker/Dockerfile similarity index 97% rename from docker/Dockerfile.build rename to docker/Dockerfile index b599474afe5..6eb413b64df 100644 --- a/docker/Dockerfile.build +++ b/docker/Dockerfile @@ -56,6 +56,7 @@ FROM builder AS tester # Pre-download Zcash Sprout and Sapling parameters # TODO: do not hardcode the user /root/ even though is a safe assumption COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/zcash-params /root/.zcash-params /root/.zcash-params +COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /lightwalletd /usr/local/bin # Skip IPv6 tests by default, as some CI environment don't have IPv6 available ARG ZEBRA_SKIP_IPV6_TESTS=1 diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test deleted file mode 100644 index b479050ba42..00000000000 --- a/docker/Dockerfile.test +++ /dev/null @@ -1,30 +0,0 @@ -FROM rust:buster - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - make cmake g++ gcc llvm libclang-dev clang - -RUN mkdir /zebra -WORKDIR /zebra - -ARG SHORT_SHA -ENV SHORT_SHA $SHORT_SHA - -ENV RUST_BACKTRACE full -ENV CARGO_HOME /zebra/.cargo/ -# Optimize builds. In particular, regenerate-stateful-test-disks.yml was reaching the -# GitHub Actions time limit (6 hours), so we needed to make it faster. -ENV RUSTFLAGS -O - -RUN rustc -V; cargo -V; rustup -V - -EXPOSE 8233 18233 - -COPY . . - -# Pre-download Zcash Sprout and Sapling parameters -RUN cargo run --verbose --bin zebrad download - -RUN cargo test --all --no-run - -CMD cargo test --workspace --no-fail-fast -- -Zunstable-options --include-ignored diff --git a/docker/zcash-lightwalletd/Dockerfile b/docker/zcash-lightwalletd/Dockerfile new file mode 100644 index 00000000000..9dbe46f1f34 --- /dev/null +++ b/docker/zcash-lightwalletd/Dockerfile @@ -0,0 +1,61 @@ +ARG ZCASHD_CONF_PATH=/etc/zcash.conf +ARG LWD_GRPC_PORT=9067 +ARG LWD_HTTP_PORT=9068 + +## +## Build +## +FROM golang:1.17 AS build + +# Create and change to the app directory. +WORKDIR /app + +# Retrieve application dependencies. +# This allows the container build to reuse cached dependencies. +# Expecting to copy go.mod and if present go.sum. +COPY go.mod ./ +COPY go.sum ./ + +# Do not use `go get` as it updates the requirements listed in your go.mod file. +# `go mod download` does not add new requirements or update existing requirements. +RUN go mod download + +# Copy local code to the container image. +COPY . ./ + +# Build and install the binary. +RUN go build -v -o /lightwalletd + +ARG ZCASHD_CONF_PATH + +RUN set -ex; \ + { \ + echo "rpcuser=zcashrpc"; \ + echo "rpcpassword=`head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13 ; echo ''`" \ + echo "rpcbind=127.0.0.1"; \ + echo "rpcport=8232"; \ + } > "${ZCASHD_CONF_PATH}" + +ENTRYPOINT ["/lightwalletd"] +CMD ["--no-tls-very-insecure", "--grpc-bind-addr=0.0.0.0:9067", "--http-bind-addr=0.0.0.0:9068", "--log-file=/dev/stdout", "--log-level=7"] + +## +## Deploy +## +FROM debian:bullseye-slim as runtime + +ARG ZCASHD_CONF_PATH +# Maintain backward compatibility with mainstream repo using this ARGs in docker-compose +ARG LWD_GRPC_PORT +ARG LWD_HTTP_PORT + +WORKDIR / + +COPY --from=build /lightwalletd /usr/local/bin +COPY --from=build $ZCASHD_CONF_PATH ./ + +EXPOSE 9067 +EXPOSE 9068 + +ENTRYPOINT ["lightwalletd"] +CMD ["--no-tls-very-insecure", "--grpc-bind-addr=0.0.0.0:9067", "--http-bind-addr=0.0.0.0:9068", "--log-file=/dev/stdout", "--log-level=7"] diff --git a/docker/Dockerfile.params b/docker/zcash-params/Dockerfile similarity index 100% rename from docker/Dockerfile.params rename to docker/zcash-params/Dockerfile diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 7c857171889..8df54ac58fd 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -1247,12 +1247,9 @@ fn cached_mandatory_checkpoint_test_config() -> Result { /// /// If `check_legacy_chain` is true, make sure the logs contain the legacy chain check. /// -/// Callers can supply an extra `test_child_predicate`, which is called on -/// the `TestChild` between the startup checks, and the final -/// `STOP_AT_HEIGHT_REGEX` check. -/// -/// The `TestChild` is spawned with a timeout, so the predicate should use -/// `expect_stdout_line_matches` or `expect_stderr_line_matches`. +/// The test passes when `zebrad` logs the `stop_regex`. +/// Typically this is `STOP_AT_HEIGHT_REGEX`, +/// with an extra check for checkpoint or full validation. /// /// This test ignores the `ZEBRA_SKIP_NETWORK_TESTS` env var. /// @@ -1260,16 +1257,13 @@ fn cached_mandatory_checkpoint_test_config() -> Result { /// /// Returns an error if the child exits or the fixed timeout elapses /// before `STOP_AT_HEIGHT_REGEX` is found. -fn create_cached_database_height

( +fn create_cached_database_height( network: Network, height: Height, debug_skip_parameter_preload: bool, checkpoint_sync: bool, - test_child_predicate: impl Into>, -) -> Result<()> -where - P: FnOnce(&mut TestChild) -> Result<()>, -{ + stop_regex: &str, +) -> Result<()> { println!("Creating cached database"); // 16 hours let timeout = Duration::from_secs(60 * 60 * 16); @@ -1295,11 +1289,7 @@ where child.expect_stdout_line_matches("starting legacy chain check")?; child.expect_stdout_line_matches("no legacy chain found")?; - if let Some(test_child_predicate) = test_child_predicate.into() { - test_child_predicate(&mut child)?; - } - - child.expect_stdout_line_matches(STOP_AT_HEIGHT_REGEX)?; + child.expect_stdout_line_matches(stop_regex)?; child.kill()?; @@ -1308,38 +1298,31 @@ where fn create_cached_database(network: Network) -> Result<()> { let height = network.mandatory_checkpoint_height(); + let checkpoint_stop_regex = format!("{}.*CommitFinalized request", STOP_AT_HEIGHT_REGEX); + create_cached_database_height( network, height, true, // Use checkpoints to increase sync performance while caching the database true, - |test_child: &mut TestChild| { - // make sure pre-cached databases finish before the mandatory checkpoint - // - // TODO: this check passes even if we reach the mandatory checkpoint, - // because we sync finalized state, then non-finalized state. - // Instead, fail if we see "best non-finalized chain root" in the logs. - test_child.expect_stdout_line_matches("CommitFinalized request")?; - Ok(()) - }, + // Check that we're still using checkpoints when we finish the cached sync + &checkpoint_stop_regex, ) } fn sync_past_mandatory_checkpoint(network: Network) -> Result<()> { let height = network.mandatory_checkpoint_height() + 1200; + let full_validation_stop_regex = + format!("{}.*best non-finalized chain root", STOP_AT_HEIGHT_REGEX); + create_cached_database_height( network, height.unwrap(), false, // Test full validation by turning checkpoints off false, - |test_child: &mut TestChild| { - // make sure cached database tests finish after the mandatory checkpoint, - // using the non-finalized state (the checkpoint_sync config must be false) - test_child.expect_stdout_line_matches("best non-finalized chain root")?; - Ok(()) - }, + &full_validation_stop_regex, ) }