chore: add recursive integration test

.github/workflows/test-integration.yml

@@ -7,6 +7,9 @@ on:
       - ./compiler/integration-tests
   schedule:
     - cron: "0 2 * * *" # Run nightly at 2 AM UTC
+  push:
+    branches:
+      - kh-recursive-e2e
 
 jobs:
   wasm-packages-build-test:
@@ -24,33 +27,38 @@ jobs:
           repository: noir-lang/acvm
           path: acvm
 
-      - name: Setup Nix
-        uses: cachix/install-nix-action@v22
+      - uses: nixbuild/nix-quick-install-action@v25
         with:
-          nix_path: nixpkgs=channel:nixos-23.05
-          github_access_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - uses: cachix/cachix-action@v12
-        with:
-          name: barretenberg
-          authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
-
-      - name: Restore nix store cache
-        uses: actions/cache/restore@v3
-        id: cache
+          nix_conf: |
+            substituters = https://cache.nixos.org/ https://nix-community.cachix.org
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=
+            keep-outputs = true
+      - name: Restore and cache Nix store - ${{ matrix.id }}
+        uses: nix-community/cache-nix-action@v4
         with:
-          path: ${{ env.CACHED_PATH }}
+          # save a new cache every time ci file changes
           key: ${{ runner.os }}-flake-wasm-${{ hashFiles('*.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-flake-wasm
+            gc-linux: true
+            gc-max-store-size-linux: 1000000000
+
+            purge-caches: true
+            purge-key: ${{ runner.os }}-flake-wasm
+            purge-created: true
+            purge-created-max-age: 42
+
+      # - name: Setup Nix
+      #   uses: cachix/install-nix-action@v22
+      #   with:
+      #     nix_path: nixpkgs=channel:nixos-23.05
+      #     github_access_token: ${{ secrets.GITHUB_TOKEN }}
+
+      # - uses: cachix/cachix-action@v12
+      #   with:
+      #     name: barretenberg
+      #     authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
 
-      # Based on https://github.com/marigold-dev/deku/blob/b5016f0cf4bf6ac48db9111b70dd7fb49b969dfd/.github/workflows/build.yml#L26
-      - name: Copy cache into nix store
-        if: steps.cache.outputs.cache-hit == 'true'
-        # We don't check the signature because we're the one that created the cache
-        run: |
-          for narinfo in ${{ env.CACHED_PATH }}/*.narinfo; do
-            path=$(head -n 1 "$narinfo" | awk '{print $2}')
-            nix copy --no-check-sigs --from "file://${{ env.CACHED_PATH }}" "$path"
-          done
 
       - name: Build noir_wasm package
         run: |

compiler/integration-tests/package.json

@@ -7,19 +7,20 @@
     "build": "echo Integration Test build step",
     "test": "yarn test:browser",
     "test:browser": "web-test-runner",
-    "test:integration:browser": "web-test-runner test/integration/browser/**/*.test.ts",
-    "test:integration:browser:watch": "web-test-runner test/integration/browser/**/*.test.ts --watch",
+    "test:integration:browser": "web-test-runner test/integration/browser/recursion.test.ts",
+    "test:integration:browser:watch": "web-test-runner test/integration/browser/recursion.test.ts --watch",
     "lint": "NODE_NO_WARNINGS=1 eslint . --ext .ts --ignore-path ./.eslintignore  --max-warnings 0"
   },
   "dependencies": {
-    "@aztec/bb.js": "^0.6.7",
+    "@aztec/bb.js": "0.7.2",
     "@noir-lang/noir_js": "workspace:*",
     "@noir-lang/noir_wasm": "workspace:*",
     "@noir-lang/source-resolver": "workspace:*",
     "@web/dev-server-esbuild": "^0.3.6",
     "@web/test-runner": "^0.15.3",
     "@web/test-runner-webdriver": "^0.7.0",
     "fflate": "^0.8.0",
-    "smol-toml": "^1.1.2"
+    "smol-toml": "^1.1.2",
+    "tslog": "^4.9.2"
   }
 }

compiler/integration-tests/test/circuits/main/Nargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "main"
+type = "bin"
+authors = [""]
+compiler_version = "0.9.0"
+
+[dependencies]

compiler/integration-tests/test/circuits/main/Prover.toml

@@ -0,0 +1,2 @@
+x = 1
+y = 2

compiler/integration-tests/test/circuits/main/src/main.nr

@@ -0,0 +1,3 @@
+fn main(x : Field, y : pub Field) {
+    assert(x != y);
+}

compiler/integration-tests/test/circuits/recursion/Nargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "recursion"
+type = "bin"
+authors = [""]
+compiler_version = "0.9.0"
+
+[dependencies]

compiler/integration-tests/test/circuits/recursion/Prover.toml

@@ -0,0 +1 @@
+input_aggregation_object = ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0"]

compiler/integration-tests/test/circuits/recursion/src/main.nr

@@ -0,0 +1,20 @@
+use dep::std;
+
+fn main(
+    verification_key : [Field; 114], 
+    proof : [Field; 94], 
+    public_inputs : [Field; 1], 
+    key_hash : Field, 
+    input_aggregation_object : [Field; 16],
+) -> pub [Field;16]{
+    let vk : [Field] = verification_key;
+    let p : [Field] = proof;
+    let pi : [Field] = public_inputs;
+    std::verify_proof(
+        vk, 
+        p, 
+        pi, 
+        key_hash, 
+        input_aggregation_object
+    )
+}

compiler/integration-tests/test/environment.js

@@ -0,0 +1 @@
+export const TEST_LOG_LEVEL = 5; // 0: silly, 1: trace, 2: debug, 3: info, 4: warn, 5: error, 6: fatal

compiler/integration-tests/test/integration/browser/recursion.test.ts

@@ -0,0 +1,236 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
+import { expect } from "@esm-bundle/chai";
+import { TEST_LOG_LEVEL } from "../../environment.js";
+import { Logger } from "tslog";
+import { initializeResolver } from "@noir-lang/source-resolver";
+import newCompiler, {
+  compile,
+  init_log_level as compilerLogLevel,
+} from "@noir-lang/noir_wasm";
+import { decompressSync as gunzip } from "fflate";
+import newABICoder, { abiEncode } from "@noir-lang/noirc_abi";
+import initACVM, {
+  executeCircuit,
+  WitnessMap,
+  compressWitness,
+} from "@noir-lang/acvm_js";
+
+// @ts-ignore
+import { Barretenberg, RawBuffer, Crs } from "@aztec/bb.js";
+
+import * as TOML from "smol-toml";
+
+const logger = new Logger({ name: "test", minLevel: TEST_LOG_LEVEL });
+
+await newCompiler();
+await newABICoder();
+await initACVM();
+
+compilerLogLevel("INFO");
+
+const numberOfThreads = navigator.hardwareConcurrency || 1;
+
+const base_relative_path = "../../../../..";
+const circuit_main = "compiler/integration-tests/test/circuits/main";
+const circuit_recursion = "compiler/integration-tests/test/circuits/recursion";
+
+async function getFile(url: URL): Promise<string> {
+  const response = await fetch(url);
+
+  if (!response.ok) throw new Error("Network response was not OK");
+
+  return await response.text();
+}
+
+const CIRCUIT_SIZE = 2 ** 19;
+
+const api = await Barretenberg.new(numberOfThreads);
+await api.commonInitSlabAllocator(CIRCUIT_SIZE);
+// Plus 1 needed!
+const crs = await Crs.new(CIRCUIT_SIZE + 1);
+await api.srsInitSrs(
+  new RawBuffer(crs.getG1Data()),
+  crs.numPoints,
+  new RawBuffer(crs.getG2Data())
+);
+
+const acirComposer = await api.acirNewAcirComposer(CIRCUIT_SIZE);
+
+async function getCircuit(noirSource) {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  initializeResolver((id: string) => {
+    return noirSource;
+  });
+
+  return compile({});
+}
+
+async function generateWitness(acir, abi, inputs) {
+  const witnessMap: WitnessMap = abiEncode(abi, inputs, null);
+
+  const compressedByteCode = Uint8Array.from(atob(acir), (c) =>
+    c.charCodeAt(0)
+  );
+
+  return executeCircuit(compressedByteCode, witnessMap, () => {
+    throw Error("unexpected oracle");
+  });
+}
+
+async function generateProof(
+  acirUint8Array: Uint8Array,
+  witnessUint8Array: Uint8Array,
+  optimizeForRecursion: boolean
+) {
+  // This took ~6.5 minutes!
+  return api.acirCreateProof(
+    acirComposer,
+    acirUint8Array,
+    witnessUint8Array,
+    optimizeForRecursion
+  );
+}
+
+async function verifyProof(proof: Uint8Array, optimizeForRecursion: boolean) {
+  await api.acirInitVerificationKey(acirComposer);
+  const verified = await api.acirVerifyProof(
+    acirComposer,
+    proof,
+    optimizeForRecursion
+  );
+  return verified;
+}
+
+describe("It compiles noir program code, receiving circuit bytes and abi object.", () => {
+  let circuit_main_source;
+  let circuit_main_toml;
+  let circuit_recursion_source;
+
+  before(async () => {
+    const circuit_main_source_url = new URL(
+      `${base_relative_path}/${circuit_main}/src/main.nr`,
+      import.meta.url
+    );
+    const circuit_main_toml_url = new URL(
+      `${base_relative_path}/${circuit_main}/Prover.toml`,
+      import.meta.url
+    );
+
+    circuit_main_source = await getFile(circuit_main_source_url);
+    circuit_main_toml = await getFile(circuit_main_toml_url);
+
+    const circuit_recursion_source_url = new URL(
+      `${base_relative_path}/${circuit_recursion}/src/main.nr`,
+      import.meta.url
+    );
+
+    circuit_recursion_source = await getFile(circuit_recursion_source_url);
+  });
+
+  it("Should generate valid inner proof for correct input, then verify proof within a proof", async () => {
+    //@ts-ignore
+    const { circuit: main_circuit, abi: main_abi } = await getCircuit(
+      circuit_main_source
+    );
+    const main_inputs = TOML.parse(circuit_main_toml);
+
+    const main_witness = await generateWitness(
+      main_circuit,
+      main_abi,
+      main_inputs
+    );
+    const main_compressedByteCode = Uint8Array.from(atob(main_circuit), (c) =>
+      c.charCodeAt(0)
+    );
+    const main_compressedWitness = compressWitness(main_witness);
+    const main_acirUint8Array = gunzip(main_compressedByteCode);
+    const main_witnessUint8Array = gunzip(main_compressedWitness);
+
+    const optimizeMainProofForRecursion = true;
+
+    const main_proof = await generateProof(
+      main_acirUint8Array,
+      main_witnessUint8Array,
+      optimizeMainProofForRecursion
+    );
+
+    const main_verification = await verifyProof(
+      main_proof,
+      optimizeMainProofForRecursion
+    );
+
+    logger.debug("main_verification", main_verification);
+
+    expect(main_verification).to.be.true;
+
+    const numPublicInputs = 1;
+    const proofAsFields = (
+      await api.acirSerializeProofIntoFields(
+        acirComposer,
+        main_proof,
+        numPublicInputs
+      )
+    ).map((p) => p.toString());
+
+    const vk = await api.acirSerializeVerificationKeyIntoFields(acirComposer);
+
+    const vkAsFields = vk[0].map((vk) => vk.toString());
+    const vkHash = vk[1].toString();
+
+    const recursion_inputs = {
+      verification_key: vkAsFields,
+      proof: proofAsFields,
+      public_inputs: [main_inputs.y],
+      key_hash: vkHash,
+      // eslint-disable-next-line prettier/prettier
+      input_aggregation_object: ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0"]
+    };
+
+    logger.debug("recursion_inputs", recursion_inputs);
+
+    const { circuit: recursion_circuit, abi: recursion_abi } = await getCircuit(
+      circuit_recursion_source
+    );
+
+    const recursion_witness = await generateWitness(
+      recursion_circuit,
+      recursion_abi,
+      recursion_inputs
+    );
+
+    const recursion_compressedByteCode = Uint8Array.from(
+      atob(recursion_circuit),
+      (c) => c.charCodeAt(0)
+    );
+
+    const recursion_compressedWitness = compressWitness(recursion_witness);
+    const recursion_acirUint8Array = gunzip(recursion_compressedByteCode);
+    const recursion_witnessUint8Array = gunzip(recursion_compressedWitness);
+
+    const optimizeRecursionProofForRecursion = false;
+
+    const recursion_proof = await generateProof(
+      recursion_acirUint8Array,
+      recursion_witnessUint8Array,
+      optimizeRecursionProofForRecursion
+    );
+
+    const recursion_numPublicInputs = 1;
+
+    const recursion_proofAsFields = (
+      await api.acirSerializeProofIntoFields(
+        acirComposer,
+        recursion_proof,
+        recursion_numPublicInputs
+      )
+    ).map((p) => p.toString());
+
+    logger.debug("recursion_proofAsFields", recursion_proofAsFields);
+
+    const recursion_verification = await verifyProof(recursion_proof, false);
+
+    logger.debug("recursion_verification", recursion_verification);
+
+    expect(recursion_verification).to.be.true;
+  }).timeout(60 * 20e3);
+});