test: refactor foundry test directory into unit, e2e, and integration…

… tests (#11023)

* chore(foundry.toml): adds `@mento-core/...` remapping

Also moves existing mappings into groups for better readability

* refactor(test-sol/common): moves files to `unit/common/`

Also updates imports respectively using remappings.

* refactor(test-sol/identity): moves files to `unit/identity/`

Also updates imports using remappings where necessary.

* refactor(test-sol/stability): moves files to `unit/stability/`

Also updates imports using remappings where necessary.

* refactor(test-sol/voting): moves files to `unit/voting/`

Also updates imports using remappings where necessary.

* refactor(test-sol/governance): moves files to `unit/governance/`

Also updates imports using remappings where necessary.

* chore(test-sol): update missing remappings

With these changes all contracts compile and resolve correctly.

* chore(workflows/protocol_tests): updates paths to `test-sol/unit`

* chore(workflows/protocol_tests): adds integration and e2e tests

* style(workflows/protocol_tests): refactors `forge test` for better readability

* chore(migrations_sol): moves scripts to `scripts/foundry/

Moves all bash scripts from `migrations_sol` to `scripts/foundry/`, and updates paths where necessary. We already have a directory for `scripts/truffle/` and `scripts/bash/`, so this makes it easier to find foundry-related bash scripts.

* test(governance/mock): move `MockGovernance` to `unit/` folder

* refactor(foundry.toml): rename `migrations-sol/` remapping

* refactor(workflows): refactor "run everything" step

Runs all tests in the `unit/` directory instead of all test files in the repo.
This makes sense from my perspective, because e2e tests (in the `e2e/` directory) and integration tests (in the `integration/` directory) require a connection to an anvil devchain serving at localhost.

The intent of this command is to ensure that no unit tests are forgotten, since unit tests are run explicitly by going through the directories above. But, the intention is not to run all tests in the repo generally.

* style(workflows/protocol-devchain-anvil): splits migration into 2 steps

This helps the script becoming more readable, and ensure error logs are clearly associated with a workflow step rather than a single bash script.

* chore(workflows): defines `ANVIL_PORT` env variable in workflow

Previously, the `$ANVIL_PORT` variable was exported and passed around as an env variable in a bash script.

But that required generating anvil migrations and running a devchain repeatedly.
Instead, the workflow does that once and different steps can access the devchain.

But, in that case the env variable is not passed around, so it has to be defined at the workflow level.

Source: https://docs.github.com/en/actions/learn-github-actions/variables

* chore(workflows/protocol_tests): removes code comment

* feat(scripts/foundry): adds `stop_anvil.sh`

* refactor(scripts/foundry): use `stop_anvil.sh` to stop anvil

* feat(protocol/package.json): adds `anvil-devchain:...` (`start` and `stop`)

* refactor(scripts/foundry): use `stop_anvil.sh` to stop anvil

* refactor(migrations_sol): update `@migrations-sol` remapping

Previous the remapping was called `@celo-migrations`.

* docs(migrations_sol/README): renames README file

Also changes title to match NPM package name. This is a matter of personal preference. IMO this is a small improvement in readability for 3rd party users.

* docs(scripts/foundry): adds more links to `package.json`

Adds `repository` and `directory` to `package.json`.
This ensures npmjs.org displays hyperlinks to the github repository.

* docs(migrations_sol/CONTRIBUTING): adds MVP dev docs

We can complete this doc with additional information regarding the anvil devchain and migrations going forward.

* docs(migrations_sol/README): adds "how we work" section

This is helpful for 3rd party developers that found this package on npmjs.org.
The links help developers take action and help improve the anvil devchain.

.github/workflows/protocol-devchain-anvil.yml

@@ -7,6 +7,7 @@ on:
 env:
   # Increment these to force cache rebuilding
   FOUNDRY_CACHE_KEY: 1
+  ANVIL_PORT: 8546
 
 jobs:
   build:
@@ -86,8 +87,16 @@ jobs:
         id: date
         run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
 
-      - name: Generate migrations
-        run: ./migrations_sol/run_integration_tests_in_anvil.sh
+      - name: Generate migrations and run devchain
+        if: success() || failure()
+        run: ./scripts/foundry/create_and_migrate_anvil_devchain.sh
+
+      - name: Run integration tests against local anvil devchain
+        if: success() || failure()
+        run: |
+          forge test -vvv \
+          --match-path "test-sol/integration/*" \
+          --fork-url http://127.0.0.1:${{ env.ANVIL_PORT }}
 
       - name: Sanitize ref name
         id: sanitize-ref-name

.github/workflows/protocol_tests.yml

@@ -11,6 +11,7 @@ on:
 env:
   # Increment these to force cache rebuilding
   FOUNDRY_CACHE_KEY: 2
+  ANVIL_PORT: 8546
 
 jobs:
   check:
@@ -61,30 +62,47 @@ jobs:
       - name: Compile Contracts
         run: forge --version && forge compile
 
-      - name: Run tests common
+      - name: Run unit tests common
         # can't use gas limit because some setUp function use more than the limit
-        run: forge test -vvv --match-path "test-sol/common/*" # --block-gas-limit 50000000
+        run: |
+          forge test -vvv \
+          --match-path "test-sol/unit/common/*"
 
-      - name: Run tests governance/network
+      - name: Run unit tests governance/network
         if: success() || failure()
-        run: forge test -vvv --block-gas-limit 50000000 --match-path "test-sol/governance/network/*"
+        run: |
+          forge test -vvv \
+          --match-path "test-sol/unit/governance/network/*" \
+          --block-gas-limit 50000000
 
-      - name: Run tests governance/validators
+      - name: Run unit tests governance/validators
         if: success() || failure()
-        run: forge test -vvv --block-gas-limit 50000000 --match-path "test-sol/governance/validators/*"
+        run: | 
+          forge test -vvv \
+          --match-path "test-sol/unit/governance/validators/*" \
+          --block-gas-limit 50000000
 
-      - name: Run tests governance/voting
+      - name: Run unit tests governance/voting
         # can't use gas limit because some setUp function use more than the limit
         if: success() || failure()
-        run: forge test -vvv --match-path "test-sol/governance/voting/*" --block-gas-limit 50000000
+        run: | 
+          forge test -vvv \
+          --match-path "test-sol/unit/governance/voting/*" \
+          --block-gas-limit 50000000
 
-      - name: Run tests stability
+      - name: Run unit tests stability
         if: success() || failure()
-        run: forge test -vvv --block-gas-limit 50000000 --match-path "test-sol/stability/*"
+        run: |
+          forge test -vvv \
+          --match-path "test-sol/unit/stability/*" \
+          --block-gas-limit 50000000
 
-      - name: Run tests identity
+      - name: Run unit tests identity
         if: success() || failure()
-        run: forge test -vvv --block-gas-limit 50000000 --match-path "test-sol/identity/*"
+        run: |
+          forge test -vvv \
+          --match-path "test-sol/unit/identity/*" \
+          --block-gas-limit 50000000
 
       - name: Fail if there are tests without folder
         if: success() || failure()
@@ -94,11 +112,28 @@ jobs:
             exit 1
           fi
 
-      - name: Run everything just in case something was missed (excl. integration and e2e tests)
+      - name: Run all unit tests in case some were missed (excl. integration and e2e tests)
         # can't use gas limit because some setUp function use more than the limit
-        # Excludes integration tests, because they require an anvil devchain running on the localhost.
-        run: forge test -vvv --no-match-contract "RegistryIntegrationTest|E2EDemo"
+        # Excludes e2e and integration tests, because they require a connection to an anvil devchain
+        # serving at localhost.
+        run: |
+          forge test -vvv \
+          --match-path "test-sol/unit/*"
       
-      - name: Generate migrations
+      - name: Generate migrations and run devchain
         if: success() || failure()
-        run: ./migrations_sol/run_integration_tests_in_anvil.sh
+        run: ./scripts/foundry/create_and_migrate_anvil_devchain.sh
+
+      - name: Run integration tests against local anvil devchain
+        if: success() || failure()
+        run: |
+          forge test -vvv \
+          --match-path "test-sol/integration/*" \
+          --fork-url http://127.0.0.1:${{ env.ANVIL_PORT }}
+      
+      - name: Run e2e tests against local anvil devchain
+        if: success() || failure()
+        run: |
+          forge test -vvv \
+          --match-path "test-sol/e2e/*" \
+          --fork-url http://127.0.0.1:${{ env.ANVIL_PORT }}

packages/protocol/foundry.toml

@@ -4,20 +4,20 @@ out = 'out'
 test = 'test-sol'
 libs = ['lib', 'node_modules']
 remappings = [
+    '@celo-contracts/=contracts/',
+    '@celo-contracts-8=contracts-0.8/',
+    '@mento-core/=lib/mento-core',
     'openzeppelin-solidity/=lib/openzeppelin-contracts/',
-    'solidity-bytes-utils/=lib/solidity-bytes-utils/',
-    'forge-std/=lib/celo-foundry/lib/forge-std/src/',
-    'ds-test/=lib/celo-foundry/lib/forge-std/lib/ds-test/src/',
+    '@openzeppelin/contracts8/=lib/openzeppelin-contracts8/contracts/',
     'celo-foundry/=lib/celo-foundry/src/',
-    '@summa-tx/memview.sol/=lib/memview.sol',
     'celo-foundry-8/=lib/celo-foundry-8/src/',
-    'forge-std-8/=lib/celo-foundry-8/lib/forge-std/src/',
-    '@celo-contracts-8=contracts-0.8/',
-    '@openzeppelin/contracts8/=lib/openzeppelin-contracts8/contracts/',
-    '@celo-contracts/=contracts/',
-    '@celo-migrations/=migrations_sol/',
     '@test-sol/=test-sol/',
-    '@mento-core/=lib/mento-core/'
+    '@migrations-sol/=migrations_sol/',
+    'forge-std/=lib/celo-foundry/lib/forge-std/src/',
+    'forge-std-8/=lib/celo-foundry-8/lib/forge-std/src/',
+    '@summa-tx/memview.sol/=lib/memview.sol',
+    'solidity-bytes-utils/=lib/solidity-bytes-utils/',
+    'ds-test/=lib/celo-foundry/lib/forge-std/lib/ds-test/src/',
 ]
 
 no_match_test = "skip"

packages/protocol/migrations_sol/CONTRIBUTING.md

@@ -0,0 +1,57 @@
+# Anvil migrations 
+
+The "Anvil migrations" are a set of scripts that generate a local anvil-based devchain. 
+This devchain is useful for testing and development of the Celo protocol.
+
+## Usage
+
+### Start devchain
+
+```sh
+$ yarn anvil-devchain:start
+``` 
+
+Starts a new anvil devchain serving at localhost (default port 8546).
+
+For example:
+
+```sh
+$ yarn anvil-devchain:start
+
+yarn run v1.22.22
+$ ./scripts/foundry/create_and_migrate_anvil_devchain.sh
+# ...
+Total elapsed time: 193 seconds
+✨  Done in 193.09s.
+```
+
+You can now run commands against the local devchain.
+
+For example:
+
+```sh
+cast block-number \
+--rpc-url http://127.0.0.1:8546
+266
+```
+
+### Stop devchain
+
+```sh
+$ yarn anvil-devchain:stop
+```
+
+Terminates any anvil nodes serving at localhost.
+
+For example:
+
+```sh
+# in packages/protocol/ directory
+$ yarn anvil-devchain:stop
+
+yarn run v1.22.22
+$ ./scripts/foundry/stop_anvil.sh
+Connection to localhost port 8546 [tcp/*] succeeded!
+Killed Anvil
+✨  Done in 0.11s.
+```

packages/protocol/migrations_sol/Migration.s.sol

@@ -43,12 +43,12 @@ import "@celo-contracts/stability/interfaces/ISortedOracles.sol";
 import "@celo-contracts-8/common/interfaces/IGasPriceMinimumInitializer.sol";
 import "@celo-contracts-8/common/interfaces/IMintGoldScheduleInitializer.sol";
 
-import "@celo-migrations/HelperInterFaces.sol";
+import "@migrations-sol/HelperInterFaces.sol";
 import "@openzeppelin/contracts8/utils/math/Math.sol";
 
 import "@celo-contracts-8/common/UsingRegistry.sol";
 
-import { Constants } from "@celo-migrations/constants.sol";
+import { Constants } from "@migrations-sol/constants.sol";
 
 contract ForceTx {
   // event to trigger so a tx can be processed

packages/protocol/migrations_sol/README.md

@@ -0,0 +1,30 @@
+# @celo/devchain-anvil
+
+Anvil state with Celo core contracts for local testing and development.
+
+# Usage
+
+```bash
+npm install --save-dev @celo/devchain-anvil
+anvil --state <path_to_devchain.json>
+```
+
+## How we work
+
+We are a GitHub-first team, which means we have a strong preference for communicating via GitHub. 
+Please use GitHub to:
+
+🐞 [File a bug report](https://github.com/celo-org/celo-monorepo/issues/new/choose)
+
+💬 [Ask a question](https://github.com/celo-org/celo-monorepo/discussions)
+
+✨ [Suggest a feature](https://github.com/celo-org/celo-monorepo/issues/new/choose)
+
+🧑‍💻 [Contribute!](https://github.com/celo-org/celo-monorepo/tree/master/packages/protocol/migrations_sol/CONTRIBUTING.md)
+
+🚔 [Report a security vulnerability](https://github.com/celo-org/celo-monorepo/issues/new/choose)
+
+> [!TIP]
+> 
+> Please avoid messaging us via Slack, Telegram, or email. We are more likely to respond to you on 
+> GitHub than if you message us anywhere else. We actively monitor GitHub, and will get back to you shortly 🌟

packages/protocol/migrations_sol/run_e2e_tests_in_anvil.sh

@@ -12,10 +12,6 @@ forge test \
 --match-path "*test-sol/e2e/*" \
 --fork-url http://127.0.0.1:$ANVIL_PORT
 
-# helper kill anvil
-# kill $(lsof -i tcp:$ANVIL_PORT | tail -n 1 | awk '{print $2}')
-
-echo "Killing Anvil"
-if [[ -n $ANVIL_PID ]]; then
-    kill $ANVIL_PID
-fi
+# Stop devchain
+echo "Stopping devchain..."
+source $PWD/scripts/foundry/stop_anvil.sh

packages/protocol/package.json

@@ -50,6 +50,8 @@
     "truffle:migrate": "truffle migrate",
     "devchain": "yarn ts-node scripts/devchain.ts",
     "devchain:reset": "yarn devchain generate-tar .tmp/devchain.tar.gz --upto 29",
+    "anvil-devchain:start": "./scripts/foundry/create_and_migrate_anvil_devchain.sh",
+    "anvil-devchain:stop": "./scripts/foundry/stop_anvil.sh",
     "view-tags": "git for-each-ref 'refs/tags/core-contracts.*' --sort=-committerdate --format='%(color:magenta)%(committerdate:short) %(color:blue)%(tree) %(color:green)github.com/celo-org/celo-monorepo/releases/tag/%(color:yellow)%(refname:short)'",
     "generate-stabletoken-files": "yarn ts-node ./scripts/generate-stabletoken-files.ts",
     "truffle-verify": "yarn truffle run verify"

packages/protocol/migrations_sol/create_and_migrate_anvil_devchain.sh → packages/protocol/scripts/foundry/create_and_migrate_anvil_devchain.sh

@@ -22,10 +22,10 @@ fi
 mkdir -p $TEMP_FOLDER
 
 # Start a local anvil instance
-source $PWD/migrations_sol/start_anvil.sh
+source $PWD/scripts/foundry/start_anvil.sh
 
 # Deploy libraries to the anvil instance
-source $PWD/migrations_sol/deploy_libraries.sh
+source $PWD/scripts/foundry/deploy_libraries.sh
 echo "Library flags are: $LIBRARY_FLAGS"
 
 # Build all contracts with deployed libraries
@@ -35,7 +35,7 @@ echo "Compiling with libraries... "
 time forge build $LIBRARY_FLAGS
 
 # Deploy precompile contracts
-source $PWD/migrations_sol/deploy_precompiles.sh
+source $PWD/scripts/foundry/deploy_precompiles.sh
 
 echo "Setting Registry Proxy"
 REGISTRY_ADDRESS="0x000000000000000000000000000000000000ce10"

packages/protocol/migrations_sol/run_integration_tests_in_anvil.sh → packages/protocol/scripts/foundry/run_integration_tests_in_anvil.sh

@@ -3,7 +3,7 @@ set -euo pipefail
 
 # Generate and run devchain
 echo "Generating and running devchain before running integration tests..."
-source $PWD/migrations_sol/create_and_migrate_anvil_devchain.sh
+source $PWD/scripts/foundry/create_and_migrate_anvil_devchain.sh
 
 # Run integration tests
 echo "Running integration tests..."
@@ -12,10 +12,6 @@ forge test \
 --match-contract RegistryIntegrationTest \
 --fork-url http://127.0.0.1:$ANVIL_PORT
 
-# helper kill anvil
-# kill $(lsof -i tcp:$ANVIL_PORT | tail -n 1 | awk '{print $2}')
-
-echo "Killing Anvil"
-if [[ -n $ANVIL_PID ]]; then
-    kill $ANVIL_PID
-fi
+# Stop devchain
+echo "Stopping devchain..."
+source $PWD/scripts/foundry/stop_anvil.sh

packages/protocol/migrations_sol/start_anvil.sh → packages/protocol/scripts/foundry/start_anvil.sh

@@ -8,9 +8,9 @@ mkdir -p $ANVIL_FOLDER
 echo "Anvil state will be saved to $ANVIL_FOLDER"
 
 # create package.json
-echo "{\"name\": \"@celo/devchain-anvil\",\"version\": \"1.0.0\",\"description\": \"Anvil based devchain that contains core smart contracts of celo\",\"author\":\"Celo\",\"license\": \"LGPL-3.0\"}" > $TMP_FOLDER/package.json
+echo "{\"name\": \"@celo/devchain-anvil\",\"version\": \"1.0.0\",\"repository\": { \"url\": \"https://github.com/celo-org/celo-monorepo\", \"directory\": \"packages/protocol/migrations_sol\" },\"homepage\": \"https://github.com/celo-org/celo-monorepo/blob/master/packages/protocol/migrations_sol/README.md\",\"description\": \"Anvil based devchain that contains core smart contracts of celo\",\"author\":\"Celo\",\"license\": \"LGPL-3.0\"}" > $TMP_FOLDER/package.json
 
-cp $PWD/migrations_sol/celo-anvil-README.md $TMP_FOLDER/README.md
+cp $PWD/migrations_sol/README.md $TMP_FOLDER/README.md
 
 if nc -z localhost $ANVIL_PORT; then
   echo "Port already used"

packages/protocol/scripts/foundry/stop_anvil.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# A small script to terminate any instance of anvil currently serving at localhost.
+
+ANVIL_PORT=8546
+
+if nc -z localhost $ANVIL_PORT; then
+  kill $(lsof -i tcp:$ANVIL_PORT | tail -n 1 | awk '{print $2}')
+  echo "Killed Anvil"
+fi

packages/protocol/test-sol/integration/Integration.t.sol

@@ -4,7 +4,7 @@ pragma solidity >=0.8.7 <0.8.20;
 import { Test } from "forge-std-8/Test.sol";
 import "forge-std-8/console2.sol";
 
-import { Constants } from "@celo-migrations/constants.sol";
+import { Constants } from "@migrations-sol/constants.sol";
 
 import "@celo-contracts/common/interfaces/IRegistry.sol";
 import "@celo-contracts/common/interfaces/IProxy.sol";