Merge pull request #72 from trung/multitenant

Serve multi tenants from a single node

.github/workflows/run.yml

@@ -113,6 +113,7 @@ jobs:
           - 'permissions-v2 && networks/template::raft-3plus1'
           - 'privacy-enhancements-upgrade || networks/template::raft-4nodes-pe'
           - 'privacy-enhancements-upgrade || networks/template::istanbul-4nodes-pe'
+          - 'multitenancy && networks/plugins::raft-multitenancy'
         privacy-enhancements:
           - 'false'
         include:
@@ -143,17 +144,42 @@ jobs:
             echo "INFRA_FOLDER=$infraFolder" >> $GITHUB_ENV
             echo "INFRA_PROFILE=$infraProfile" >> $GITHUB_ENV
           fi
+          dockerEnvFile=${{ runner.temp }}/env.list
+          # now we check if we should use the custom docker images in this repo
+          gitref_path="${{ github.ref }}"
+          gitref_path=${gitref_path/refs\/heads\//} # for refs/heads/my-branch
+          if [[ $gitref_path == dev-* ]]; then
+            echo "${{ github.token }}" | docker login https://docker.pkg.github.com -u ${{ github.repository_owner }} --password-stdin
+            quorum_docker_image="docker.pkg.github.com/${{ github.repository }}/quorum-$gitref_path:latest"
+            tessera_docker_image="docker.pkg.github.com/${{ github.repository }}/tessera-$gitref_path:latest"
+            has_quorum_docker_image=$(docker pull $quorum_docker_image >/dev/null 2>&1; echo $?)
+            has_tessera_docker_image=$(docker pull $tessera_docker_image >/dev/null 2>&1; echo $?)
+            echo "$quorum_docker_image: $has_quorum_docker_image"
+            echo "$tessera_docker_image: $has_tessera_docker_image"
+            if [ $has_quorum_docker_image -eq 0 ]; then
+              echo "::warning ::Using $quorum_docker_image"
+              echo "TF_VAR_quorum_docker_image={name=\"$quorum_docker_image\", local=true}" >> $dockerEnvFile
+              docker pull quorumengineering/quorum:latest
+            fi
+            if [ $has_tessera_docker_image -eq 0 ]; then
+              echo "::warning ::Using $tessera_docker_image"
+              echo "TF_VAR_tessera_docker_image={name=\"$tessera_docker_image\", local=true}" >> $dockerEnvFile
+              docker pull quorumengineering/tessera:latest
+            fi
+          fi
+          echo "TF_VAR_privacy_enhancements={block=0, enabled=${{ matrix.privacy-enhancements}}}" >> $dockerEnvFile
           echo "::set-output name=tag::$tagKey"
           echo "::set-output name=mvnArg::$mvnArg"
           echo "::set-output name=dockerEnv::$dockerEnv"
           echo "::set-output name=outputDir::${{ runner.temp }}"
+          echo "::set-output name=dockerEnvFile::$dockerEnvFile"
       - name: 'Run tests using ${{ needs.condition.outputs.infra }}'
         run: |
           # we don't remove the container after run as we need to clean up the infra if used
           docker run \
               --name acctests-run ${{ steps.setup.outputs.dockerEnv }} \
               -v ${{ steps.setup.outputs.outputDir }}:${{ steps.setup.outputs.outputDir }} \
-              -e TF_VAR_privacy_enhancements='{ block = 0, enabled = ${{ matrix.privacy-enhancements }} }' \
+              --env-file ${{ steps.setup.outputs.dockerEnvFile }} \
               ${{ needs.docker-build.outputs.image_name }} test \
                 -PgaugeFailSafe \
                 -Pauto \

Dockerfile

@@ -46,4 +46,4 @@ RUN apk -q --no-cache --update add tar bash \
     && rm -rf /tmp/downloads
 
 ENTRYPOINT ["mvn", "--no-transfer-progress", "-B"]
-CMD ["test", "-Dtags=basic"]
+CMD ["test", "-Dtags=basic"]

README.md

@@ -3,7 +3,7 @@
 # Quick Start
 
 - Install [Docker Engine](https://docs.docker.com/engine/) or [Docker Desktop](https://www.docker.com/products/docker-desktop)
-- Run basic acceptance tests against a new Quorum network using Raft consensus:
+- Run basic acceptance tests against a new GoQuorum network using Raft consensus:
     ```
     docker run --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/acctests:/tmp/acctests \
             quorumengineering/acctests:latest test -Pauto -Dtags="basic || networks/typical::raft" \
@@ -14,31 +14,32 @@
 
 Development environment requires the following:
 
-* JDK 11+
-* Maven 3.6.x
-* [Solidity Compiler](https://solidity.readthedocs.io/en/latest/installing-solidity.html) (make sure `solc` is installed and not `solcjs`)
-  * For MacOS: use `brew`
-  * For Linux: use `apt`, `snap` or `emerge`
-  * For Windows: download from [here](https://github.com/ethereum/solidity/releases)
-* [Gauge](https://gauge.org/get_started)
+- JDK 11+
+- Maven 3.6.x
+- [Solidity Compiler](https://solidity.readthedocs.io/en/latest/installing-solidity.html) (make sure `solc` is installed and not `solcjs`)
+  - For MacOS: use `brew`
+  - For Linux: use `apt`, `snap` or `emerge`
+  - For Windows: download from [here](https://github.com/ethereum/solidity/releases)
+- [Gauge](https://gauge.org/get_started)
+- Run `mvn compile` to initiate the project with generated Java sources from Solidity source
 
 With built-in provisioning feature:
-* [Docker Engine](https://docs.docker.com/engine/) or [Docker Desktop](https://www.docker.com/products/docker-desktop)
-* [Terraform](https://terraform.io) 0.12.x
-* [Terraform Provider Quorum](https://bintray.com/quorumengineering/terraform/terraform-provider-quorum)
+- [Docker Engine](https://docs.docker.com/engine/) or [Docker Desktop](https://www.docker.com/products/docker-desktop)
+- [Terraform](https://terraform.io) 0.12.x
+- [Terraform Provider Quorum](https://bintray.com/quorumengineering/terraform/terraform-provider-quorum)
    - The provider should be downloaded from the link and unzipped into the directory `~/.terraform.d/plugins/`
    - Refer to [this guide](https://www.terraform.io/docs/configuration/providers.html#third-party-plugins) for more information regarding provider installation
-
+  
 **For more details on tools and versions being used, please refer to [Dockerfile](Dockerfile)**
 
 ## Writing Tests
 
-* Using [Gauge](https://github.com/getgauge/gauge) test automation framework
-* Test Specs are stored in [`src/specs`](src/specs) folder
-  * Folder `01_basic` contains specifications which describe Quorum's basic functionalities. All specifications must be tagged as `basic`
-  * Folder `02_advanced` contains specifications which are for making sure Quorum's basic functionalities are working under different conditions in the chain. All specifications must be tagged as `advanced`
-* Glue codes are written in Java under [`src/test/java`](src/test/java) folder
-* Tests are generally written against 4-node Quorum network
+- Use [Gauge](https://github.com/getgauge/gauge) test automation framework
+- Test Specs are stored in [`src/specs`](src/specs) folder
+  - Folder `01_basic` contains specifications which describe GoQuorum's basic functionalities. All specifications must be tagged as `basic`
+  - Folder `02_advanced` contains specifications which are for making sure GoQuorum's basic functionalities are working under different conditions in the chain. All specifications must be tagged as `advanced`
+- Glue codes are written in Java under [`src/test/java`](src/test/java) folder
+- Tests are generally written against 4-node GoQuorum network
 
 ## Running Tests
 
@@ -49,23 +50,23 @@ With built-in provisioning feature:
 > - Instructing Maven to provision `networks/typical` with profile `raft` when using Maven Profile `auto` (i.e.: `-Pauto`)
 > - `networks/typical` is a folder that contains Terraform configuration to provision the network
 
-* Run basic tests for raft consensus: 
+- Run basic tests for raft consensus: 
     ```
     mvn clean test -Pauto -Dtags="basic || basic-raft || networks/typical::raft"
     ```
-* Run basic tests for istanbul consensus:
+- Run basic tests for istanbul consensus:
     ```
     mvn clean test -Pauto -Dtags="basic || basic-istanbul || networks/typical::istanbul"
     ```
-* Force destroy the network after running tests:
+- Force destroy the network after running tests:
     ```
     mvn clean test -Pauto -Dtags="basic || basic-raft || networks/typical::raft" -Dnetwork.forceDestroy=true
     ```
-* Start the network without running tests:
+- Start the network without running tests:
     ```
     mvn process-test-resources -Pauto -Dnetwork.target="networks/typical::raft"
     ```
-* Destroy the network:
+- Destroy the network:
     ```
     mvn exec:exec@network.terraform-destroy -Pauto -Dnetwork.folder="networks/typical" -Dnetwork.profile=raft
     ```
@@ -74,18 +75,52 @@ Below is the summary of various parameters:
 
 | Parameters | Description |
 |------------|-------------|
-| `-Dnetwork.target="<folder>::<profile"` | Shorthand to specify the Terraform folder and profile being used to create Quorum Network |
-| `-Dnetwork.folder="<folder>"` | Terraform folder being used to create Quorum Network |
+| `-Dnetwork.target="<folder>::<profile"` | Shorthand to specify the Terraform folder and profile being used to create GoQuorum Network |
+| `-Dnetwork.folder="<folder>"` | Terraform folder being used to create GoQuorum Network |
 | `-Dnetwork.profile="<profile>"` | Terraform workspace and `terraform.<profile>.tfvars` being used |
-| `-Dnetwork.forceDestroy="true" or "false"` | Destroy the Quorum Network after test completed. Default is `false` |
-| `-Dnetwork.skipApply="true" or "false"` | Don't create Quorum Network. Default is `false` |
-| `-Dnetwork.skipWait="true" or "false"` | Don't perform health check and wait for Quorum Network. Default is `false` |
+| `-Dnetwork.forceDestroy` | Destroy the GoQuorum Network after test completed. Default is `false` |
+| `-Dnetwork.skipApply` | Don't create GoQuorum Network. Default is `false` |
+| `-Dnetwork.skipWait` | Don't perform health check and wait for GoQuorum Network. Default is `false` |
 | `-Dinfra.target="<folder>::<profile"` | Shorthand to specify the Terraform folder and profile being used to create an infrastructure to host Docker Engine |
 | `-Dinfra.folder="<folder>"` | Terraform folder being used to create the infrastructure |
 | `-Dinfra.profile="<profile>"` | Terraform workspace and `terraform.<profile>.tfvars` being used |
-| `-Dinfra.forceDestroy="true" or "false"` | Destroy the infrastructure after test completed. Default is `false` |
-| `-Dinfra.skipApply="true" or "false"` | Don't create the infrastructure. Default is `false` |
-| `-Dinfra.skipWait="true" or "false"` | Don't perform health check and wait for Quorum Network. Default is `false` |
+| `-Dinfra.forceDestroy` | Destroy the infrastructure after test completed. Default is `false` |
+| `-Dinfra.skipApply` | Don't create the infrastructure. Default is `false` |
+| `-Dinfra.skipWait` | Don't perform health check and wait for GoQuorum Network. Default is `false` |
+| `-DskipToolsCheck` | Don't check local tools required to run tests. Default is `false` |
+| `-DskipGenerateSol` | Don't generate Java stubs for Solidity files. Default is `false` |
+
+### With local binaries
+
+In order to run acceptance tests during GoQuorum/Tessera development:
+
+- Build GoQuorum/Tessera binaries locally targeting Linux arch.
+  E.g.: GoQuorum binaries are in `/xyz/go-ethereum/build/bin` folder and Tessera jar file is in `/abc/tessera/tessera-dist/tessera-app/target`
+- Mount binaries dynamically to overrride existing ones in the containers
+  > :bulb: Indices 0,1,2,3.. indicate Node id which you want to use the local binaries
+  - GoQuorum:
+  ```
+  export QUORUM_DEV_LOCAL='{host_path="/xyz/go-ethereum/build/bin", container_path="/usr/local/bin"}'
+  export TF_VAR_additional_quorum_container_vol="{0=[$QUORUM_DEV_LOCAL],1=[$QUORUM_DEV_LOCAL],2=[$QUORUM_DEV_LOCAL],3=[$QUORUM_DEV_LOCAL]}"
+  ````
+  - Tessera:
+  ```
+  export TESSERA_DEV_LOCAL='{host_path="/abc/tessera/tessera-dist/tessera-app/target", container_path="/tessera"}'
+  export TESSERA_APP_DEV_LOCAL='"/tessera/tessera-app-20.10.1-SNAPSHOT-app.jar"'
+  export TF_VAR_additional_tessera_container_vol="{0=[$TESSERA_DEV_LOCAL],1=[$TESSERA_DEV_LOCAL],2=[$TESSERA_DEV_LOCAL],3=[$TESSERA_DEV_LOCAL]}"
+  export TF_VAR_tessera_app_container_path="{0=$TESSERA_APP_DEV_LOCAL,1=$TESSERA_APP_DEV_LOCAL,2=$TESSERA_APP_DEV_LOCAL,3=$TESSERA_APP_DEV_LOCAL}"
+  ```
+
+### With custom GoQuorum/Tessera Docker images
+
+By default, official Docker images `quorumengineering/quorum:latest` and `quorumengineering/tessera:latest` in [Docker Hub](https://hub.docker.com/u/quorumengineering) will be used.
+If you need to use your custom images, please follow the below guides:
+
+- Name the branch with prefix `dev-`. E.g.: `dev-mybranch`
+- Push custom GoQuorum/Tessera Docker images to [Github Container Registry](https://docs.github.com/en/packages/guides/pushing-and-pulling-docker-images) of this repo with image name and version convention
+  - GoQuorum: `quorum-dev-mybranch:latest`
+  - Tessera: `tessera-dev-mybranch:latest`
+- Pushing changes to `dev-mybranch` will kick off github Action workflow running tests using custom images
 
 ### With existing `quorum-examples` network
 
@@ -108,10 +143,9 @@ E.g.: To start `networks/typical` with remote Docker infrastructure:
     mvn process-test-resources -Pauto -Dnetwork.target="networks/typical::raft" -Dinfra.target="networks/_infra/aws-ec2::us-east-1"
     ```
 
-
 ## Logging
 
-* Set environment variable: `LOGGING_LEVEL_COM_QUORUM_GAUGE=DEBUG`
+- Set environment variable: `LOGGING_LEVEL_COM_QUORUM_GAUGE=DEBUG`
 
 ------
 

networks/_modules/docker-helper/variables.tf

@@ -48,4 +48,4 @@ variable "tessera" {
     }
   }
   description = "tessera Docker container configuration"
-}
+}

networks/_modules/docker/geth.tf

@@ -42,6 +42,13 @@ resource "docker_container" "geth" {
     container_path = local.container_plugin_acctdir
     host_path      = length(local_file.plugin_acct_dir_files) != 0 ? dirname(local_file.plugin_acct_dir_files[count.index].filename) : dirname(local_file.plugin_acct_fallback_dir_files[count.index].filename)
   }
+  dynamic "volumes" {
+    for_each = lookup(var.additional_geth_container_vol, count.index, [])
+    content {
+      container_path = volumes.value["container_path"]
+      host_path      = volumes.value["host_path"]
+    }
+  }
   networks_advanced {
     name         = docker_network.quorum.name
     ipv4_address = var.geth_networking[count.index].ip.private
@@ -142,7 +149,7 @@ exec geth \
 %{endif~}
   --port ${var.geth_networking[count.index].port.p2p} \
   --ethstats "Node${count.index + 1}:${var.ethstats_secret}@${var.ethstats_ip}:${var.ethstats.container.port}" \
-  --unlock 0 \
+  --unlock ${join(",", range(var.accounts_count[count.index]))} \
   --password ${local.container_geth_datadir}/${var.password_file_name} \
   ${var.consensus == "istanbul" ? "--istanbul.blockperiod 1 --syncmode full --mine --minerthreads 1" : format("--raft --raftport %d", var.geth_networking[count.index].port.raft)} ${var.additional_geth_args} $ADDITIONAL_GETH_ARGS
 EOF

networks/_modules/docker/tessera.tf

@@ -27,6 +27,13 @@ resource "docker_container" "tessera" {
     container_path = local.container_tm_datadir_mounted
     host_path      = var.tessera_datadirs[count.index]
   }
+  dynamic "volumes" {
+    for_each = lookup(var.additional_tessera_container_vol, count.index, [])
+    content {
+      container_path = volumes.value["container_path"]
+      host_path      = volumes.value["host_path"]
+    }
+  }
   networks_advanced {
     name         = docker_network.quorum.name
     ipv4_address = var.tm_networking[count.index].ip.private
@@ -46,7 +53,8 @@ resource "docker_container" "tessera" {
     <<EOF
 #Tessera${count.index + 1}
 
-START_TESSERA="java -Xms128M -Xmx128M -jar /tessera/tessera-app.jar \
+START_TESSERA="java -Xms128M -Xmx128M \
+  -jar ${lookup(var.tessera_app_container_path, count.index, "/tessera/tessera-app.jar")} \
   --override jdbc.url=jdbc:h2:${local.container_tm_datadir}/db;MODE=Oracle;TRACE_LEVEL_SYSTEM_OUT=0 \
   --override serverConfigs[1].serverAddress=unix:${local.container_tm_ipc_file} \
   --override serverConfigs[2].sslConfig.serverKeyStore=${local.container_tm_datadir}/serverKeyStore \
@@ -85,7 +93,17 @@ if [ -f /data/tm/cleanStorage ]; then
 fi
 
 rm -f ${local.container_tm_ipc_file}
-exec $START_TESSERA
+exec java -Xms128M -Xmx128M \
+  -jar ${lookup(var.tessera_app_container_path, count.index, "/tessera/tessera-app.jar")} \
+  --override jdbc.url="jdbc:h2:${local.container_tm_datadir}/db;MODE=Oracle;TRACE_LEVEL_SYSTEM_OUT=0" \
+  --override serverConfigs[1].serverAddress="unix:${local.container_tm_ipc_file}" \
+  --override serverConfigs[2].sslConfig.serverKeyStore="${local.container_tm_datadir}/serverKeyStore" \
+  --override serverConfigs[2].sslConfig.serverTrustStore="${local.container_tm_datadir}/serverTrustStore" \
+  --override serverConfigs[2].sslConfig.knownClientsFile="${local.container_tm_datadir}/knownClientsFile" \
+  --override serverConfigs[2].sslConfig.clientKeyStore="${local.container_tm_datadir}/clientKeyStore" \
+  --override serverConfigs[2].sslConfig.clientTrustStore="${local.container_tm_datadir}/clientTrustStore" \
+  --override serverConfigs[2].sslConfig.knownServersFile="${local.container_tm_datadir}/knownServersFile" \
+  --configfile ${local.container_tm_datadir}/config.json
 EOF
   ]
 }

networks/_modules/docker/variables.tf

@@ -88,26 +88,47 @@ variable "network_id" {}
 
 variable "ethstats_secret" {}
 
-variable  "ethstats_ip" {}
+variable "ethstats_ip" {}
 
 variable "password_file_name" {}
 
 variable "network_cidr" {}
 
 variable "additional_geth_env" {
-  type = map(string)
-  default = {}
+  type        = map(string)
+  default     = {}
   description = "Additional environment variables for each `geth` node in the network, provided as a key/value map.  The correct PRIVATE_CONFIG is already set by this module, so any PRIVATE_CONFIG value provided in this map will be ignored."
 }
 
 variable "tm_env" {
-  type = map(string)
-  default = {}
+  type        = map(string)
+  default     = {}
   description = "Environment variables for each `tessera` node in the network, provided as a key/value map."
 }
 
 variable "host_plugin_account_dirs" {
   type        = list(string)
   description = "Path to dirs on host used for sharing data for account plugin between host and containers"
-  default = []
+  default     = []
+}
+
+variable "additional_geth_container_vol" {
+  type        = map(list(object({ container_path = string, host_path = string })))
+  default     = {}
+  description = "Additional volume mounts for geth container. Each map key is the node index (0-based)"
+}
+
+variable "additional_tessera_container_vol" {
+  type        = map(list(object({ container_path = string, host_path = string })))
+  default     = {}
+  description = "Additional volume mounts for tessera container. Each map key is the node index (0-based)"
+}
+
+variable "tessera_app_container_path" {
+  type        = map(string)
+  default     = {}
+  description = "Path to Tessera app jar file in the container. Each map key is the node index (0-based)"
 }
+
+variable "accounts_count" {
+}

networks/_modules/ignite/main.tf

@@ -12,7 +12,7 @@ locals {
   // by default we allocate one named key per TM: K0, K1 ... Kn
   // this can be overrriden by the variable
   tm_named_keys_alloc = merge(
-    { for id in local.node_indices : id => [format("K%d", id)] },
+    { for id in local.node_indices : id => [format("UnnamedKey%d", id)] },
     var.override_tm_named_key_allocation
   )
   // by default we allocate one named account per node

networks/_modules/ignite/outputs.tf

@@ -32,4 +32,8 @@ output "application_yml_file" {
 
 output "exclude_initial_nodes" {
   value = var.exclude_initial_nodes
+}
+
+output "accounts_count" {
+  value = { for id in local.node_indices : id => length(local.named_accounts_alloc[id]) }
 }