diff --git a/scenarios/nomad-consul-secure/workload-identity-consul/README.md b/scenarios/nomad-consul-secure/workload-identity-consul/README.md new file mode 100644 index 0000000..3c48be5 --- /dev/null +++ b/scenarios/nomad-consul-secure/workload-identity-consul/README.md @@ -0,0 +1,59 @@ +## Prerequisites + +The following tools are required to run these scenarios: + +* HashiCorp [Packer](https://developer.hashicorp.com/packer) (if you are building custom images) +* CDRtools (if you are building custom images) +* [Lima](https://lima-vm.io/) +* [Shikari](https://github.com/ranjandas/shikari) +* [Workload_Identity_Doc](https://developer.hashicorp.com/nomad/docs/concepts/workload-identity) +* [Workload_Identity_Tutorial](https://developer.hashicorp.com/nomad/tutorials/integrate-consul/consul-acl) +* Supported from [Nomad Version](https://github.com/hashicorp/nomad/blob/main/CHANGELOG.md#170-december-07-2023) +* Requires Packer build using enterprise binary for consul and nomad, update the [variables.pkvars.hcl] file. + +> NOTE: all the above except shikari can be installed using Homebrew + +## Usage + +Perform the following steps to execute Workload-Identity jobs for consul + +1. Execute the below command for shikari to spin up nomad and consul cluster + + ``` + Example: + $ shikari create -n wi -s 1 -c 1 -t ../shikari-scenarios/scenarios/nomad-consul-secure/workload-identity-consul/hashibox.yaml -e CONSUL_LICENSE=$(cat ~/workspace/consul.hclic) -e NOMAD_LICENSE=$(cat ~/workspace/nomad.hclic) + ``` + +2. Export the nomad and consul token as env var + + ``` + $ export CONSUL_HTTP_TOKEN=root + $ export NOMAD_TOKEN=00000000-0000-0000-0000-000000000000 + ``` + +3. Check if you can access the nomad and consul cluster by executing following commands. + + ``` + $ nomad server members + $ nomad node status + $ consul members + $ consul operator raft list-peers + ``` +4. Run a sample connect enabled job + + ``` + $ nomad job init -short -connect + Example job file written to example.nomad.hcl + $ nomad job run example.nomad.hcl + ``` + +5. This should create SI tokens for the services and register services in consul + +6. To destroy the setup execute the below commands + + ``` + $ shikari stop -n wi + $ shikari destroy -n wi -f + unset CONSUL_HTTP_TOKEN + unset NOMAD_TOKEN + ``` diff --git a/scenarios/nomad-consul-secure/workload-identity-consul/hashibox.yaml b/scenarios/nomad-consul-secure/workload-identity-consul/hashibox.yaml new file mode 100644 index 0000000..ad66590 --- /dev/null +++ b/scenarios/nomad-consul-secure/workload-identity-consul/hashibox.yaml @@ -0,0 +1,253 @@ +images: + # Try to use a local image first. + - location: `../../packer/.artifacts/c-enterprise-1.18-n-enterprise-1.7/c-enterprise-1.18-n-enterprise-1.7.qcow2 + +mounts: [] +containerd: + system: false + user: false +provision: + - mode: system # set license + script: | + #!/bin/bash + echo CONSUL_LICENSE=$CONSUL_LICENSE > /etc/consul.d/consul.env + echo NOMAD_LICENSE=$NOMAD_LICENSE > /etc/nomad.d/nomad.env + + - mode: system # configure Consul + script: | + #!/bin/bash + + function setConfig { + local mode=$1 + local cluster=$2 + + # Set the config files accordingly + if [[ $mode == "server" ]]; then + cat <<-EOF > /etc/consul.d/consul.hcl + data_dir = "/opt/consul/data" + log_level = "INFO" + bind_addr = {{ "\"{{ GetInterfaceIP \\\"eth0\\\"}}\"" }} + client_addr = "0.0.0.0" + retry_join = ["lima-$cluster-srv-01.internal"] + datacenter = "$cluster" + + ui_config { + enabled = true + } + + connect { + enabled = true + } + + server = true + bootstrap_expect = $BOOTSTRAP_EXPECT + + # required for transparent proxy + recursors = ["1.1.1.1", "1.1.0.0"] + addresses { + dns = {{ "\"{{ GetInterfaceIP \\\"eth0\\\"}}\"" }} + } + acl { + enabled = true + down_policy = "extend-cache" + default_policy = "deny" + enable_token_persistence = true + tokens = { + initial_management = "root" + agent = "root" + } + } + + EOF + + elif [[ $mode == "client" ]]; then + cat <<-EOF > /etc/consul.d/consul.hcl + data_dir = "/opt/consul/data" + log_level = "INFO" + bind_addr = {{ "\"{{ GetInterfaceIP \\\"eth0\\\"}}\"" }} + client_addr = "0.0.0.0" + retry_join = ["lima-$cluster-srv-01.internal"] + datacenter = "$cluster" + + connect { + enabled = true + } + + ports { + grpc = 8502 + } + + server = false + + # required for transparent proxy + recursors = ["1.1.1.1", "1.1.0.0"] + addresses { + dns = {{ "\"{{ GetInterfaceIP \\\"eth0\\\"}}\"" }} + } + acl { + enabled = true + down_policy = "extend-cache" + default_policy = "deny" + enable_token_persistence = true + tokens = { + agent = "root" + } + } + EOF + else + echo "Invalid mode for setConfig" + fi + } + + # Check if the environment variable 'MODE' is set + if [[ -n $MODE ]]; then + echo "MODE is set to: $MODE" + else + echo "MODE is not set." + fi + + # Check if the environment variable 'CLUSTER' is set + if [[ -n $CLUSTER ]]; then + echo "CLUSTER is set to: $CLUSTER" + else + echo "CLUSTER is not set." + fi + + setConfig $MODE $CLUSTER + + - mode: system # configure Nomad + script: | + #!/bin/bash + function setConfig { + local mode=$1 + local cluster=$2 + + # Set the config files accordingly + if [[ $mode == "server" ]]; then + cat <<-EOF > /etc/nomad.d/nomad.hcl + data_dir = "/opt/nomad/data" + bind_addr = "0.0.0.0" + + server { + enabled = true + bootstrap_expect = $BOOTSTRAP_EXPECT + + server_join { + retry_join = ["lima-$cluster-srv-01.internal"] + } + } + datacenter = "$cluster" + + # workload identity configs + # ACLS + acl { + enabled = true + } + + # consul config + consul { + address = "127.0.0.1:8500" + token = "root" + + service_identity { + aud = ["consul.io"] + ttl = "1h" + } + + task_identity { + aud = ["consul.io"] + ttl = "1h" + } + } + EOF + + elif [[ $mode == "client" ]]; then + cat <<-EOF > /etc/nomad.d/nomad.hcl + data_dir = "/opt/nomad/data" + bind_addr = "0.0.0.0" + datacenter = "$cluster" + + client { + enabled = true + servers = ["lima-$cluster-srv-01.internal"] + } + + # workload identity configs + # ACLS + acl { + enabled = true + } + + # consul config + consul { + address = "127.0.0.1:8500" + token = "root" + } + EOF + else + echo "Invalid mode for setConfig" + fi + } + + + # Check if the environment variable 'MODE' is set + if [[ -n $MODE ]]; then + echo "MODE is set to: $MODE" + else + echo "MODE is not set." + fi + + # Check if the environment variable 'CLUSTER' is set + if [[ -n $CLUSTER ]]; then + echo "CLUSTER is set to: $CLUSTER" + else + echo "CLUSTER is not set." + fi + + setConfig $MODE $CLUSTER + - mode: + script: | + systemctl enable --now docker + systemctl enable --now nomad consul + - mode: user + script: | + #!/bin/sh + nomad -autocomplete-install + consul -autocomplete-install + + - mode: system # acl bootstrap nomad + script: | + #!/bin/sh + if [[ $MODE == "server" ]]; then + # Wait for nomad servers to come up and bootstrap nomad ACL + for i in {1..10}; do + # add sleep 5 secs + set +e + sleep 5 + OUTPUT=$(echo "00000000-0000-0000-0000-000000000000"|nomad acl bootstrap - 2>&1) + # checks if the previous command (nomad acl bootstrap) failed (non-zero exit status). + if [ $? -ne 0 ]; then + echo "nomad acl bootstrap: $OUTPUT" + if [[ "$OUTPUT" = *"No cluster leader"* ]]; then + echo "nomad has no cluster leader" + continue + else + echo "nomad already bootstrapped" + exit 0 + fi + fi + set -e + done + fi + + - mode: system # setup workload identity + script: | + #!/bin/sh + # Set env vars for tool CLIs + export CONSUL_HTTP_TOKEN="root" + nomad setup consul -y + +networks: + - lima: user-v2 +vmType: qemu + diff --git a/scenarios/nomad-consul-secure/workload-identity-consul/variables.pkrvars.hcl b/scenarios/nomad-consul-secure/workload-identity-consul/variables.pkrvars.hcl new file mode 100644 index 0000000..59b58ae --- /dev/null +++ b/scenarios/nomad-consul-secure/workload-identity-consul/variables.pkrvars.hcl @@ -0,0 +1,2 @@ +consul_version = "enterprise-1.18" +nomad_version = "enterprise-1.7" \ No newline at end of file