Merge branch 'develop' into alert/fix-stuck-in-catchup-and-bootstrap

automation/terraform/modules/testworld-logging/README.md

@@ -0,0 +1,36 @@
+# Mina Incentivized Testnet: Log Infra Stack
+
+This Terraform code deploys the logging stack used for the `testworld-2-0` testnet (also known as `ITN3`).
+
+The initial version of this deployment uses a static virtual machine running Docker compose to deploy the following containers:
+
+- postgres database
+- logging front-end (GUI)
+- logging backend (Log Consumer)
+
+## Hardware Requirements
+
+The most resource heavy portion of the deployment is the logging backend container. Sizing this container is determined by how much log traffic needs to be consumed. For an initial use, paired to the `testworld-2-0` testnet. The VM that hold the Docker Compose deployment is sized at `64vCPU` and `128Gi` RAM.
+
+## VM configuration with Terraform templates
+
+This deployment relies on Terraform templates to configure the final state on top of a VM running a vanilla Debian OS image. This is done because at the time of writing, Google Cloud does not offer a machine image with Docker preinstalled. A custom machine image can be created using a tool such as _Packer_ from Hashicorp, but Terraform templates have been chosen in this case to limit the number of tools and build steps in the deployment flow.
+
+> [!NOTE]
+> More information about using Terraform templates can be found on the [Terraform website](https://registry.terraform.io/providers/hashicorp/template/latest/docs).
+
+Additional configuration can be layered on top of the VM OS by adding a new template to the `./templates` directory, declaring it as a `data` source within `vars.tf` file, and finally adding it to the `metadata` section of the VM configuration within the `main.tf` file.
+
+## Handling Secrets
+
+This deployment uses Google Secrets Manager to handle secrets. Secret values are not stored within the source code. If secrets are modified in Google Secrets Manager, note that the new values due not sync automatically and that a redeploy may be required to pull in the new values.
+
+## Terraform Outputs
+
+After deployment, the `output.tf` file is configured to print the public IP address that is assigned to the deployed virtual machine. This IP can be used to `ssh` to the machine.
+
+```
+Outputs:
+
+docker_vm_ip = "35.35.35.35" <--- example IP
+```

automation/terraform/modules/testworld-logging/main.tf

@@ -0,0 +1,112 @@
+terraform {
+  backend "gcs" {
+    bucket = "o1labs-terraform"
+    prefix = "itn3-logging"
+  }
+}
+
+provider "google" {
+  project = var.gcp_project
+  region  = var.gcp_region
+}
+
+#####################################
+# Google Cloud Secrets Imports
+#####################################
+
+data "google_secret_manager_secret_version" "itn_secret_key" {
+  provider = google
+  secret   = var.itn_secret_key
+}
+
+data "google_secret_manager_secret_version" "itn_db_pass" {
+  provider = google
+  secret   = var.db_pass
+}
+
+data "google_secret_manager_secret_version" "aws_access_id" {
+  provider = google
+  secret   = var.aws_id
+}
+
+data "google_secret_manager_secret_version" "aws_access_key" {
+  provider = google
+  secret   = var.aws_key
+}
+
+#####################################
+# Docker Compose VM Configuration
+#####################################
+
+resource "random_id" "instance_id" {
+  byte_length = 4
+}
+
+resource "google_compute_instance" "default" {
+  name         = "itn-logging-${random_id.instance_id.hex}"
+  machine_type = "n2-standard-32" # 32vCPU, 128GB RAM
+  zone         = var.gcp_zone
+
+  boot_disk {
+    initialize_params {
+      image = "debian-cloud/debian-11"
+      size  = 500 # GB
+    }
+  }
+
+  metadata = {
+    startup-script = <<SCRIPT
+  ${data.template_file.fe-config.rendered}
+  ${data.template_file.keys.rendered}
+  ${data.template_file.names-data.rendered}
+  ${data.template_file.postgres.rendered}
+  ${data.template_file.docker-script-build.rendered}
+  ${data.template_file.docker-compose-build.rendered}
+  ${data.template_file.execute-shell.rendered}
+  SCRIPT
+  }
+
+  network_interface {
+    network = "default"
+
+    access_config {
+      # do not remove
+      # empty block required for ephemeral public IP
+    }
+  }
+
+  labels = {
+    service = var.billing_label
+  }
+
+  # depends_on = [google_sql_user.database_user]
+}
+
+#####################################
+# Cloud Postgres Configuration
+#####################################
+
+# # Create a Google Cloud SQL PostgreSQL instance
+# resource "google_sql_database_instance" "postgres_instance" {
+#   name             = "my-postgres-instance"
+#   database_version = "POSTGRES_14"
+#   project          = var.gcp_project
+#   region           = var.gcp_region
+#   settings {
+#     tier = "db-custom-1-3840"
+
+#     # Set database flags
+#     database_flags {
+#       name  = "max_connections"
+#       value = "10000"
+#     }
+#   }
+#   deletion_protection = false
+# }
+
+# # Define the database user
+# resource "google_sql_user" "database_user" {
+#   name     = "my-db-user"
+#   instance = google_sql_database_instance.postgres_instance.name
+#   password = "your-password" # Change this to your desired password
+# }

automation/terraform/modules/testworld-logging/output.tf

@@ -0,0 +1,7 @@
+output "docker_vm_ip" {
+  value = google_compute_instance.default.network_interface.0.access_config.0.nat_ip
+}
+
+# output "cloud_postgres_ip" {
+#   value = google_sql_database_instance.postgres_instance.ip_address
+# }

automation/terraform/modules/testworld-logging/templates/docker-compose-build.tpl

@@ -0,0 +1,67 @@
+cat > /root/docker-compose.yml <<- "SCRIPT"
+
+version: '3'
+services:
+  internal-log-fetcher:
+    image: gcr.io/o1labs-192920/mina-internal-trace-consumer:1.2.6 # openmina/mina-internal-trace-consumer:2d3bc20
+    # image: local/mina-internal-trace-consumer
+    container_name: internal-log-fetcher
+    restart: always
+    command: "fetcher -k /keys/secret_key -o /output --db-uri 'postgresql://postgres:${password_value}@postgres:5432' discovery"
+    ports:
+      - 4000:4000
+      - 11000-11700:11000-11700
+    volumes:
+      - ./keys:/keys
+      - ./output:/output
+    environment:
+      NETWORK: ITN
+      INTERNAL_TRACE_CONSUMER_EXE: /internal_trace_consumer
+      AWS_ACCESS_KEY_ID: "${aws_id_value}"
+      AWS_SECRET_ACCESS_KEY: "${aws_key_value}"
+      AWS_DEFAULT_REGION: us-west-2
+      AWS_BUCKET: 673156464838-block-producers-uptime
+      AWS_PREFIX: berkeley
+    networks:
+      - internal-log-fetcher-network
+
+  frontend:
+    image: directcuteo/mina-frontend:663f692
+    container_name: frontend
+    restart: always
+    ports:
+      - 80:80
+    command:
+      - sh
+      - -ce
+      - |
+        ENV=$(cat /fe-config.json | tr -d '\n' | tr -s ' ' | sed -e 's/ //g') envsubst < /usr/share/nginx/html/assets/env.template.js > /usr/share/nginx/html/assets/env.js
+        exec nginx -g 'daemon off;'
+    volumes:
+      - ./fe-config.json:/fe-config.json
+    networks:
+      - internal-log-fetcher-network
+
+  postgres:
+    image: postgres
+    shm_size: 1g
+    container_name: postgres
+    restart: always
+    ports:
+      - 5455:5432
+    command: "-c max_connections=10000 -c shared_buffers=2048MB"
+    volumes:
+      - ./postgresql:/var/lib/postgresql/data
+    environment:
+      PGDATA: /var/lib/postgresql/data/pgdata
+      POSTGRES_PASSWORD: ${password_value}
+    networks:
+      - internal-log-fetcher-network
+
+networks: #use same network across containers to simplify communication between containers
+  internal-log-fetcher-network:
+    #driver: bridge
+    external: # network created previously by 'docker network create internal-log-fetcher-network' command
+      name: internal-log-fetcher-network
+
+SCRIPT

automation/terraform/modules/testworld-logging/templates/docker-mounts/fe-config.tpl

@@ -0,0 +1,18 @@
+cat > /root/fe-config.json <<- "SCRIPT"
+{
+	"production": true,
+	"aggregator": "/aggregator",
+	"isVanilla": true,
+	"nodeLister": {
+		"domain": "http://localhost",
+		"port": 4000
+	},
+	"globalConfig": {
+		"features": {
+			"dashboard": ["nodes"],
+			"tracing": ["overview", "blocks"]
+		}
+	},
+	"configs": []
+}
+SCRIPT

automation/terraform/modules/testworld-logging/templates/docker-mounts/keys.tpl

@@ -0,0 +1,6 @@
+cat > /root/secret_key.sh <<- "SCRIPT"
+#!/bin/bash
+mkdir -p /root/keys/
+touch /root/keys/secret_key
+echo "${key_value}" > /root/keys/secret_key
+SCRIPT

automation/terraform/modules/testworld-logging/templates/docker-mounts/names-data.tpl

@@ -0,0 +1,6 @@
+cat > /root/node_names.sh <<- "SCRIPT"
+#!/bin/bash
+mkdir -p /root/names-data/
+touch /root/names-data/node_names.json
+echo "{}" > /root/names-data/node_names.json
+SCRIPT

automation/terraform/modules/testworld-logging/templates/docker-mounts/postgres.tpl

@@ -0,0 +1,4 @@
+cat > /root/postgres_dir.sh <<- "SCRIPT"
+#!/bin/bash
+mkdir -p /root/postgresql/
+SCRIPT

automation/terraform/modules/testworld-logging/templates/docker-script-build.tpl

@@ -0,0 +1,11 @@
+cat > /root/docker-install.sh <<- "SCRIPT"
+#!/bin/bash
+
+##################Install docker#################
+curl -sSL https://get.docker.com | sh
+
+#################Install Docker Compose##########
+sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+
+SCRIPT

automation/terraform/modules/testworld-logging/templates/execute-shell.tpl

@@ -0,0 +1,29 @@
+#############Execute docker install script#############
+sudo apt-get update
+chmod 700 /root/docker-install.sh
+/root/docker-install.sh >> /root/docker-install.log
+mv /root/docker-install.sh /root/docker-install.sh.EXECUTED #prevent it from running again
+
+#############Install Docker network#############
+#create a network most containers will use
+docker network create internal-log-fetcher-network >> /root/internal-log-fetcher-network.log
+docker network ls >> /root/internal-log-fetcher-network.log
+
+#############Execute key install script#############
+chmod 700 /root/secret_key.sh
+/root/secret_key.sh >> secret_key-install.log
+mv /root/secret_key.sh /root/secret_key.sh.EXECUTED
+
+#############Execute node names script#############
+chmod 700 /root/node_names.sh
+/root/node_names.sh >> node_names-install.log
+mv /root/node_names.sh /root/node_names.sh.EXECUTED
+
+#############Create postgres dir#############
+chmod 700 /root/postgres_dir.sh
+/root/postgres_dir.sh >> postgres_dir-install.log
+mv /root/postgres_dir.sh /root/postgres_dir.sh.EXECUTED
+
+#############Bring up docker containers############
+mv /root/keys/secret_key.txt /root/keys/secret_key
+docker-compose -f /root/docker-compose.yml up -d 

automation/terraform/modules/testworld-logging/vars.tf

@@ -0,0 +1,76 @@
+variable "gcp_project" {
+  default = "o1labs-192920"
+}
+
+variable "gcp_region" {
+  default = "us-east4"
+}
+
+variable "gcp_zone" {
+  default = "us-east4-b"
+}
+
+variable "billing_label" {
+  default = "itn3"
+}
+
+#####################################
+# Secret Vars
+#####################################
+
+variable "itn_secret_key" {
+  default = "itn_secret_key" # name of secret in Google Cloud
+}
+
+variable "db_pass" {
+  default = "itn-db-pass"
+}
+
+variable "aws_id" {
+  default = "itn-aws-id"
+}
+
+variable "aws_key" {
+  default = "itn-aws-key"
+}
+
+#####################################
+# Passing Secrets To Templates
+#####################################
+
+
+data "template_file" "docker-script-build" {
+  template = file("templates/docker-script-build.tpl")
+}
+
+data "template_file" "docker-compose-build" {
+  template = file("templates/docker-compose-build.tpl")
+  vars = {
+    password_value = data.google_secret_manager_secret_version.itn_db_pass.secret_data
+    aws_id_value   = data.google_secret_manager_secret_version.aws_access_id.secret_data
+    aws_key_value  = data.google_secret_manager_secret_version.aws_access_key.secret_data
+  }
+}
+
+data "template_file" "execute-shell" {
+  template = file("templates/execute-shell.tpl")
+}
+
+data "template_file" "fe-config" {
+  template = file("templates/docker-mounts/fe-config.tpl")
+}
+
+data "template_file" "keys" {
+  template = file("templates/docker-mounts/keys.tpl")
+  vars = {
+    key_value = data.google_secret_manager_secret_version.itn_secret_key.secret_data
+  }
+}
+
+data "template_file" "names-data" {
+  template = file("templates/docker-mounts/names-data.tpl")
+}
+
+data "template_file" "postgres" {
+  template = file("templates/docker-mounts/postgres.tpl")
+}

automation/terraform/testnets/berkeley/main.tf

@@ -139,7 +139,7 @@ module "berkeley" {
 
   snark_coordinators = [
     {
-      snark_coordinator_name       = "snark-coordinator"
+      snark_coordinator_name       = local.testnet_name
       snark_worker_replicas        = 5
       snark_worker_fee             = "0.01"
       snark_worker_public_key      = "B62qmQsEHcsPUs5xdtHKjEmWqqhUPRSF2GNmdguqnNvpEZpKftPC69e"