diff --git a/cli/src/commands/test-validator/index.ts b/cli/src/commands/test-validator/index.ts index fa9430563..042d52c73 100644 --- a/cli/src/commands/test-validator/index.ts +++ b/cli/src/commands/test-validator/index.ts @@ -77,7 +77,7 @@ class SetupCommand extends Command { }), stop: Flags.boolean({ description: - "Stops the test validator and dependent processes. Use with --skip-indexer, --skip-prover, --skip-forester to keep specific services running.", + "Stops the test validator and dependent processes. Use with --skip-indexer, --skip-prover to keep specific services running.", required: false, default: false, }), diff --git a/forester/alert_rules.yml b/forester/alert_rules.yml deleted file mode 100644 index e003fab1e..000000000 --- a/forester/alert_rules.yml +++ /dev/null @@ -1,27 +0,0 @@ -groups: - - name: forester - rules: - - alert: ForesterNotRunning - expr: absent(forester_last_run_timestamp) or (time() - forester_last_run_timestamp) > 600 - for: 5m - labels: - severity: critical - annotations: - summary: "Forester is not running" - description: "Forester has not pushed metrics for more than 10 minutes." - - alert: ForesterQueueNearlyFull - expr: (queue_length / 28807) * 100 > 40 - for: 5m - labels: - severity: critical - annotations: - summary: "Forester queue is nearly full" - description: 'Queue {{ $labels.tree_type }} ({{ $labels.tree_pubkey }}) is {{ $value | printf "%.2f" }}% full.' - - alert: ForesterLowSolBalance - expr: forester_sol_balance < 0.1 - for: 5m - labels: - severity: critical - annotations: - summary: "Forester SOL balance is low" - description: 'Forester ({{ $labels.pubkey }}) SOL balance is {{ $value | printf "%.2f" }} SOL, which is below 0.5 SOL.' \ No newline at end of file diff --git a/forester/alertmanager.yml b/forester/alertmanager.yml deleted file mode 100644 index 43990c0ac..000000000 --- a/forester/alertmanager.yml +++ /dev/null @@ -1,40 +0,0 @@ -global: - resolve_timeout: 5m - pagerduty_url: 'https://events.pagerduty.com/v2/enqueue' - -route: - group_by: ['alertname'] - group_wait: 30s - group_interval: 5m - repeat_interval: 5m - receiver: 'pagerduty-notifications' - routes: - - match: - alertname: ForesterQueueNearlyFull - receiver: 'pagerduty-notifications' - group_wait: 5s - repeat_interval: 3m - -receivers: - - name: 'pagerduty-notifications' - pagerduty_configs: - - routing_key: '' - send_resolved: true - client: 'Alertmanager' - client_url: 'http://alertmanager:9093' - description: '{{ .GroupLabels.alertname }}: {{ .CommonAnnotations.summary }}' - severity: '{{ if eq .GroupLabels.severity "critical" }}critical{{ else }}warning{{ end }}' - class: '{{ .GroupLabels.alertname }}' - component: '{{ .GroupLabels.instance }}' - group: '{{ .GroupLabels.job }}' - details: - firing: '{{ .Alerts.Firing | len }}' - resolved: '{{ .Alerts.Resolved | len }}' - all_alerts: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}' - -inhibit_rules: - - source_match: - severity: 'critical' - target_match: - severity: 'warning' - equal: ['alertname', 'dev', 'instance'] \ No newline at end of file diff --git a/forester/docker-compose.yml b/forester/docker-compose.yml deleted file mode 100644 index d26922220..000000000 --- a/forester/docker-compose.yml +++ /dev/null @@ -1,90 +0,0 @@ -services: - db: - image: postgres:16-alpine - restart: always - environment: - - POSTGRES_USER=photon - - POSTGRES_DB=photon - - POSTGRES_PASSWORD=photon - ports: - - 5432:5432 - volumes: - - db_data:/var/lib/postgresql/data - pgadmin: - image: dpage/pgadmin4 - restart: always - ports: - - "8888:80" - environment: - - PGADMIN_DEFAULT_EMAIL=photon@lightprotocol.com - - PGADMIN_DEFAULT_PASSWORD=photon - volumes: - - pgadmin-data:/var/lib/pgadmin - forester: - build: - context: .. - dockerfile: ./forester/Dockerfile - volumes: - - ./forester.toml:/app/config/forester.toml - env_file: - - ./.env - networks: - - forester-net - command: ["status"] - - pushgateway: - image: prom/pushgateway - ports: - - "9092:9091" - networks: - - forester-net - - alertmanager: - image: prom/alertmanager - ports: - - "9093:9093" - volumes: - - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml - command: - - '--config.file=/etc/alertmanager/alertmanager.yml' - - '--storage.path=/alertmanager' - - '--log.level=debug' - environment: - - PAGERDUTY_SERVICE_KEY=${PAGERDUTY_SERVICE_KEY} - networks: - - forester-net - - prometheus: - image: prom/prometheus - ports: - - "9090:9090" - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/usr/share/prometheus/console_libraries' - - '--web.console.templates=/usr/share/prometheus/consoles' - - '--log.level=debug' - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - - ./alert_rules.yml:/etc/prometheus/alert_rules.yml - networks: - - forester-net - - grafana: - image: grafana/grafana - ports: - - "3000:3000" - environment: - - GF_SECURITY_ADMIN_PASSWORD=admin - volumes: - - grafana-storage:/var/lib/grafana - networks: - - forester-net - -networks: - forester-net: - -volumes: - grafana-storage: - db_data: - pgadmin-data: diff --git a/forester/forester.toml b/forester/forester.toml deleted file mode 100644 index 1835f5f70..000000000 --- a/forester/forester.toml +++ /dev/null @@ -1,15 +0,0 @@ -CU_LIMIT=1_000_000 -RPC_POOL_SIZE=20 -MAX_RETRIES=5 -RETRY_DELAY=1000 -RETRY_TIMEOUT=120_000 -INDEXER_BATCH_SIZE=50 -INDEXER_MAX_CONCURRENT_BATCHES=50 -TRANSACTION_BATCH_SIZE=1 -TRANSACTION_MAX_CONCURRENT_BATCHES=50 -SLOT_UPDATE_INTERVAL_SECONDS=30 -TREE_DISCOVERY_INTERVAL_SECONDS=5 -STATE_QUEUE_START_INDEX=0 -STATE_QUEUE_LENGTH=28807 -ADDRESS_QUEUE_START_INDEX=0 -ADDRESS_QUEUE_LENGTH=28807 \ No newline at end of file diff --git a/forester/package.json b/forester/package.json index 432bd1e10..30aa32f7f 100644 --- a/forester/package.json +++ b/forester/package.json @@ -3,28 +3,12 @@ "version": "0.3.0", "license": "GPL-3.0", "scripts": { - "restart-db": "docker compose down -v && docker compose up -d db", - "migrate-db": "../.local/cargo/bin/photon-migration --database-url postgres://photon:photon@localhost:5432/postgres", - "wait-for-db": "./scripts/wait_for_db.sh", - "start-validator": "../cli/test_bin/run test-validator --indexer-db-url=postgres://photon:photon@localhost:5432/postgres --skip-forester", - "restart-validator": "pnpm restart-db && pnpm wait-for-db && pnpm migrate-db && pnpm start-validator", "build": "cargo build", - "test": "cargo test --package forester -- --test-threads=1 --nocapture", - "test-lint": "RUSTFLAGS=\"--cfg tokio_unstable -D warnings\" cargo test --package forester -- --test-threads=1 --nocapture", - "generate": "ts-node ./scripts/generate.ts", - "docker:build": "docker build --tag forester -f Dockerfile ..", - "docker:down": "docker compose down", - "docker:up": "docker compose up -d --build", - "docker:logs": "docker compose logs -f" + "test": "RUSTFLAGS=\"--cfg tokio_unstable -D warnings\" cargo test --package forester -- --test-threads=1 --nocapture", + "docker:build": "docker build --tag forester -f Dockerfile .." }, "devDependencies": { - "@lightprotocol/stateless.js": "workspace:*", - "@lightprotocol/zk-compression-cli": "workspace:*", - "@solana/web3.js": "^1.95.0", - "borsh": "^2.0.0", - "dotenv": "^16.4.5", - "ts-node": "^10.9.2", - "tweetnacl": "^1.0.3" + "@lightprotocol/zk-compression-cli": "workspace:*" }, "nx": { "targets": { diff --git a/forester/prometheus.yml b/forester/prometheus.yml deleted file mode 100644 index e70b6fead..000000000 --- a/forester/prometheus.yml +++ /dev/null @@ -1,17 +0,0 @@ -global: - scrape_interval: 15s - -alerting: - alertmanagers: - - static_configs: - - targets: - - 'alertmanager:9093' - -rule_files: - - 'alert_rules.yml' - -scrape_configs: - - job_name: 'pushgateway' - honor_labels: true - static_configs: - - targets: ['pushgateway:9091'] \ No newline at end of file diff --git a/forester/scripts/generate.ts b/forester/scripts/generate.ts deleted file mode 100644 index 757788189..000000000 --- a/forester/scripts/generate.ts +++ /dev/null @@ -1,149 +0,0 @@ -import {PublicKey, Signer, Keypair, LAMPORTS_PER_SOL} from '@solana/web3.js'; -import { - airdropSol, - createRpc, - compress, - transfer, - Rpc, - LightSystemProgram, createAccount -} from '@lightprotocol/stateless.js'; -import { randomBytes } from 'tweetnacl'; -import * as dotenv from "dotenv"; -dotenv.config(); - -const RPC_API_KEY = process.env.RPC_API_KEY; -console.log("RPC_API_KEY: ", RPC_API_KEY); - -const LAMPORTS = 1.1 * LAMPORTS_PER_SOL; -const COMPRESS_AMOUNT = 1 * LAMPORTS_PER_SOL; -const TOTAL_TX = 1000; -const CONCURRENT_TX = 20; -const TRANSFER_AMOUNT = 10; - -const aliceKeypair = [ - 46, 239, 29, 58, 196, 181, 39, 77, 196, 54, 249, 108, 80, 144, 32, 168, 245, - 161, 146, 92, 180, 79, 231, 37, 50, 88, 220, 48, 9, 146, 249, 82, 130, 60, - 106, 251, 24, 224, 192, 108, 70, 59, 111, 251, 186, 50, 23, 103, 106, 233, - 113, 148, 57, 190, 158, 111, 163, 28, 157, 47, 201, 41, 249, 59, -]; - -const bobKeypair = [ - 125, 14, 244, 185, 193, 42, 156, 191, 212, 42, 239, 56, 169, 240, 239, 52, 95, - 215, 240, 86, 151, 212, 245, 230, 198, 148, 12, 230, 83, 57, 56, 244, 191, - 129, 151, 233, 233, 129, 21, 255, 101, 163, 48, 212, 218, 82, 134, 36, 29, - 185, 30, 215, 183, 242, 244, 222, 8, 10, 158, 214, 99, 237, 126, 9, -]; - - -function generateKeypairs(count: number): Keypair[] { - const keypairs = []; - for (let i = 0; i < count; i++) { - keypairs.push(Keypair.generate()); - } - return keypairs; -} - -const payerKeypairs = generateKeypairs(CONCURRENT_TX); //[Keypair.fromSecretKey(Uint8Array.from(aliceKeypair))]; // -const receiverKeypairs = generateKeypairs(CONCURRENT_TX); //[Keypair.fromSecretKey(Uint8Array.from(bobKeypair))]; - -async function transferAsync(i: number, rpc: Rpc, payer: Signer, receiverPublicKey: PublicKey): Promise { - const transferSig = await transfer(rpc, payer, TRANSFER_AMOUNT, payer, receiverPublicKey); - console.log(`transfer ${i} of ${TOTAL_TX}: ${transferSig}`); -} - -async function createAccountAsync(i: number, rpc: Rpc, payer: Signer): Promise { - const seed = new Uint8Array(randomBytes(32)); - const tx = await createAccount( - rpc, - payer, - seed, - // TRANSFER_AMOUNT, - LightSystemProgram.programId, - ); - console.log(`create account ${i} of ${TOTAL_TX}: ${tx}`); -} - -function localRpc(): Rpc { - let validatorUrl = 'http://0.0.0.0:8899'; - let photonUrl = 'http://0.0.0.0:8784'; - let proverUrl = 'http://0.0.0.0:3001'; - return createRpc(validatorUrl, photonUrl, proverUrl); -} - -function devnetRpc(): Rpc { - const url = `https://devnet.helius-rpc.com?api-key=${RPC_API_KEY}`; - return createRpc(url, url); -} - -async function prefillNullifierQueue() { - const rpc = localRpc(); - - const isAirdropNeeded = true; - if (isAirdropNeeded) { - await Promise.all([ - ...payerKeypairs.map(async payer => await airdropSol({ - connection: rpc, - lamports: LAMPORTS, - recipientPublicKey: payer.publicKey - })), - ...receiverKeypairs.map(async receiver => await airdropSol({ - connection: rpc, - lamports: LAMPORTS, - recipientPublicKey: receiver.publicKey - })) - ]); - } - - await Promise.all( - payerKeypairs.map(async (payer) => { - const balance = await rpc.getBalance(payer.publicKey); - console.log(`Payer ${payer.publicKey.toBase58()} balance:`, balance); - }) - ); - - const isCompressNeeded = true; - if (isCompressNeeded) { - await Promise.all( - payerKeypairs.map(async (payer) => { - const compressSig = await compress(rpc, payer, COMPRESS_AMOUNT, payer.publicKey); - console.log(`Compress tx sig for payer ${payer.publicKey.toBase58()}:`, compressSig); - }) - ); - } - - await Promise.all( - payerKeypairs.map(async (payer) => { - const balance = await rpc.getCompressedBalanceByOwner(payer.publicKey); - console.log(`Payer ${payer.publicKey.toBase58()} compressed balance:`, balance); - }) - ); - - const isTransferNeeded = false; - if (isTransferNeeded) { - for (let i = 0; i < TOTAL_TX; i += CONCURRENT_TX) { - const promises = []; - for (let j = 0; j < CONCURRENT_TX; j++) { - promises.push(transferAsync(i + j, rpc, payerKeypairs[j], receiverKeypairs[j].publicKey)); - } - await Promise.all(promises); - } - } - - const isCreateAccountNeeded = true; - if (isCreateAccountNeeded) { - for (let i = 0; i < TOTAL_TX; i += CONCURRENT_TX) { - const promises = []; - for (let j = 0; j < CONCURRENT_TX; j++) { - promises.push(createAccountAsync(i + j, rpc, payerKeypairs[j])); - } - await Promise.all(promises); - } - } - -} - -prefillNullifierQueue().then(() => { - console.log('Transfer completed.'); -}).catch((error) => { - console.error('An error occurred:', error); -}); diff --git a/forester/scripts/spawn.sh b/forester/scripts/spawn.sh deleted file mode 100755 index 4db7b675d..000000000 --- a/forester/scripts/spawn.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# The program to run -PROGRAM="cargo run -- nullify-addresses" - -# Number of instances to spawn -NUM_INSTANCES=2 - -# Start time -start_time=$(date +%s.%N) - -# Spawn instances and store PIDs -pids=() -for i in $(seq 1 $NUM_INSTANCES); do - $PROGRAM & - pids+=($!) -done - -# Wait for all instances to finish -for pid in "${pids[@]}"; do - wait $pid -done - -# End time -end_time=$(date +%s.%N) - -# Calculate and print execution time -execution_time=$(echo "$end_time - $start_time" | bc) -printf "Total execution time: %.3f seconds\n" "$execution_time" \ No newline at end of file diff --git a/forester/scripts/wait_for_db.sh b/forester/scripts/wait_for_db.sh deleted file mode 100755 index 1a320672a..000000000 --- a/forester/scripts/wait_for_db.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -e - -until docker compose exec db pg_isready -h localhost -p 5432; do - >&2 echo "Postgres is unavailable - sleeping" - sleep 1 -done - ->&2 echo "Postgres is up" \ No newline at end of file diff --git a/forester/src/utils.rs b/forester/src/utils.rs index 0f77604af..faa0ffd24 100644 --- a/forester/src/utils.rs +++ b/forester/src/utils.rs @@ -11,7 +11,6 @@ pub struct LightValidatorConfig { pub path: String, pub enable_indexer: bool, pub enable_prover: bool, - pub enable_forester: bool, pub wait_time: u64, } @@ -21,7 +20,6 @@ impl Default for LightValidatorConfig { path: "../cli/test_bin/run test-validator".to_string(), enable_indexer: false, enable_prover: false, - enable_forester: false, wait_time: 35, } } @@ -37,10 +35,6 @@ pub async fn spawn_validator(config: LightValidatorConfig) { if !config.enable_prover { path.push_str(" --skip-prover"); } - if !config.enable_forester { - path.push_str(" --skip-forester"); - } - debug!("Starting validator with command: {}", path); Command::new("sh") diff --git a/forester/tests/e2e_test.rs b/forester/tests/e2e_test.rs index 28b1e1e76..2e0190eca 100644 --- a/forester/tests/e2e_test.rs +++ b/forester/tests/e2e_test.rs @@ -33,7 +33,6 @@ async fn test_epoch_monitor_with_test_indexer_and_1_forester() { init(Some(LightValidatorConfig { enable_indexer: false, enable_prover: true, - enable_forester: false, wait_time: 10, ..LightValidatorConfig::default() })) @@ -246,7 +245,6 @@ async fn test_epoch_monitor_with_2_foresters() { init(Some(LightValidatorConfig { enable_indexer: false, enable_prover: true, - enable_forester: false, wait_time: 10, ..LightValidatorConfig::default() })) @@ -549,7 +547,6 @@ async fn test_epoch_double_registration() { init(Some(LightValidatorConfig { enable_indexer: false, enable_prover: true, - enable_forester: false, wait_time: 10, ..LightValidatorConfig::default() })) diff --git a/forester/tests/test_utils.rs b/forester/tests/test_utils.rs index eb72ceae1..88109a5da 100644 --- a/forester/tests/test_utils.rs +++ b/forester/tests/test_utils.rs @@ -28,7 +28,6 @@ pub async fn spawn_test_validator(config: Option) { let config = LightValidatorConfig { enable_indexer: false, enable_prover: false, - enable_forester: false, ..LightValidatorConfig::default() }; spawn_validator(config).await; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 523a571d7..dff5dc3d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -322,27 +322,9 @@ importers: forester: devDependencies: - '@lightprotocol/stateless.js': - specifier: workspace:* - version: link:../js/stateless.js '@lightprotocol/zk-compression-cli': specifier: workspace:* version: link:../cli - '@solana/web3.js': - specifier: ^1.95.0 - version: 1.95.0(bufferutil@4.0.8)(utf-8-validate@5.0.10) - borsh: - specifier: ^2.0.0 - version: 2.0.0 - dotenv: - specifier: ^16.4.5 - version: 16.4.5 - ts-node: - specifier: ^10.9.2 - version: 10.9.2(@types/node@22.4.1)(typescript@5.5.4) - tweetnacl: - specifier: ^1.0.3 - version: 1.0.3 hasher.rs: devDependencies: @@ -413,6 +395,10 @@ importers: specifier: ^1.6.0 version: 1.6.0(@types/node@22.4.1)(@vitest/browser@1.6.0)(terser@5.31.0) + hasher.rs/src/main/wasm: {} + + hasher.rs/src/main/wasm-simd: {} + js/compressed-token: dependencies: '@coral-xyz/anchor': @@ -3690,9 +3676,6 @@ packages: borsh@0.7.0: resolution: {integrity: sha512-CLCsZGIBCFnPtkNnieW/a8wmreDmfUtjU2m9yHrzPXIlNbqVs0AQrSatSG6vdNYUqdc83tkQi2eHfF98ubzQLA==} - borsh@2.0.0: - resolution: {integrity: sha512-kc9+BgR3zz9+cjbwM8ODoUB4fs3X3I5A/HtX7LZKxCLaMrEeDFoBpnhZY//DTS1VZBSs6S5v46RZRbZjRFspEg==} - bowser@2.11.0: resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==} @@ -11880,8 +11863,6 @@ snapshots: bs58: 4.0.1 text-encoding-utf-8: 1.0.2 - borsh@2.0.0: {} - bowser@2.11.0: {} brace-expansion@1.1.11: