diff --git a/README.md b/README.md index 459a5288..c284c617 100644 --- a/README.md +++ b/README.md @@ -247,13 +247,17 @@ services: POSTGRES_DATA_PATH= docker-compose up ``` -# Boot from a snapshot +# Snapshots +Snapshots are a quicker way to sync to tip and get started. + +See [Snapshots Docs](docs/snapshots_docs.md) for instructions on creating and restoring snapshots -* Mainnet (not yet available) -* Testnet ([2024-11-22](https://eigenlayer-sidecar.s3.us-east-1.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky-20241122.tar.gz)) +### Snapshot Sources +* Mainnet Ethereum (not yet available) +* Testnet Holesky ([2024-11-22](https://eigenlayer-sidecar.s3.us-east-1.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky-20241122.tar.gz)) -## Example boot from testnet snapshot +### Example boot from testnet snapshot (default schema) ```bash curl -LO https://eigenlayer-sidecar.s3.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky-20241122.tar.gz @@ -269,95 +273,6 @@ tar -xvf sidecar-testnet-2024-11-22.tar.gz --database.schema_name=public ``` -# Snapshots - -### `create-snapshot` -```bash -go run main.go create-snapshot --help -Create a snapshot of the database. - -Usage: - sidecar create-snapshot [flags] - -Flags: - -h, --help help for create-snapshot - --output_file string Path to save the snapshot file to (required) - -Global Flags: - -c, --chain string The chain to use (mainnet, holesky, preprod (default "mainnet") - --database.db_name string PostgreSQL database name (default "sidecar") - --database.host string PostgreSQL host (default "localhost") - --database.password string PostgreSQL password - --database.port int PostgreSQL port (default 5432) - --database.schema_name string PostgreSQL schema name (default "public") - --database.user string PostgreSQL username (default "sidecar") - --datadog.statsd.enabled e.g. "true" or "false" - --datadog.statsd.url string e.g. "localhost:8125" - --debug "true" or "false" - --ethereum.chunked_batch_call_size int The number of calls to make in parallel when using the chunked batch call method (default 10) - --ethereum.contract_call_batch_size int The number of contract calls to batch together when fetching data from the Ethereum node (default 25) - --ethereum.native_batch_call_size int The number of calls to batch together when using the native eth_call method (default 500) - --ethereum.rpc-url string e.g. "http://:8545" - --ethereum.use_native_batch_call Use the native eth_call method for batch calls (default true) - --prometheus.enabled e.g. "true" or "false" - --prometheus.port int The port to run the prometheus server on (default 2112) - --rewards.generate_staker_operators_table Generate staker operators table while indexing - --rewards.validate_rewards_root Validate rewards roots while indexing (default true) - --rpc.grpc-port int gRPC port (default 7100) - --rpc.http-port int http rpc port (default 7101) -``` - -#### Example use: -``` -go run main.go create-snapshot \ - --database.host=localhost \ - --database.user=sidecar \ - --database.password=sidecar \ - --database.port=5432 \ - --database.db_name=sidecar \ - --database.schema_name=public \ - --database.create_snapshot_output=example.dump -``` - - - -### `restore-snapshot` -```bash -go run main.go restore-snapshot --help -Restore the database from a previously created snapshot file. -The snapshot file is expected to be a pg_dump custom format file. - -Usage: - sidecar restore-snapshot [flags] - -Flags: - -h, --help help for restore-snapshot - --input_file string Path to the snapshot file (required) - -Global Flags: - -c, --chain string The chain to use (mainnet, holesky, preprod (default "mainnet") - --database.db_name string PostgreSQL database name (default "sidecar") - --database.host string PostgreSQL host (default "localhost") - --database.password string PostgreSQL password - --database.port int PostgreSQL port (default 5432) - --database.schema_name string PostgreSQL schema name (default "public") - --database.user string PostgreSQL username (default "sidecar") - --datadog.statsd.enabled e.g. "true" or "false" - --datadog.statsd.url string e.g. "localhost:8125" - --debug "true" or "false" - --ethereum.chunked_batch_call_size int The number of calls to make in parallel when using the chunked batch call method (default 10) - --ethereum.contract_call_batch_size int The number of contract calls to batch together when fetching data from the Ethereum node (default 25) - --ethereum.native_batch_call_size int The number of calls to batch together when using the native eth_call method (default 500) - --ethereum.rpc-url string e.g. "http://:8545" - --ethereum.use_native_batch_call Use the native eth_call method for batch calls (default true) - --prometheus.enabled e.g. "true" or "false" - --prometheus.port int The port to run the prometheus server on (default 2112) - --rewards.generate_staker_operators_table Generate staker operators table while indexing - --rewards.validate_rewards_root Validate rewards roots while indexing (default true) - --rpc.grpc-port int gRPC port (default 7100) - --rpc.http-port int http rpc port (default 7101) -``` - ## RPC Routes ### Get current block height diff --git a/cmd/restoreSnapshot.go b/cmd/restoreSnapshot.go index b7008eee..58f5c836 100644 --- a/cmd/restoreSnapshot.go +++ b/cmd/restoreSnapshot.go @@ -16,7 +16,9 @@ var restoreSnapshotCmd = &cobra.Command{ Use: "restore-snapshot", Short: "Restore database from a snapshot file", Long: `Restore the database from a previously created snapshot file. -The snapshot file is expected to be a pg_dump custom format file.`, + +Note: This command restores --database.schema_name only if it's present in InputFile snapshot. +Follow the snapshot docs if you need to convert the snapshot to a different schema name than was used during snapshot creation.`, RunE: func(cmd *cobra.Command, args []string) error { initRestoreSnapshotCmd(cmd) cfg := config.NewConfig() diff --git a/docs/snapshots_docs.md b/docs/snapshots_docs.md new file mode 100644 index 00000000..d9d546b8 --- /dev/null +++ b/docs/snapshots_docs.md @@ -0,0 +1,143 @@ +# Snapshots +Snapshots are useful for backing up the current state of the database and quickly booting up and reducing the sync to tip time. + +Below shows how to create your own snapshots, restore from the snapshot and how to use custom schemas + + +### `create-snapshot` +```bash +go run main.go create-snapshot --help +Create a snapshot of the database. + +Usage: + sidecar create-snapshot [flags] + +Flags: + -h, --help help for create-snapshot + --output_file string Path to save the snapshot file to (required) + +Global Flags: + -c, --chain string The chain to use (mainnet, holesky, preprod (default "mainnet") + --database.db_name string PostgreSQL database name (default "sidecar") + --database.host string PostgreSQL host (default "localhost") + --database.password string PostgreSQL password + --database.port int PostgreSQL port (default 5432) + --database.schema_name string PostgreSQL schema name (default "public") + --database.user string PostgreSQL username (default "sidecar") + --datadog.statsd.enabled e.g. "true" or "false" + --datadog.statsd.url string e.g. "localhost:8125" + --debug "true" or "false" + --ethereum.chunked_batch_call_size int The number of calls to make in parallel when using the chunked batch call method (default 10) + --ethereum.contract_call_batch_size int The number of contract calls to batch together when fetching data from the Ethereum node (default 25) + --ethereum.native_batch_call_size int The number of calls to batch together when using the native eth_call method (default 500) + --ethereum.rpc-url string e.g. "http://:8545" + --ethereum.use_native_batch_call Use the native eth_call method for batch calls (default true) + --prometheus.enabled e.g. "true" or "false" + --prometheus.port int The port to run the prometheus server on (default 2112) + --rewards.generate_staker_operators_table Generate staker operators table while indexing + --rewards.validate_rewards_root Validate rewards roots while indexing (default true) + --rpc.grpc-port int gRPC port (default 7100) + --rpc.http-port int http rpc port (default 7101) +``` + +#### Example use: +``` +go run main.go create-snapshot \ + --database.host=localhost \ + --database.user=sidecar \ + --database.password=sidecar \ + --database.port=5432 \ + --database.db_name=sidecar \ + --database.schema_name=public \ + --database.create_snapshot_output=example.dump +``` + +### `restore-snapshot` +```bash +go run main.go restore-snapshot --help +Restore the database from a previously created snapshot file. + +Note: This command restores --database.schema_name only if it's present in InputFile snapshot. +Follow the snapshot docs if you need to convert the snapshot to a different schema name than was used during snapshot creation. + +Usage: + sidecar restore-snapshot [flags] + +Flags: + -h, --help help for restore-snapshot + --input_file string Path to the snapshot file (required) + +Global Flags: + -c, --chain string The chain to use (mainnet, holesky, preprod (default "mainnet") + --database.db_name string PostgreSQL database name (default "sidecar") + --database.host string PostgreSQL host (default "localhost") + --database.password string PostgreSQL password + --database.port int PostgreSQL port (default 5432) + --database.schema_name string PostgreSQL schema name (default "public") + --database.user string PostgreSQL username (default "sidecar") + --datadog.statsd.enabled e.g. "true" or "false" + --datadog.statsd.url string e.g. "localhost:8125" + --debug "true" or "false" + --ethereum.chunked_batch_call_size int The number of calls to make in parallel when using the chunked batch call method (default 10) + --ethereum.contract_call_batch_size int The number of contract calls to batch together when fetching data from the Ethereum node (default 25) + --ethereum.native_batch_call_size int The number of calls to batch together when using the native eth_call method (default 500) + --ethereum.rpc-url string e.g. "http://:8545" + --ethereum.use_native_batch_call Use the native eth_call method for batch calls (default true) + --prometheus.enabled e.g. "true" or "false" + --prometheus.port int The port to run the prometheus server on (default 2112) + --rewards.generate_staker_operators_table Generate staker operators table while indexing + --rewards.validate_rewards_root Validate rewards roots while indexing (default true) + --rpc.grpc-port int gRPC port (default 7100) + --rpc.http-port int http rpc port (default 7101) +``` + + +## Converting the Schema of a Dump + +If you're using a custom schema and want to use a public snapshot, you likely want to convert the dump. + +This section provides a step-by-step runbook for converting a snapshot dump to use a different schema name. + +Commonly the input schema is + +``` +# Can use the script +./scripts/convertSnapshotSchema.sh + +```bash +./scripts/convertSnapshotSchema.sh input_file.dump output_file.dump +``` + + +```bash +# Open your terminal and create a temporary database to work with: +psql -c "CREATE DATABASE temp_sidecar_dump_schema_conversion_db;" + +# Use the Sidecar CLI to restore the snapshot dump into the temporary database: +go run main.go restore-snapshot \ + --database.host=localhost \ + --database.user=... \ + --database.password=... \ + --database.port=5432 \ + --database.db_name=temp_sidecar_dump_schema_conversion_db \ + --database.schema_name= \ + --input_file=snapshot.dump + +# Connect to the temporary database and execute the SQL command to rename the schema: +psql -d temp_sidecar_dump_schema_conversion_db -c "ALTER SCHEMA RENAME TO ;" + +# Use the Sidecar CLI to create a new snapshot with the updated schema: +go run main.go create-snapshot \ + --database.host=localhost \ + --database.user=... \ + --database.password=... \ + --database.port=5432 \ + --database.db_name=temp_sidecar_dump_schema_conversion_db \ + --database.schema_name= \ + --output_file=new_snapshot.dump + +# Drop the temporary database to free up resources: +psql -c "DROP DATABASE IF EXISTS temp_sidecar_dump_schema_conversion_db;" +``` + + diff --git a/scripts/convertSnapshotSchema.sh b/scripts/convertSnapshotSchema.sh new file mode 100755 index 00000000..a5a93e7c --- /dev/null +++ b/scripts/convertSnapshotSchema.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Check if all required arguments are provided +if [ "$#" -ne 6 ]; then + echo "Usage: $0 " + exit 1 +fi + +INPUT_SCHEMA_NAME=$1 +OUTPUT_SCHEMA_NAME=$2 +INPUT_FILE=$3 +OUTPUT_FILE=$4 +DB_USER=$5 +DB_PASSWORD=$6 + +# Generate a unique hash for the temporary database +HASH=$(date +%s | sha256sum | head -c 20) +TEMP_DB_NAME="temp_sidecar_dump_schema_conversion_db_${HASH}" + +# Drop the temporary database if it exists to ensure a clean slate +echo "Dropping temporary database if it exists: $TEMP_DB_NAME" +PGPASSWORD=$DB_PASSWORD psql -U $DB_USER -c "DROP DATABASE IF EXISTS $TEMP_DB_NAME;" || { echo "Failed during DROP DATABASE IF EXISTS "; exit 1; } + +# Create a temporary database +echo "Creating temporary database: $TEMP_DB_NAME" +PGPASSWORD=$DB_PASSWORD psql -U $DB_USER -c "CREATE DATABASE $TEMP_DB_NAME;" || { echo "Failed to create database"; exit 1; } + +# Restore the snapshot dump into the temporary database +echo "Restoring snapshot into temporary database" +go run main.go restore-snapshot \ + --database.host=localhost \ + --database.port=5432 \ + --database.user=$DB_USER \ + --database.password=$DB_PASSWORD \ + --database.db_name=$TEMP_DB_NAME \ + --database.schema_name=$INPUT_SCHEMA_NAME \ + --input_file=$INPUT_FILE || { echo "Failed to restore snapshot"; exit 1; } + +# Rename the schema in the temporary database +echo "Renaming schema from $INPUT_SCHEMA_NAME to $OUTPUT_SCHEMA_NAME" +PGPASSWORD=$DB_PASSWORD psql -U $DB_USER -d $TEMP_DB_NAME -c "ALTER SCHEMA $INPUT_SCHEMA_NAME RENAME TO $OUTPUT_SCHEMA_NAME;" || { echo "Failed to rename schema"; exit 1; } + +# Create a new snapshot with the updated schema +echo "Creating new snapshot with updated schema" +go run main.go create-snapshot \ + --database.host=localhost \ + --database.port=5432 \ + --database.user=$DB_USER \ + --database.password=$DB_PASSWORD \ + --database.db_name=$TEMP_DB_NAME \ + --database.schema_name=$OUTPUT_SCHEMA_NAME \ + --output_file=$OUTPUT_FILE || { echo "Failed to create snapshot"; exit 1; } + +# Drop the temporary database +echo "Dropping temporary database: $TEMP_DB_NAME" +PGPASSWORD=$DB_PASSWORD psql -U $DB_USER -c "DROP DATABASE IF EXISTS $TEMP_DB_NAME;" || { echo "Failed to drop database"; exit 1; } + +echo "Schema conversion completed successfully. Output saved to $OUTPUT_FILE."