Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for metrics collection #129

Merged
merged 4 commits into from
Dec 3, 2024
Merged

feat: add support for metrics collection #129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7cbde66
feat: add support for metrics collection
seanmcgary Dec 2, 2024
c991b5f
Remove old statsd url, update readme
seanmcgary Dec 3, 2024
bf47812
Reduce log noise
seanmcgary Dec 3, 2024
70fe2a9
Make the log date human readable
seanmcgary Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,17 +122,21 @@ Global Flags:
--database.port int PostgreSQL port (default 5432)
--database.schema_name string PostgreSQL schema name (default "public")
--database.user string PostgreSQL username (default "sidecar")
--datadog.statsd.enabled e.g. "true" or "false"
--datadog.statsd.url string e.g. "localhost:8125"
--debug "true" or "false"
--ethereum.chunked_batch_call_size int The number of calls to make in parallel when using the chunked batch call method (default 10)
--ethereum.contract_call_batch_size int The number of contract calls to batch together when fetching data from the Ethereum node (default 25)
--ethereum.native_batch_call_size int The number of calls to batch together when using the native eth_call method (default 500)
--ethereum.rpc-url string e.g. "http://34.229.43.36:8545"
--ethereum.use_native_batch_call Use the native eth_call method for batch calls (default true)
--prometheus.enabled e.g. "true" or "false"
--prometheus.port int The port to run the prometheus server on (default 2112)
--rewards.generate_staker_operators_table Generate staker operators table while indexing
--rewards.validate_rewards_root Validate rewards roots while indexing (default true)
--rpc.grpc-port int gRPC port (default 7100)
--rpc.http-port int http rpc port (default 7101)
--statsd.url string e.g. "localhost:8125"


```

Expand Down
11 changes: 8 additions & 3 deletions cmd/database.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,14 @@ var runDatabaseCmd = &cobra.Command{
zap.String("commit", version.GetCommit()),
)

sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl)
metricsClients, err := metrics.InitMetricsSinksFromConfig(cfg, l)
if err != nil {
l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err))
l.Sugar().Fatal("Failed to setup metrics sink", zap.Error(err))
}

sdc, err := metrics.NewMetricsSink(&metrics.MetricsSinkConfig{}, metricsClients)
if err != nil {
l.Sugar().Fatal("Failed to setup metrics sink", zap.Error(err))
}

client := ethereum.NewClient(ethereum.ConvertGlobalConfigToEthereumConfig(&cfg.EthereumRpcConfig), l)
Expand Down Expand Up @@ -96,7 +101,7 @@ var runDatabaseCmd = &cobra.Command{
l.Sugar().Fatalw("Failed to create rewards calculator", zap.Error(err))
}

_ = pipeline.NewPipeline(fetchr, idxr, mds, sm, rc, cfg, l)
_ = pipeline.NewPipeline(fetchr, idxr, mds, sm, rc, cfg, sdc, l)

l.Sugar().Infow("Done")
},
Expand Down
11 changes: 8 additions & 3 deletions cmd/debugger/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,14 @@ func main() {

l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})

sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl)
metricsClients, err := metrics.InitMetricsSinksFromConfig(cfg, l)
if err != nil {
l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err))
l.Sugar().Fatal("Failed to setup metrics sink", zap.Error(err))
}

sdc, err := metrics.NewMetricsSink(&metrics.MetricsSinkConfig{}, metricsClients)
if err != nil {
l.Sugar().Fatal("Failed to setup metrics sink", zap.Error(err))
}

client := ethereum.NewClient(ethereum.ConvertGlobalConfigToEthereumConfig(&cfg.EthereumRpcConfig), l)
Expand Down Expand Up @@ -87,7 +92,7 @@ func main() {
l.Sugar().Fatalw("Failed to create rewards calculator", zap.Error(err))
}

p := pipeline.NewPipeline(fetchr, idxr, mds, sm, rc, cfg, l)
p := pipeline.NewPipeline(fetchr, idxr, mds, sm, rc, cfg, sdc, l)

// Create new sidecar instance
sidecar := sidecar.NewSidecar(&sidecar.SidecarConfig{
Expand Down
9 changes: 7 additions & 2 deletions cmd/operatorRestakedStrategies.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,14 @@ var runOperatorRestakedStrategiesCmd = &cobra.Command{

l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug})

sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl)
metricsClients, err := metrics.InitMetricsSinksFromConfig(cfg, l)
if err != nil {
l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err))
l.Sugar().Fatal("Failed to setup metrics sink", zap.Error(err))
}

sdc, err := metrics.NewMetricsSink(&metrics.MetricsSinkConfig{}, metricsClients)
if err != nil {
l.Sugar().Fatal("Failed to setup metrics sink", zap.Error(err))
}

client := ethereum.NewClient(ethereum.ConvertGlobalConfigToEthereumConfig(&cfg.EthereumRpcConfig), l)
Expand Down
7 changes: 6 additions & 1 deletion cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ func init() {

rootCmd.PersistentFlags().Bool("debug", false, `"true" or "false"`)
rootCmd.PersistentFlags().StringP("chain", "c", "mainnet", "The chain to use (mainnet, holesky, preprod")
rootCmd.PersistentFlags().String("statsd.url", "", `e.g. "localhost:8125"`)

rootCmd.PersistentFlags().String("ethereum.rpc-url", "", `e.g. "http://34.229.43.36:8545"`)
rootCmd.PersistentFlags().Int(config.EthereumRpcContractCallBatchSize, 25, `The number of contract calls to batch together when fetching data from the Ethereum node`)
Expand All @@ -47,6 +46,12 @@ func init() {
rootCmd.PersistentFlags().Int("rpc.grpc-port", 7100, `gRPC port`)
rootCmd.PersistentFlags().Int("rpc.http-port", 7101, `http rpc port`)

rootCmd.PersistentFlags().Bool("datadog.statsd.enabled", false, `e.g. "true" or "false"`)
rootCmd.PersistentFlags().String("datadog.statsd.url", "", `e.g. "localhost:8125"`)

rootCmd.PersistentFlags().Bool("prometheus.enabled", false, `e.g. "true" or "false"`)
rootCmd.PersistentFlags().Int("prometheus.port", 2112, `The port to run the prometheus server on`)

rootCmd.PersistentFlags().VisitAll(func(f *pflag.Flag) {
key := config.KebabToSnakeCase(f.Name)
viper.BindPFlag(key, f) //nolint:errcheck
Expand Down
16 changes: 13 additions & 3 deletions cmd/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,14 @@ var runCmd = &cobra.Command{
zap.String("chain", cfg.Chain.String()),
)

sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl)
metricsClients, err := metrics.InitMetricsSinksFromConfig(cfg, l)
if err != nil {
l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err))
l.Sugar().Fatal("Failed to setup metrics sink", zap.Error(err))
}

sdc, err := metrics.NewMetricsSink(&metrics.MetricsSinkConfig{}, metricsClients)
if err != nil {
l.Sugar().Fatal("Failed to setup metrics sink", zap.Error(err))
}

client := ethereum.NewClient(ethereum.ConvertGlobalConfigToEthereumConfig(&cfg.EthereumRpcConfig), l)
Expand Down Expand Up @@ -103,7 +108,7 @@ var runCmd = &cobra.Command{
l.Sugar().Fatalw("Failed to create rewards calculator", zap.Error(err))
}

p := pipeline.NewPipeline(fetchr, idxr, mds, sm, rc, cfg, l)
p := pipeline.NewPipeline(fetchr, idxr, mds, sm, rc, cfg, sdc, l)

// Create new sidecar instance
sidecar := sidecar.NewSidecar(&sidecar.SidecarConfig{
Expand All @@ -117,6 +122,11 @@ var runCmd = &cobra.Command{
l.Sugar().Fatalw("Failed to start RPC server", zap.Error(err))
}

promChan := make(chan bool)
if cfg.PrometheusConfig.Enabled {
sidecar.WithPrometheusServer(promChan)
}

// Start the sidecar main process in a goroutine so that we can listen for a shutdown signal
go sidecar.Start(ctx)

Expand Down
7 changes: 7 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0
github.com/lib/pq v1.10.9
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.12.0
github.com/rs/cors v1.7.0
github.com/shopspring/decimal v1.4.0
github.com/spf13/cobra v1.8.1
Expand All @@ -34,9 +35,11 @@ require (
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.13.0 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/consensys/bavard v0.1.13 // indirect
github.com/consensys/gnark-crypto v0.12.1 // indirect
github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
Expand Down Expand Up @@ -64,10 +67,14 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rs/zerolog v1.33.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
Expand Down
Loading
Loading