Generate StakerOperators table after generating rewards (#124)

Layr-Labs · Dec 2, 2024 · 1581008 · 1581008
2 parents 437fe68 + c5f33b7
commit 1581008
Show file tree

Hide file tree

Showing 44 changed files with 1,379 additions and 146 deletions.
diff --git a/cmd/database.go b/cmd/database.go
@@ -18,6 +18,7 @@ import (
 	"github.com/Layr-Labs/sidecar/pkg/postgres"
 	"github.com/Layr-Labs/sidecar/pkg/postgres/migrations"
 	"github.com/Layr-Labs/sidecar/pkg/rewards"
+	"github.com/Layr-Labs/sidecar/pkg/rewards/stakerOperators"
 	pgStorage "github.com/Layr-Labs/sidecar/pkg/storage/postgres"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -88,7 +89,9 @@ var runDatabaseCmd = &cobra.Command{
 
 		idxr := indexer.NewIndexer(mds, contractStore, cm, client, fetchr, cc, grm, l, cfg)
 
-		rc, err := rewards.NewRewardsCalculator(cfg, grm, mds, l)
+		sog := stakerOperators.NewStakerOperatorGenerator(grm, l, cfg)
+
+		rc, err := rewards.NewRewardsCalculator(cfg, grm, mds, sog, l)
 		if err != nil {
 			l.Sugar().Fatalw("Failed to create rewards calculator", zap.Error(err))
 		}

diff --git a/cmd/debugger/main.go b/cmd/debugger/main.go
@@ -13,6 +13,7 @@ import (
 	"github.com/Layr-Labs/sidecar/pkg/pipeline"
 	"github.com/Layr-Labs/sidecar/pkg/postgres"
 	"github.com/Layr-Labs/sidecar/pkg/rewards"
+	"github.com/Layr-Labs/sidecar/pkg/rewards/stakerOperators"
 	"github.com/Layr-Labs/sidecar/pkg/sidecar"
 	pgStorage "github.com/Layr-Labs/sidecar/pkg/storage/postgres"
 	"log"
@@ -79,7 +80,9 @@ func main() {
 
 	idxr := indexer.NewIndexer(mds, contractStore, cm, client, fetchr, cc, grm, l, cfg)
 
-	rc, err := rewards.NewRewardsCalculator(cfg, grm, mds, l)
+	sog := stakerOperators.NewStakerOperatorGenerator(grm, l, cfg)
+
+	rc, err := rewards.NewRewardsCalculator(cfg, grm, mds, sog, l)
 	if err != nil {
 		l.Sugar().Fatalw("Failed to create rewards calculator", zap.Error(err))
 	}

diff --git a/cmd/root.go b/cmd/root.go
@@ -38,6 +38,9 @@ func init() {
 	rootCmd.PersistentFlags().String(config.DatabaseDbName, "sidecar", `Defaults to 'sidecar'`)
 	rootCmd.PersistentFlags().String(config.DatabaseSchemaName, "", `Defaults to "public"`)
 
+	rootCmd.PersistentFlags().Bool(config.RewardsValidateRewardsRoot, true, `Validate rewards roots while indexing`)
+	rootCmd.PersistentFlags().Bool(config.RewardsGenerateStakerOperatorsTable, false, `Generate staker operators table while indexing`)
+
 	rootCmd.PersistentFlags().Int("rpc.grpc-port", 7100, `e.g. 7100`)
 	rootCmd.PersistentFlags().Int("rpc.http-port", 7101, `e.g. 7101`)
 

diff --git a/cmd/run.go b/cmd/run.go
@@ -14,6 +14,7 @@ import (
 	"github.com/Layr-Labs/sidecar/pkg/pipeline"
 	"github.com/Layr-Labs/sidecar/pkg/postgres"
 	"github.com/Layr-Labs/sidecar/pkg/rewards"
+	"github.com/Layr-Labs/sidecar/pkg/rewards/stakerOperators"
 	"github.com/Layr-Labs/sidecar/pkg/shutdown"
 	"github.com/Layr-Labs/sidecar/pkg/sidecar"
 	pgStorage "github.com/Layr-Labs/sidecar/pkg/storage/postgres"
@@ -94,7 +95,9 @@ var runCmd = &cobra.Command{
 
 		idxr := indexer.NewIndexer(mds, contractStore, cm, client, fetchr, cc, grm, l, cfg)
 
-		rc, err := rewards.NewRewardsCalculator(cfg, grm, mds, l)
+		sog := stakerOperators.NewStakerOperatorGenerator(grm, l, cfg)
+
+		rc, err := rewards.NewRewardsCalculator(cfg, grm, mds, sog, l)
 		if err != nil {
 			l.Sugar().Fatalw("Failed to create rewards calculator", zap.Error(err))
 		}

diff --git a/internal/config/config.go b/internal/config/config.go
@@ -44,6 +44,7 @@ type Config struct {
 	DatabaseConfig    DatabaseConfig
 	RpcConfig         RpcConfig
 	Chain             Chain
+	Rewards           RewardsConfig
 }
 
 type EthereumRpcConfig struct {
@@ -65,6 +66,11 @@ type RpcConfig struct {
 	HttpPort int
 }
 
+type RewardsConfig struct {
+	ValidateRewardsRoot          bool
+	GenerateStakerOperatorsTable bool
+}
+
 func StringWithDefault(value, defaultValue string) string {
 	if value == "" {
 		return defaultValue
@@ -79,6 +85,9 @@ var (
 	DatabasePassword   = "database.password"
 	DatabaseDbName     = "database.db_name"
 	DatabaseSchemaName = "database.schema_name"
+
+	RewardsValidateRewardsRoot          = "rewards.validate_rewards_root"
+	RewardsGenerateStakerOperatorsTable = "rewards.generate_staker_operators_table"
 )
 
 func NewConfig() *Config {
@@ -105,6 +114,11 @@ func NewConfig() *Config {
 			GrpcPort: viper.GetInt(normalizeFlagName("rpc.grpc_port")),
 			HttpPort: viper.GetInt(normalizeFlagName("rpc.http_port")),
 		},
+
+		Rewards: RewardsConfig{
+			ValidateRewardsRoot:          viper.GetBool(normalizeFlagName(RewardsValidateRewardsRoot)),
+			GenerateStakerOperatorsTable: viper.GetBool(normalizeFlagName(RewardsGenerateStakerOperatorsTable)),
+		},
 	}
 }
 

diff --git a/pkg/pipeline/pipeline.go b/pkg/pipeline/pipeline.go
@@ -199,6 +199,15 @@ func (p *Pipeline) RunForFetchedBlock(ctx context.Context, block *fetcher.Fetche
 				continue
 			}
 
+			if !p.globalConfig.Rewards.ValidateRewardsRoot {
+				p.Logger.Sugar().Warnw("Rewards validation is disabled, skipping rewards validation",
+					zap.Uint64("blockNumber", blockNumber),
+					zap.Uint64("rootIndex", rs.RootIndex),
+					zap.String("root", rs.Root),
+				)
+				continue
+			}
+
 			// The RewardsCalculationEnd date is the max(snapshot) from the gold table at the time, NOT the exclusive
 			// cutoff date that was actually used to generate the rewards. To get that proper cutoff date, we need
 			// to add 1 day to the RewardsCalculationEnd date.

diff --git a/pkg/pipeline/pipelineIntegration_test.go b/pkg/pipeline/pipelineIntegration_test.go
@@ -13,6 +13,7 @@ import (
 	"github.com/Layr-Labs/sidecar/pkg/indexer"
 	"github.com/Layr-Labs/sidecar/pkg/postgres"
 	"github.com/Layr-Labs/sidecar/pkg/rewards"
+	"github.com/Layr-Labs/sidecar/pkg/rewards/stakerOperators"
 	"github.com/Layr-Labs/sidecar/pkg/storage"
 	pgStorage "github.com/Layr-Labs/sidecar/pkg/storage/postgres"
 	"log"
@@ -79,8 +80,8 @@ func setup() (
 	if err := eigenState.LoadEigenStateModels(sm, grm, l, cfg); err != nil {
 		l.Sugar().Fatalw("Failed to load eigen state models", zap.Error(err))
 	}
-
-	rc, _ := rewards.NewRewardsCalculator(cfg, grm, mds, l)
+	sog := stakerOperators.NewStakerOperatorGenerator(grm, l, cfg)
+	rc, _ := rewards.NewRewardsCalculator(cfg, grm, mds, sog, l)
 
 	fetchr := fetcher.NewFetcher(client, cfg, l)
 

diff --git a/pkg/postgres/migrations/202412021311_stakerOperatorTables/up.go b/pkg/postgres/migrations/202412021311_stakerOperatorTables/up.go
@@ -0,0 +1,39 @@
+package _202412021311_stakerOperatorTables
+
+import (
+	"database/sql"
+	"gorm.io/gorm"
+)
+
+type Migration struct {
+}
+
+func (m *Migration) Up(db *sql.DB, grm *gorm.DB) error {
+	queries := []string{
+		`CREATE TABLE IF NOT EXISTS staker_operator (
+			earner text,
+			operator text,
+			reward_type text,
+			avs text,
+			token text,
+			strategy text,
+			multiplier numeric(78),
+			shares numeric,
+			amount numeric,
+			reward_hash text,
+			snapshot date
+		);`,
+		`alter table staker_operator add constraint uniq_staker_operator unique (earner, operator, snapshot, reward_hash, strategy, reward_type);`,
+	}
+
+	for _, query := range queries {
+		if err := grm.Exec(query).Error; err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m *Migration) GetName() string {
+	return "202412021311_stakerOperatorTables"
+}
diff --git a/pkg/postgres/migrations/migrator.go b/pkg/postgres/migrations/migrator.go
@@ -33,6 +33,7 @@ import (
 	_202411130953_addHashColumns "github.com/Layr-Labs/sidecar/pkg/postgres/migrations/202411130953_addHashColumns"
 	_202411131200_eigenStateModelConstraints "github.com/Layr-Labs/sidecar/pkg/postgres/migrations/202411131200_eigenStateModelConstraints"
 	_202411191947_cleanupUnusedTables "github.com/Layr-Labs/sidecar/pkg/postgres/migrations/202411191947_cleanupUnusedTables"
+	_202412021311_stakerOperatorTables "github.com/Layr-Labs/sidecar/pkg/postgres/migrations/202412021311_stakerOperatorTables"
 	"go.uber.org/zap"
 	"gorm.io/gorm"
 	"time"
@@ -104,6 +105,7 @@ func (m *Migrator) MigrateAll() error {
 		&_202411130953_addHashColumns.Migration{},
 		&_202411131200_eigenStateModelConstraints.Migration{},
 		&_202411191947_cleanupUnusedTables.Migration{},
+		&_202412021311_stakerOperatorTables.Migration{},
 	}
 
 	for _, migration := range migrations {

diff --git a/pkg/rewards/1_goldActiveRewards.go b/pkg/rewards/1_goldActiveRewards.go
@@ -2,6 +2,7 @@ package rewards
 
 import (
 	"database/sql"
+	"github.com/Layr-Labs/sidecar/pkg/rewardsUtils"
 	"go.uber.org/zap"
 )
 
@@ -100,8 +101,8 @@ select * from active_rewards_final
 // @param startDate: The lower bound of when to calculate rewards from. If we're running rewards for the first time,
 // this will be "1970-01-01". If this is a subsequent run, this will be the last snapshot date.
 func (r *RewardsCalculator) Generate1ActiveRewards(snapshotDate string) error {
-	allTableNames := getGoldTableNames(snapshotDate)
-	destTableName := allTableNames[Table_1_ActiveRewards]
+	allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate)
+	destTableName := allTableNames[rewardsUtils.Table_1_ActiveRewards]
 
 	rewardsStart := "1970-01-01 00:00:00" // This will always start as this date and get's updated later in the query
 
@@ -111,7 +112,7 @@ func (r *RewardsCalculator) Generate1ActiveRewards(snapshotDate string) error {
 		zap.String("destTableName", destTableName),
 	)
 
-	query, err := renderQueryTemplate(_1_goldActiveRewardsQuery, map[string]string{
+	query, err := rewardsUtils.RenderQueryTemplate(_1_goldActiveRewardsQuery, map[string]string{
 		"destTableName": destTableName,
 		"rewardsStart":  rewardsStart,
 		"cutoffDate":    snapshotDate,

diff --git a/pkg/rewards/2_goldStakerRewardAmounts.go b/pkg/rewards/2_goldStakerRewardAmounts.go
@@ -3,6 +3,7 @@ package rewards
 import (
 	"database/sql"
 	"github.com/Layr-Labs/sidecar/internal/config"
+	"github.com/Layr-Labs/sidecar/pkg/rewardsUtils"
 	"go.uber.org/zap"
 )
 
@@ -133,8 +134,8 @@ ORDER BY reward_hash, snapshot, staker, operator
 `
 
 func (rc *RewardsCalculator) GenerateGold2StakerRewardAmountsTable(snapshotDate string, forks config.ForkMap) error {
-	allTableNames := getGoldTableNames(snapshotDate)
-	destTableName := allTableNames[Table_2_StakerRewardAmounts]
+	allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate)
+	destTableName := allTableNames[rewardsUtils.Table_2_StakerRewardAmounts]
 
 	rc.logger.Sugar().Infow("Generating staker reward amounts",
 		zap.String("cutoffDate", snapshotDate),
@@ -143,9 +144,9 @@ func (rc *RewardsCalculator) GenerateGold2StakerRewardAmountsTable(snapshotDate
 		zap.String("nileHardforkDate", forks[config.Fork_Nile]),
 	)
 
-	query, err := renderQueryTemplate(_2_goldStakerRewardAmountsQuery, map[string]string{
+	query, err := rewardsUtils.RenderQueryTemplate(_2_goldStakerRewardAmountsQuery, map[string]string{
 		"destTableName":      destTableName,
-		"activeRewardsTable": allTableNames[Table_1_ActiveRewards],
+		"activeRewardsTable": allTableNames[rewardsUtils.Table_1_ActiveRewards],
 	})
 	if err != nil {
 		rc.logger.Sugar().Errorw("Failed to render query template", "error", err)

diff --git a/pkg/rewards/3_goldOperatorRewardAmounts.go b/pkg/rewards/3_goldOperatorRewardAmounts.go
@@ -1,6 +1,7 @@
 package rewards
 
 import (
+	"github.com/Layr-Labs/sidecar/pkg/rewardsUtils"
 	"go.uber.org/zap"
 )
 
@@ -36,17 +37,17 @@ SELECT * FROM distinct_operators
 `
 
 func (rc *RewardsCalculator) GenerateGold3OperatorRewardAmountsTable(snapshotDate string) error {
-	allTableNames := getGoldTableNames(snapshotDate)
-	destTableName := allTableNames[Table_3_OperatorRewardAmounts]
+	allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate)
+	destTableName := allTableNames[rewardsUtils.Table_3_OperatorRewardAmounts]
 
 	rc.logger.Sugar().Infow("Generating staker reward amounts",
 		zap.String("cutoffDate", snapshotDate),
 		zap.String("destTableName", destTableName),
 	)
 
-	query, err := renderQueryTemplate(_3_goldOperatorRewardAmountsQuery, map[string]string{
+	query, err := rewardsUtils.RenderQueryTemplate(_3_goldOperatorRewardAmountsQuery, map[string]string{
 		"destTableName":            destTableName,
-		"stakerRewardAmountsTable": allTableNames[Table_2_StakerRewardAmounts],
+		"stakerRewardAmountsTable": allTableNames[rewardsUtils.Table_2_StakerRewardAmounts],
 	})
 	if err != nil {
 		rc.logger.Sugar().Errorw("Failed to render query template", "error", err)

diff --git a/pkg/rewards/4_goldRewardsForAll.go b/pkg/rewards/4_goldRewardsForAll.go
@@ -1,6 +1,7 @@
 package rewards
 
 import (
+	"github.com/Layr-Labs/sidecar/pkg/rewardsUtils"
 	"go.uber.org/zap"
 )
 
@@ -67,17 +68,17 @@ SELECT * from staker_tokens
 `
 
 func (rc *RewardsCalculator) GenerateGold4RewardsForAllTable(snapshotDate string) error {
-	allTableNames := getGoldTableNames(snapshotDate)
-	destTableName := allTableNames[Table_4_RewardsForAll]
+	allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate)
+	destTableName := allTableNames[rewardsUtils.Table_4_RewardsForAll]
 
 	rc.logger.Sugar().Infow("Generating rewards for all table",
 		zap.String("cutoffDate", snapshotDate),
 		zap.String("destTableName", destTableName),
 	)
 
-	query, err := renderQueryTemplate(_4_goldRewardsForAllQuery, map[string]string{
+	query, err := rewardsUtils.RenderQueryTemplate(_4_goldRewardsForAllQuery, map[string]string{
 		"destTableName":      destTableName,
-		"activeRewardsTable": allTableNames[Table_1_ActiveRewards],
+		"activeRewardsTable": allTableNames[rewardsUtils.Table_1_ActiveRewards],
 	})
 	if err != nil {
 		rc.logger.Sugar().Errorw("Failed to render query template", "error", err)

diff --git a/pkg/rewards/5_goldRfaeStakers.go b/pkg/rewards/5_goldRfaeStakers.go
@@ -3,6 +3,7 @@ package rewards
 import (
 	"database/sql"
 	"github.com/Layr-Labs/sidecar/internal/config"
+	"github.com/Layr-Labs/sidecar/pkg/rewardsUtils"
 	"go.uber.org/zap"
 )
 
@@ -118,17 +119,17 @@ ORDER BY reward_hash, snapshot, staker, operator
 `
 
 func (rc *RewardsCalculator) GenerateGold5RfaeStakersTable(snapshotDate string, forks config.ForkMap) error {
-	allTableNames := getGoldTableNames(snapshotDate)
-	destTableName := allTableNames[Table_5_RfaeStakers]
+	allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate)
+	destTableName := allTableNames[rewardsUtils.Table_5_RfaeStakers]
 
 	rc.logger.Sugar().Infow("Generating rfae stakers table",
 		zap.String("cutoffDate", snapshotDate),
 		zap.String("destTableName", destTableName),
 	)
 
-	query, err := renderQueryTemplate(_5_goldRfaeStakersQuery, map[string]string{
+	query, err := rewardsUtils.RenderQueryTemplate(_5_goldRfaeStakersQuery, map[string]string{
 		"destTableName":      destTableName,
-		"activeRewardsTable": allTableNames[Table_1_ActiveRewards],
+		"activeRewardsTable": allTableNames[rewardsUtils.Table_1_ActiveRewards],
 	})
 	if err != nil {
 		rc.logger.Sugar().Errorw("Failed to render query template", "error", err)

diff --git a/pkg/rewards/6_goldRfaeOperators.go b/pkg/rewards/6_goldRfaeOperators.go
@@ -1,6 +1,7 @@
 package rewards
 
 import (
+	"github.com/Layr-Labs/sidecar/pkg/rewardsUtils"
 	"go.uber.org/zap"
 )
 
@@ -36,17 +37,17 @@ SELECT * FROM distinct_operators
 `
 
 func (rc *RewardsCalculator) GenerateGold6RfaeOperatorsTable(snapshotDate string) error {
-	allTableNames := getGoldTableNames(snapshotDate)
-	destTableName := allTableNames[Table_6_RfaeOperators]
+	allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate)
+	destTableName := allTableNames[rewardsUtils.Table_6_RfaeOperators]
 
 	rc.logger.Sugar().Infow("Generating rfae operators table",
 		zap.String("cutoffDate", snapshotDate),
 		zap.String("destTableName", destTableName),
 	)
 
-	query, err := renderQueryTemplate(_6_goldRfaeOperatorsQuery, map[string]string{
+	query, err := rewardsUtils.RenderQueryTemplate(_6_goldRfaeOperatorsQuery, map[string]string{
 		"destTableName":    destTableName,
-		"rfaeStakersTable": allTableNames[Table_5_RfaeStakers],
+		"rfaeStakersTable": allTableNames[rewardsUtils.Table_5_RfaeStakers],
 	})
 	if err != nil {
 		rc.logger.Sugar().Errorw("Failed to render query template", "error", err)