release: Rewards v2 (#106)

Adds support for the new Rewards V2

.gitignore

@@ -2,6 +2,8 @@
 /*.sw*
 *.terraform
 *.terraform*
+*.sql
+*.dump
 /bin
 /scripts/runLocal
 sidecar.db*
@@ -30,5 +32,5 @@ node_modules
 chart_releases
 /snapshots/**/*.sql
 /snapshots/**/*.csv
-*.sql
-*.dump
+
+.env

.testdataVersion

@@ -1 +1 @@
-e94b8e364dfddd0743746f089f89a3d570751f03
+da8f00be49d1447f22934629d9d6819e3d763af9

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/wealdtech/go-merkletree/v2 v2.6.0
 	github.com/wk8/go-ordered-map/v2 v2.1.8
 	go.uber.org/zap v1.27.0
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
 	google.golang.org/grpc v1.65.0
 	gorm.io/driver/postgres v1.5.9
 	gorm.io/gorm v1.25.10

go.sum

@@ -670,6 +670,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=

internal/config/config.go

@@ -3,9 +3,11 @@ package config
 import (
 	"errors"
 	"fmt"
-	"github.com/spf13/viper"
 	"strconv"
 	"strings"
+	"time"
+
+	"github.com/spf13/viper"
 )
 
 type EnvScope string
@@ -29,6 +31,7 @@ const (
 	Fork_Nile   ForkName = "nile"
 	Fork_Amazon ForkName = "amazon"
 	Fork_Panama ForkName = "panama"
+	Fork_Arno   ForkName = "arno"
 
 	ENV_PREFIX = "SIDECAR"
 )
@@ -252,18 +255,21 @@ func (c *Config) GetForkDates() (ForkMap, error) {
 			Fork_Amazon: "1970-01-01", // Amazon hard fork was never on preprod as we backfilled
 			Fork_Nile:   "2024-08-14", // Last calculation end timestamp was 8-13: https://holesky.etherscan.io/tx/0xb5a6855e88c79312b7c0e1c9f59ae9890b97f157ea27e69e4f0fadada4712b64#eventlog
 			Fork_Panama: "2024-10-01",
+			Fork_Arno:   "2024-12-11",
 		}, nil
 	case Chain_Holesky:
 		return ForkMap{
 			Fork_Amazon: "1970-01-01", // Amazon hard fork was never on testnet as we backfilled
 			Fork_Nile:   "2024-08-13", // Last calculation end timestamp was 8-12: https://holesky.etherscan.io/tx/0x5fc81b5ed2a78b017ef313c181d8627737a97fef87eee85acedbe39fc8708c56#eventlog
 			Fork_Panama: "2024-10-01",
+			Fork_Arno:   "2024-12-13",
 		}, nil
 	case Chain_Mainnet:
 		return ForkMap{
 			Fork_Amazon: "2024-08-02", // Last calculation end timestamp was 8-01: https://etherscan.io/tx/0x2aff6f7b0132092c05c8f6f41a5e5eeeb208aa0d95ebcc9022d7823e343dd012#eventlog
 			Fork_Nile:   "2024-08-12", // Last calculation end timestamp was 8-11: https://etherscan.io/tx/0x922d29d93c02d189fc2332041f01a80e0007cd7a625a5663ef9d30082f7ef66f#eventlog
 			Fork_Panama: "2024-10-01",
+			Fork_Arno:   "2025-01-07",
 		}, nil
 	}
 	return nil, errors.New("unsupported chain")
@@ -291,6 +297,23 @@ func (c *Config) GetOperatorRestakedStrategiesStartBlock() uint64 {
 	return 0
 }
 
+func (c *Config) IsRewardsV2EnabledForCutoffDate(cutoffDate string) (bool, error) {
+	forks, err := c.GetForkDates()
+	if err != nil {
+		return false, err
+	}
+	cutoffDateTime, err := time.Parse(time.DateOnly, cutoffDate)
+	if err != nil {
+		return false, errors.Join(fmt.Errorf("failed to parse cutoff date %s", cutoffDate), err)
+	}
+	arnoForkDateTime, err := time.Parse(time.DateOnly, forks[Fork_Arno])
+	if err != nil {
+		return false, errors.Join(fmt.Errorf("failed to parse Arno fork date %s", forks[Fork_Arno]), err)
+	}
+
+	return cutoffDateTime.Compare(arnoForkDateTime) >= 0, nil
+}
+
 // CanIgnoreIncorrectRewardsRoot returns true if the rewards root can be ignored for the given block number
 //
 // Due to inconsistencies in the rewards root calculation on testnet, we know that some roots
@@ -306,6 +329,11 @@ func (c *Config) CanIgnoreIncorrectRewardsRoot(blockNumber uint64) bool {
 		if blockNumber == 2812052 {
 			return true
 		}
+
+		// ignore rewards-v2 deployment/testing range
+		if blockNumber >= 2877938 && blockNumber <= 2909856 {
+			return true
+		}
 	case Chain_Holesky:
 		// roughly 2024-08-01
 		if blockNumber < 2046020 {

internal/tests/testdata/avsOperators/README.md

@@ -16,4 +16,24 @@ where
     address = '0x135dda560e946695d6f155dacafc6f1f25c1f5af'
     and event_name = 'OperatorAVSRegistrationStatusUpdated'
     and block_number < 20613003
-``
+```
+
+## preprod rewardsv2
+
+
+```sql
+select
+    transaction_hash,
+    transaction_index,
+    block_number,
+    address,
+    arguments,
+    event_name,
+    log_index,
+    output_data
+from transaction_logs
+where
+    address = '0x141d6995556135d4997b2ff72eb443be300353bc'
+    and event_name = 'OperatorAVSRegistrationStatusUpdated'
+    and block_number < 2909490
+```

internal/tests/testdata/combinedRewards/README.md

@@ -69,3 +69,23 @@ select
 from dbt_mainnet_ethereum_rewards.rewards_combined
 where block_time < '2024-08-20'
 ```
+
+## preprod rewardsv2
+
+```sql
+select
+    avs,
+    reward_hash,
+    token,
+    amount as amount,
+    strategy,
+    strategy_index,
+    multiplier as multiplier,
+    start_timestamp::timestamp(6) as start_timestamp,
+    end_timestamp::timestamp(6) as end_timestamp,
+    reward_type,
+    duration,
+    block_number as block_number
+from dbt_preprod_holesky_rewards.rewards_combined
+where block_time < '2024-12-13'
+```

internal/tests/testdata/fetchExpectedResults.sh

@@ -5,20 +5,28 @@ NETWORK=$1
 sqlFileName="generateExpectedResults.sql"
 outputFile="expectedResults.csv"
 
+postgresPort=5432
+
 if [[ -z $NETWORK ]]; then
     echo "Usage: $0 <network>"
     exit 1
 fi
 
 if [[ $NETWORK == "mainnet-reduced" ]]; then
     sqlFileName="mainnetReduced_${sqlFileName}"
+    postgresPort=5434
 fi
 
 if [[ $NETWORK == "testnet-reduced" ]]; then
     sqlFileName="testnetReduced_${sqlFileName}"
 fi
 
-for d in operatorShares; do
+if [[ $NETWORK == "preprod-rewardsV2" ]]; then
+    sqlFileName="preprodRewardsV2_${sqlFileName}"
+    postgresPort=5435
+fi
+
+for d in stakerShareSnapshots; do
     echo "Processing directory: $d"
         if [[ $d == "7_goldStaging" ]]; then
             files=$(ls "./${d}" | grep "_generateExpectedResults_")
@@ -30,12 +38,12 @@ for d in operatorShares; do
                 sqlFileWithPath="${d}/$f"
                 outputFileWithPath="${d}/expectedResults_${snapshotDate}.csv"
                 echo "Generating expected results for ${sqlFileWithPath} to ${outputFileWithPath}"
-                psql --host localhost --port 5434 --user blocklake --dbname blocklake --password < $sqlFileWithPath > $outputFileWithPath
+                psql --host localhost --port $postgresPort --user blocklake --dbname blocklake --password < $sqlFileWithPath > $outputFileWithPath
             done
         else
             echo "Generating expected results for $d"
             sqlFileWithPath="${d}/${sqlFileName}"
             outputFileWithPath="${d}/${outputFile}"
-            psql --host localhost --port 5434 --user blocklake --dbname blocklake --password < $sqlFileWithPath > $outputFileWithPath
+            psql --host localhost --port $postgresPort --user blocklake --dbname blocklake --password < $sqlFileWithPath > $outputFileWithPath
         fi
 done

internal/tests/testdata/operatorAvsRegistrationSnapshots/README.md

@@ -81,6 +81,34 @@ select
 from filtered
 ```
 
+preprod rewardsv2
+
+```sql
+with filtered as (
+    SELECT
+        lower(t.arguments #>> '{0,Value}') as operator,
+        lower(t.arguments #>> '{1,Value}') as avs,
+        case when (t.output_data ->> 'status')::integer = 1 then true else false end as status,
+    t.transaction_hash,
+    t.log_index,
+    b.block_time::timestamp(6),
+    to_char(b.block_time, 'YYYY-MM-DD') AS block_date,
+    t.block_number
+FROM transaction_logs t
+    LEFT JOIN blocks b ON t.block_sequence_id = b.id
+WHERE t.address = '0x141d6995556135d4997b2ff72eb443be300353bc'
+  AND t.event_name = 'OperatorAVSRegistrationStatusUpdated'
+  AND date_trunc('day', b.block_time) < TIMESTAMP '2024-12-13'
+    )
+select
+    operator,
+    avs,
+    status as registered,
+    log_index,
+    block_number
+from filtered
+```
+
 ## Expected results
 
 _See `generateExpectedResults.sql`_

internal/tests/testdata/operatorAvsRegistrationSnapshots/preprodRewardsV2_generateExpectedResults.sql

@@ -0,0 +1,67 @@
+COPY (
+     with filtered as (
+    select * from dbt_preprod_holesky_rewards.operator_avs_status
+    where block_time < '2024-12-11'
+),
+marked_statuses AS (
+    SELECT
+        operator,
+        avs,
+        registered,
+        block_time,
+        block_date,
+        LEAD(block_time) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_block_time,
+        LEAD(registered) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_registration_status,
+        LEAD(block_date) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_block_date,
+        LAG(registered) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS prev_registered,
+        LAG(block_date) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS prev_block_date
+    FROM filtered
+),
+     removed_same_day_deregistrations AS (
+         SELECT * from marked_statuses
+         WHERE NOT (
+             (registered = TRUE AND
+              COALESCE(next_registration_status = FALSE, false) AND
+              COALESCE(block_date = next_block_date, false)) OR
+             (registered = FALSE AND
+              COALESCE(prev_registered = TRUE, false) and
+              COALESCE(block_date = prev_block_date, false)
+                 )
+             )
+     ),
+     registration_periods AS (
+         SELECT
+             operator,
+             avs,
+             block_time AS start_time,
+             COALESCE(next_block_time, TIMESTAMP '2024-12-11') AS end_time,
+             registered
+         FROM removed_same_day_deregistrations
+         WHERE registered = TRUE
+     ),
+     registration_windows_extra as (
+         SELECT
+             operator,
+             avs,
+             date_trunc('day', start_time) + interval '1' day as start_time,
+             date_trunc('day', end_time) as end_time
+         FROM registration_periods
+     ),
+     operator_avs_registration_windows as (
+         SELECT * from registration_windows_extra
+         WHERE start_time != end_time
+     ),
+     cleaned_records AS (
+        SELECT * FROM operator_avs_registration_windows
+            WHERE start_time < end_time
+    ),
+    final_results as (
+        SELECT
+            operator,
+            avs,
+            to_char(d, 'YYYY-MM-DD') AS snapshot
+        FROM cleaned_records
+        CROSS JOIN generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS d
+        )
+        select * from final_results
+) TO STDOUT WITH DELIMITER ',' CSV HEADER;

internal/tests/testdata/operatorAvsSplitSnapshots/README.md

@@ -0,0 +1,18 @@
+## preprod rewardsv2
+
+```sql
+select
+    lower(arguments #>> '{1, Value}') as operator,
+    lower(arguments #>> '{2, Value}') as avs,
+    to_timestamp((output_data ->> 'activatedAt')::integer)::timestamp(6) as activated_at,
+    output_data ->> 'oldOperatorAVSSplitBips' as old_operator_avs_split_bips,
+    output_data ->> 'newOperatorAVSSplitBips' as new_operator_avs_split_bips,
+    block_number,
+    transaction_hash,
+    log_index
+from transaction_logs
+where
+    address = '0xb22ef643e1e067c994019a4c19e403253c05c2b0'
+    and event_name = 'OperatorAVSSplitBipsSet'
+order by block_number desc
+```

internal/tests/testdata/operatorDirectedRewardSubmissions/README.md

@@ -0,0 +1,57 @@
+## preprod rewards-v2
+
+```sql
+WITH strategies AS (
+    SELECT
+        tl.*,
+        lower(arguments #>> '{2, Value}') as reward_hash,
+        output_data->'operatorDirectedRewardsSubmission'->>'token' as token,
+    output_data->'operatorDirectedRewardsSubmission'->>'duration' as duration,
+    output_data->'operatorDirectedRewardsSubmission'->>'startTimestamp' as start_timestamp,
+    strategy_data,
+    strategy_idx - 1 as strategy_idx  -- Subtract 1 for 0-based indexing
+FROM transaction_logs as tl,
+    jsonb_array_elements(output_data->'operatorDirectedRewardsSubmission'->'strategiesAndMultipliers')
+WITH ORDINALITY AS t(strategy_data, strategy_idx)
+where
+    address = '0xb22ef643e1e067c994019a4c19e403253c05c2b0'
+  and event_name = 'OperatorDirectedAVSRewardsSubmissionCreated'
+    ),
+    operators AS (
+SELECT
+    lower(arguments #>> '{2, Value}') as reward_hash,
+    operator_data,
+    operator_data->>'operator' as operator,
+    output_data->'operatorDirectedRewardsSubmission' as rewards_submission,
+    operator_idx - 1 as operator_idx  -- Subtract 1 to make it 0-based indexing
+FROM transaction_logs,
+    jsonb_array_elements(output_data->'operatorDirectedRewardsSubmission'->'operatorRewards')
+WITH ORDINALITY AS t(operator_data, operator_idx)
+where
+    address = '0xb22ef643e1e067c994019a4c19e403253c05c2b0'
+  and event_name = 'OperatorDirectedAVSRewardsSubmissionCreated'
+    ),
+    joined_data as (
+SELECT
+    lower(arguments #>> '{1, Value}') as avs,
+    lower(arguments #>> '{2, Value}') as reward_hash,
+    strategies.token,
+    operator_data->>'operator' as operator,
+    operator_idx as operator_index,
+    operator_data->>'amount' as amount,
+    strategy_data->>'strategy' as strategy,
+    strategy_idx as strategy_index,
+    strategy_data->>'multiplier' as multiplier,
+    (to_timestamp((rewards_submission->>'startTimestamp')::int))::timestamp(6) as start_timestamp,
+    (rewards_submission->>'duration')::int as duration,
+    to_timestamp((rewards_submission->>'startTimestamp')::int + (rewards_submission->>'duration')::int)::timestamp(6) as end_timestamp,
+    block_number,
+    transaction_hash,
+    log_index
+FROM strategies
+    inner join operators on(
+    strategies.reward_hash = operators.reward_hash
+    )
+    )
+select * from joined_data
+```