From 9ddfb77d596df4ef88a4471c49a3884a896ce3f3 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Tue, 10 Sep 2024 17:02:28 -0400 Subject: [PATCH 01/21] add update affiliate info roundtable --- .../postgres/__tests__/helpers/constants.ts | 4 +- .../stores/affiliate-info-table.test.ts | 274 +++++++++++++++++- .../__tests__/stores/wallet-table.test.ts | 104 ++++--- ...410_change_fills_affiliaterevshare_type.ts | 16 + ...l_precision_and_add_totalreferredvolume.ts | 24 ++ indexer/packages/postgres/src/index.ts | 1 + .../src/models/affiliate-info-model.ts | 5 + .../src/stores/affiliate-info-table.ts | 163 +++++++++++ .../postgres/src/stores/wallet-table.ts | 3 +- .../src/types/affiliate-info-types.ts | 2 + .../postgres/src/types/db-model-types.ts | 1 + .../src/types/persistent-cache-types.ts | 5 + .../tasks/update-affiliate-info.test.ts | 244 ++++++++++++++++ .../tasks/update-wallet-total-volume.test.ts | 40 +-- .../src/tasks/update-affiliate-info.ts | 49 ++++ .../src/tasks/update-wallet-total-volume.ts | 27 +- 16 files changed, 867 insertions(+), 95 deletions(-) create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20240910101410_change_fills_affiliaterevshare_type.ts create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_totalreferredvolume.ts create mode 100644 indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts create mode 100644 indexer/services/roundtable/src/tasks/update-affiliate-info.ts diff --git a/indexer/packages/postgres/__tests__/helpers/constants.ts b/indexer/packages/postgres/__tests__/helpers/constants.ts index ad0663240c..dceca35523 100644 --- a/indexer/packages/postgres/__tests__/helpers/constants.ts +++ b/indexer/packages/postgres/__tests__/helpers/constants.ts @@ -556,7 +556,7 @@ export const defaultFill: FillCreateObject = { createdAtHeight: createdHeight, clientMetadata: '0', fee: '1.1', - affiliateRevShare: '1.1', + affiliateRevShare: '1.10', }; export const isolatedMarketFill: FillCreateObject = { @@ -974,6 +974,7 @@ export const defaultAffiliateInfo: AffiliateInfoCreateObject = { totalReferredUsers: 5, referredNetProtocolEarnings: '20.00', firstReferralBlockHeight: '1', + totalReferredVolume: '1000.00', }; export const defaultAffiliateInfo1: AffiliateInfoCreateObject = { @@ -985,4 +986,5 @@ export const defaultAffiliateInfo1: AffiliateInfoCreateObject = { totalReferredUsers: 5, referredNetProtocolEarnings: '21.00', firstReferralBlockHeight: '11', + totalReferredVolume: '1100.00', }; diff --git a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts index 7a8ac32bd7..13c2406ad0 100644 --- a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts @@ -1,7 +1,25 @@ -import { AffiliateInfoFromDatabase } from '../../src/types'; +import { PersistentCacheKeys, AffiliateInfoFromDatabase, Liquidity } from '../../src/types'; import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; -import { defaultAffiliateInfo, defaultAffiliateInfo1 } from '../helpers/constants'; +import { + defaultOrder, + defaultWallet, + defaultFill, + defaultWallet2, + defaultAffiliateInfo, + defaultAffiliateInfo1, + defaultTendermintEventId, + defaultTendermintEventId2, + defaultTendermintEventId3, + defaultTendermintEventId4, + vaultAddress, +} from '../helpers/constants'; import * as AffiliateInfoTable from '../../src/stores/affiliate-info-table'; +import * as OrderTable from '../../src/stores/order-table'; +import * as AffiliateReferredUsersTable from '../../src/stores/affiliate-referred-users-table'; +import * as FillTable from '../../src/stores/fill-table'; +import * as PersistentCacheTable from '../../src/stores/persistent-cache-table'; +import { seedData } from '../helpers/mock-generators'; +import { DateTime } from 'luxon'; describe('Affiliate info store', () => { beforeAll(async () => { @@ -56,13 +74,259 @@ describe('Affiliate info store', () => { ])); }); - it('Successfully finds an affiliate info', async () => { + it('Successfully finds affiliate info by Id', async () => { await AffiliateInfoTable.create(defaultAffiliateInfo); - const info: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( defaultAffiliateInfo.address, ); - expect(info).toEqual(expect.objectContaining(defaultAffiliateInfo)); }); + + describe('Affiliate info .updateInfo()', () => { + it('Successfully creates new affiliate info', async () => { + const referenceDt = await populateFillsAndReferrals(); + + // Perform update + await AffiliateInfoTable.updateInfo( + referenceDt.minus({ minutes: 2 }).toISO(), + referenceDt.toISO(), + ); + + // Get affiliate info (wallet2 is affiliate) + const updatedInfo: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( + defaultWallet2.address, + ); + + const expectedAffiliateInfo: AffiliateInfoFromDatabase = { + address: defaultWallet2.address, + affiliateEarnings: '1000', + referredMakerTrades: 1, + referredTakerTrades: 1, + totalReferredFees: '2000', + totalReferredUsers: 1, + referredNetProtocolEarnings: '1000', + firstReferralBlockHeight: '1', + totalReferredVolume: '2', + }; + + expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + }); + + it('Successfully updates/increments affiliate info, both stats and metadata', async () => { + const referenceDt = await populateFillsAndReferrals(); + + // Perform update: catches first 2 fills + await AffiliateInfoTable.updateInfo( + referenceDt.minus({ minutes: 3 }).toISO(), + referenceDt.minus({ minutes: 2 }).toISO(), + ); + + let updatedInfo: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( + defaultWallet2.address, + ); + let expectedAffiliateInfo: AffiliateInfoFromDatabase = { + address: defaultWallet2.address, + affiliateEarnings: '1000', + referredMakerTrades: 2, + referredTakerTrades: 0, + totalReferredFees: '2000', + totalReferredUsers: 1, + referredNetProtocolEarnings: '1000', + firstReferralBlockHeight: '1', + totalReferredVolume: '2', + }; + expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + + // Perform update: catches next 2 fills + await AffiliateInfoTable.updateInfo( + referenceDt.minus({ minutes: 2 }).toISO(), + referenceDt.minus({ minutes: 1 }).toISO(), + ); + + updatedInfo = await AffiliateInfoTable.findById( + defaultWallet2.address, + ); + expectedAffiliateInfo = { + address: defaultWallet2.address, + affiliateEarnings: '2000', + referredMakerTrades: 3, + referredTakerTrades: 1, + totalReferredFees: '4000', + totalReferredUsers: 1, + referredNetProtocolEarnings: '2000', + firstReferralBlockHeight: '1', + totalReferredVolume: '4', + }; + expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + + // Perform update: catches no fills but new affiliate referral + await AffiliateReferredUsersTable.create({ + affiliateAddress: defaultWallet2.address, + refereeAddress: vaultAddress, + referredAtBlock: '2', + }); + await AffiliateInfoTable.updateInfo( + referenceDt.minus({ minutes: 1 }).toISO(), + referenceDt.toISO(), + ); + updatedInfo = await AffiliateInfoTable.findById( + defaultWallet2.address, + ); + expectedAffiliateInfo = { + address: defaultWallet2.address, + affiliateEarnings: '2000', + referredMakerTrades: 3, + referredTakerTrades: 1, + totalReferredFees: '4000', + totalReferredUsers: 2, + referredNetProtocolEarnings: '2000', + firstReferralBlockHeight: '1', + totalReferredVolume: '4', + }; + expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + }); + + it('Successfully upserts persistent cache', async () => { + const referenceDt = await populateFillsAndReferrals(); + + // First update sets persistent cache + await AffiliateInfoTable.updateInfo( + referenceDt.minus({ minutes: 2 }).toISO(), + referenceDt.minus({ minutes: 1 }).toISO(), + ); + let persistentCache = await PersistentCacheTable.findById( + PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + ); + let lastUpdateTime = persistentCache?.value; + expect(lastUpdateTime).not.toBeUndefined(); + if (lastUpdateTime !== undefined) { + expect(lastUpdateTime).toEqual(referenceDt.minus({ minutes: 1 }).toISO()); + } + + // Second update upserts persistent cache + await AffiliateInfoTable.updateInfo( + referenceDt.minus({ minutes: 1 }).toISO(), + referenceDt.toISO(), + ); + persistentCache = await PersistentCacheTable.findById( + PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + ); + lastUpdateTime = persistentCache?.value; + expect(lastUpdateTime).not.toBeUndefined(); + if (lastUpdateTime !== undefined) { + expect(lastUpdateTime).toEqual(referenceDt.toISO()); + } + }); + + it('Does not use fills from before referal block height', async () => { + const referenceDt = DateTime.utc(); + + await seedData(); + await OrderTable.create(defaultOrder); + + // Referal at block 2 but fill is at block 1 + await AffiliateReferredUsersTable.create({ + affiliateAddress: defaultWallet2.address, + refereeAddress: defaultWallet.address, + referredAtBlock: '2', + }); + await FillTable.create({ + ...defaultFill, + liquidity: Liquidity.TAKER, + subaccountId: defaultOrder.subaccountId, + createdAt: referenceDt.toISO(), + createdAtHeight: '1', + eventId: defaultTendermintEventId, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }); + + await AffiliateInfoTable.updateInfo( + referenceDt.minus({ minutes: 1 }).toISO(), + referenceDt.toISO(), + ); + + const updatedInfo = await AffiliateInfoTable.findById( + defaultWallet2.address, + ); + // expect one referred user but no fill stats + const expectedAffiliateInfo = { + address: defaultWallet2.address, + affiliateEarnings: '0', + referredMakerTrades: 0, + referredTakerTrades: 0, + totalReferredFees: '0', + totalReferredUsers: 1, + referredNetProtocolEarnings: '0', + firstReferralBlockHeight: '2', + totalReferredVolume: '0', + }; + expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + }); + }); }); + +async function populateFillsAndReferrals(): Promise { + const referenceDt = DateTime.utc(); + + await seedData(); + + // defaultWallet2 will be affiliate and defaultWallet will be referee + await AffiliateReferredUsersTable.create({ + affiliateAddress: defaultWallet2.address, + refereeAddress: defaultWallet.address, + referredAtBlock: '1', + }); + + // Create order and fils for defaultWallet (referee) + await OrderTable.create(defaultOrder); + + await FillTable.create({ + ...defaultFill, + liquidity: Liquidity.TAKER, + subaccountId: defaultOrder.subaccountId, + createdAt: referenceDt.minus({ minutes: 1 }).toISO(), + eventId: defaultTendermintEventId, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }); + await FillTable.create({ + ...defaultFill, + liquidity: Liquidity.MAKER, + subaccountId: defaultOrder.subaccountId, + createdAt: referenceDt.minus({ minutes: 1 }).toISO(), + eventId: defaultTendermintEventId2, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }); + await FillTable.create({ + ...defaultFill, + liquidity: Liquidity.MAKER, // use uneven number of maker/taker + subaccountId: defaultOrder.subaccountId, + createdAt: referenceDt.minus({ minutes: 2 }).toISO(), + eventId: defaultTendermintEventId3, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }); + await FillTable.create({ + ...defaultFill, + liquidity: Liquidity.MAKER, + subaccountId: defaultOrder.subaccountId, + createdAt: referenceDt.minus({ minutes: 2 }).toISO(), + eventId: defaultTendermintEventId4, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }); + + return referenceDt; +} diff --git a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts index 4e74c5e55c..0961af39bd 100644 --- a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts @@ -1,4 +1,4 @@ -import { WalletFromDatabase } from '../../src/types'; +import { WalletFromDatabase, PersistentCacheKeys } from '../../src/types'; import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; import { DateTime } from 'luxon'; import { @@ -112,48 +112,68 @@ describe('Wallet store', () => { expect(wallets[0]).toEqual(expect.objectContaining(defaultWallet3)); }); - it('Successfully updates totalVolume for time window multiple times', async () => { - const firstFillTime = await populateWalletSubaccountFill(); - - // Update totalVolume for a time window that covers all fills - await WalletTable.updateTotalVolume( - firstFillTime.minus({ hours: 1 }).toISO(), // need to minus because left bound is exclusive - firstFillTime.plus({ hours: 1 }).toISO(), - ); - let wallet = await WalletTable.findById(defaultWallet.address); - expect(wallet).toEqual(expect.objectContaining({ - ...defaultWallet, - totalVolume: '103', - })); - - // Update totalVolume for a time window that excludes some fills - // For convenience, we will reuse the existing fills data. The total volume calculated in this - // window should be added to the total volume above. - await WalletTable.updateTotalVolume( - firstFillTime.toISO(), // exclusive -> filters out first fill from each subaccount - firstFillTime.plus({ minutes: 2 }).toISO(), - ); - wallet = await WalletTable.findById(defaultWallet.address); - expect(wallet).toEqual(expect.objectContaining({ - ...defaultWallet, - totalVolume: '105', // 103 + 2 - })); - }); - - it('Successfully updates totalVolumeUpdateTime in persistent cache', async () => { - const leftBound = DateTime.utc().minus({ hours: 1 }); - const rightBound = DateTime.utc(); - await WalletTable.updateTotalVolume(leftBound.toISO(), rightBound.toISO()); - - const persistentCache = await PersistentCacheTable.findById('totalVolumeUpdateTime'); - const lastUpdateTime = persistentCache?.value - ? DateTime.fromISO(persistentCache.value) - : undefined; + describe('Wallet .updateTotalVolume()', () => { + it('Successfully updates totalVolume for time window multiple times', async () => { + const firstFillTime = await populateWalletSubaccountFill(); + + // Update totalVolume for a time window that covers all fills + await WalletTable.updateTotalVolume( + firstFillTime.minus({ hours: 1 }).toISO(), // need to minus because left bound is exclusive + firstFillTime.plus({ hours: 1 }).toISO(), + ); + let wallet = await WalletTable.findById(defaultWallet.address); + expect(wallet).toEqual(expect.objectContaining({ + ...defaultWallet, + totalVolume: '103', + })); + + // Update totalVolume for a time window that excludes some fills + // For convenience, we will reuse the existing fills data. The total volume calculated in this + // window should be added to the total volume above. + await WalletTable.updateTotalVolume( + firstFillTime.toISO(), // exclusive -> filters out first fill from each subaccount + firstFillTime.plus({ minutes: 2 }).toISO(), + ); + wallet = await WalletTable.findById(defaultWallet.address); + expect(wallet).toEqual(expect.objectContaining({ + ...defaultWallet, + totalVolume: '105', // 103 + 2 + })); + }); - expect(lastUpdateTime).not.toBeUndefined(); - if (lastUpdateTime?.toMillis() !== undefined) { - expect(lastUpdateTime.toMillis()).toEqual(rightBound.toMillis()); - } + it('Successfully upserts persistent cache', async () => { + const referenceDt = DateTime.utc(); + + // Sets initial persistent cache value + let leftBound = referenceDt.minus({ hours: 2 }); + let rightBound = referenceDt.minus({ hours: 1 }); + + await WalletTable.updateTotalVolume(leftBound.toISO(), rightBound.toISO()); + + let persistentCache = await PersistentCacheTable.findById( + PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, + ); + let lastUpdateTime = persistentCache?.value; + expect(lastUpdateTime).not.toBeUndefined(); + if (lastUpdateTime !== undefined) { + expect(lastUpdateTime).toEqual(rightBound.toISO()); + } + + // Updates persistent cache value + leftBound = referenceDt.minus({ hours: 1 }); + rightBound = referenceDt; + + await WalletTable.updateTotalVolume(leftBound.toISO(), rightBound.toISO()); + + persistentCache = await PersistentCacheTable.findById( + PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, + ); + lastUpdateTime = persistentCache?.value; + expect(lastUpdateTime).not.toBeUndefined(); + if (lastUpdateTime !== undefined) { + expect(lastUpdateTime).toEqual(rightBound.toISO()); + } + }); }); }); diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240910101410_change_fills_affiliaterevshare_type.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240910101410_change_fills_affiliaterevshare_type.ts new file mode 100644 index 0000000000..dd61af5d3b --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240910101410_change_fills_affiliaterevshare_type.ts @@ -0,0 +1,16 @@ +import * as Knex from 'knex'; + +// No data has been stored added at time of commit +export async function up(knex: Knex): Promise { + return knex.schema.alterTable('fills', (table) => { + // decimal('columnName') has is 8,2 precision and scale + // decimal('columnName', null) has variable precision and scale + table.decimal('affiliateRevShare', null).notNullable().defaultTo(0).alter(); + }); +} + +export async function down(knex: Knex): Promise { + return knex.schema.alterTable('fills', (table) => { + table.string('affiliateRevShare').notNullable().defaultTo('0').alter(); + }); +} diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_totalreferredvolume.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_totalreferredvolume.ts new file mode 100644 index 0000000000..1aa4cd1ea0 --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_totalreferredvolume.ts @@ -0,0 +1,24 @@ +import * as Knex from 'knex'; + +// No data has been stored added at time of commit +export async function up(knex: Knex): Promise { + return knex.schema.alterTable('affiliate_info', (table) => { + // decimal('columnName') has is 8,2 precision and scale + // decimal('columnName', null) has variable precision and scale + table.decimal('affiliateEarnings', null).notNullable().defaultTo(0).alter(); + table.decimal('totalReferredFees', null).notNullable().defaultTo(0).alter(); + table.decimal('referredNetProtocolEarnings', null).notNullable().defaultTo(0).alter(); + + table.decimal('totalReferredVolume', null).notNullable().defaultTo(0); + }); +} + +export async function down(knex: Knex): Promise { + return knex.schema.alterTable('affiliate_info', (table) => { + table.decimal('affiliateEarnings').notNullable().defaultTo(0).alter(); + table.decimal('totalReferredFees').notNullable().defaultTo(0).alter(); + table.decimal('referredNetProtocolEarnings').notNullable().defaultTo(0).alter(); + + table.dropColumn('totalReferredVolume'); + }); +} diff --git a/indexer/packages/postgres/src/index.ts b/indexer/packages/postgres/src/index.ts index 9a27d297b3..4018717ee6 100644 --- a/indexer/packages/postgres/src/index.ts +++ b/indexer/packages/postgres/src/index.ts @@ -47,6 +47,7 @@ export * as LeaderboardPnlTable from './stores/leaderboard-pnl-table'; export * as SubaccountUsernamesTable from './stores/subaccount-usernames-table'; export * as PersistentCacheTable from './stores/persistent-cache-table'; export * as AffiliateReferredUsersTable from './stores/affiliate-referred-users-table'; +export * as AffiliateInfoTable from './stores/affiliate-info-table'; export * as perpetualMarketRefresher from './loops/perpetual-market-refresher'; export * as assetRefresher from './loops/asset-refresher'; diff --git a/indexer/packages/postgres/src/models/affiliate-info-model.ts b/indexer/packages/postgres/src/models/affiliate-info-model.ts index 1fe37b2b61..cc01227894 100644 --- a/indexer/packages/postgres/src/models/affiliate-info-model.ts +++ b/indexer/packages/postgres/src/models/affiliate-info-model.ts @@ -23,6 +23,7 @@ export default class AffiliateInfoModel extends BaseModel { 'totalReferredUsers', 'referredNetProtocolEarnings', 'firstReferralBlockHeight', + 'totalReferredVolume', ], properties: { address: { type: 'string' }, @@ -33,6 +34,7 @@ export default class AffiliateInfoModel extends BaseModel { totalReferredUsers: { type: 'int' }, referredNetProtocolEarnings: { type: 'string', pattern: NonNegativeNumericPattern }, firstReferralBlockHeight: { type: 'string', pattern: NonNegativeNumericPattern }, + totalReferredVolume: { type: 'string', pattern: NonNegativeNumericPattern }, }, }; } @@ -53,6 +55,7 @@ export default class AffiliateInfoModel extends BaseModel { totalReferredUsers: 'int', referredNetProtocolEarnings: 'string', firstReferralBlockHeight: 'string', + totalReferredVolume: 'string', }; } @@ -73,4 +76,6 @@ export default class AffiliateInfoModel extends BaseModel { referredNetProtocolEarnings!: string; firstReferralBlockHeight!: string; + + totalReferredVolume!: string; } diff --git a/indexer/packages/postgres/src/stores/affiliate-info-table.ts b/indexer/packages/postgres/src/stores/affiliate-info-table.ts index 3f6695592a..dcac551a63 100644 --- a/indexer/packages/postgres/src/stores/affiliate-info-table.ts +++ b/indexer/packages/postgres/src/stores/affiliate-info-table.ts @@ -1,6 +1,7 @@ import { QueryBuilder } from 'objection'; import { DEFAULT_POSTGRES_OPTIONS } from '../constants'; +import { knexReadReplica } from '../helpers/knex'; import { setupBaseQuery, verifyAllRequiredFields } from '../helpers/stores-helpers'; import Transaction from '../helpers/transaction'; import AffiliateInfoModel from '../models/affiliate-info-model'; @@ -13,6 +14,8 @@ import { AffiliateInfoCreateObject, AffiliateInfoFromDatabase, AffiliateInfoQueryConfig, + Liquidity, + PersistentCacheKeys, } from '../types'; export async function findAll( @@ -92,3 +95,163 @@ export async function findById( .findById(address) .returning('*'); } + +export async function updateInfo( + windowStartTs: string, // exclusive + windowEndTs: string, // inclusive +) : Promise { + + await knexReadReplica.getConnection().raw( + ` +BEGIN; + +-- Get metadata for all affiliates +-- STEP 1: Aggregate affiliate_referred_users +WITH affiliate_metadata AS ( + SELECT + "affiliateAddress", + COUNT(*) AS "totalReferredUsers", + MIN("referredAtBlock") AS "firstReferralBlockHeight" + FROM + affiliate_referred_users + GROUP BY + "affiliateAddress" +), + +-- Calculate fill related stats for affiliates +-- Step 2a: Inner join affiliate_referred_users with subaccounts to get subaccounts referred by the affiliate +affiliate_referred_subaccounts AS ( + SELECT + affiliate_referred_users."affiliateAddress", + affiliate_referred_users."referredAtBlock", + subaccounts."id" + FROM + affiliate_referred_users + INNER JOIN + subaccounts + ON + affiliate_referred_users."refereeAddress" = subaccounts."address" +), + +-- Step 2b: Filter fills by time window +filtered_fills AS ( + SELECT + fills."subaccountId", + fills."liquidity", + fills."createdAt", + CAST(fills."fee" AS decimal) AS "fee", + fills."affiliateRevShare", + fills."createdAtHeight", + fills."price", + fills."size" + FROM + fills + WHERE + fills."createdAt" > '${windowStartTs}' + AND fills."createdAt" <= '${windowEndTs}' +), + +-- Step 2c: Inner join filtered_fills with affiliate_referred_subaccounts and filter +affiliate_fills AS ( + SELECT + filtered_fills."subaccountId", + filtered_fills."liquidity", + filtered_fills."createdAt", + filtered_fills."fee", + filtered_fills."affiliateRevShare", + filtered_fills."price", + filtered_fills."size", + affiliate_referred_subaccounts."affiliateAddress", + affiliate_referred_subaccounts."referredAtBlock" + FROM + filtered_fills + INNER JOIN + affiliate_referred_subaccounts + ON + filtered_fills."subaccountId" = affiliate_referred_subaccounts."id" + WHERE + filtered_fills."createdAtHeight" >= affiliate_referred_subaccounts."referredAtBlock" +), + +-- Step 2d: Groupby to get affiliate level stats +affiliate_stats AS ( + SELECT + affiliate_fills."affiliateAddress", + SUM(affiliate_fills."fee") AS "totalReferredFees", + SUM(affiliate_fills."affiliateRevShare") AS "affiliateEarnings", + SUM(affiliate_fills."fee") - SUM(affiliate_fills."affiliateRevShare") AS "referredNetProtocolEarnings", + COUNT(CASE WHEN affiliate_fills."liquidity" = '${Liquidity.MAKER}' THEN 1 END) AS "referredMakerTrades", + COUNT(CASE WHEN affiliate_fills."liquidity" = '${Liquidity.TAKER}' THEN 1 END) AS "referredTakerTrades", + SUM(affiliate_fills."price" * affiliate_fills."size") AS "totalReferredVolume" + FROM + affiliate_fills + GROUP BY + affiliate_fills."affiliateAddress" +), + +-- Prepare to update affiliate_info +-- STEP 3a: Left join affiliate_stats onto affiliate_metadata. affiliate_stats only has values for +-- addresses with fills in the time window +affiliate_info_update AS ( + SELECT + affiliate_metadata."affiliateAddress", + affiliate_metadata."totalReferredUsers", + affiliate_metadata."firstReferralBlockHeight", + COALESCE(affiliate_stats."totalReferredFees", 0) AS "totalReferredFees", + COALESCE(affiliate_stats."affiliateEarnings", 0) AS "affiliateEarnings", + COALESCE(affiliate_stats."referredNetProtocolEarnings", 0) AS "referredNetProtocolEarnings", + COALESCE(affiliate_stats."referredMakerTrades", 0) AS "referredMakerTrades", + COALESCE(affiliate_stats."referredTakerTrades", 0) AS "referredTakerTrades", + COALESCE(affiliate_stats."totalReferredVolume", 0) AS "totalReferredVolume" + FROM + affiliate_metadata + LEFT JOIN + affiliate_stats + ON affiliate_metadata."affiliateAddress" = affiliate_stats."affiliateAddress" +) + +-- Step 3b: Update/upsert the affiliate info table with the new stats +INSERT INTO affiliate_info ( + "address", + "totalReferredUsers", + "firstReferralBlockHeight", + "affiliateEarnings", + "referredMakerTrades", + "referredTakerTrades", + "totalReferredFees", + "referredNetProtocolEarnings", + "totalReferredVolume" +) +SELECT + "affiliateAddress", + "totalReferredUsers", + "firstReferralBlockHeight", + "affiliateEarnings", + "referredMakerTrades", + "referredTakerTrades", + "totalReferredFees", + "referredNetProtocolEarnings", + "totalReferredVolume" +FROM + affiliate_info_update +ON CONFLICT ("address") +DO UPDATE SET + "totalReferredUsers" = EXCLUDED."totalReferredUsers", + "firstReferralBlockHeight" = EXCLUDED."firstReferralBlockHeight", + "affiliateEarnings" = affiliate_info."affiliateEarnings" + EXCLUDED."affiliateEarnings", + "referredMakerTrades" = affiliate_info."referredMakerTrades" + EXCLUDED."referredMakerTrades", + "referredTakerTrades" = affiliate_info."referredTakerTrades" + EXCLUDED."referredTakerTrades", + "totalReferredFees" = affiliate_info."totalReferredFees" + EXCLUDED."totalReferredFees", + "referredNetProtocolEarnings" = affiliate_info."referredNetProtocolEarnings" + EXCLUDED."referredNetProtocolEarnings", + "totalReferredVolume" = affiliate_info."totalReferredVolume" + EXCLUDED."totalReferredVolume"; + +-- Step 5: Upsert new affiliateInfoLastUpdateTime to persistent_cache table +INSERT INTO persistent_cache (key, value) +VALUES ('${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME}', '${windowEndTs}') +ON CONFLICT (key) +DO UPDATE SET value = EXCLUDED.value; + +COMMIT; + `, + ); +} diff --git a/indexer/packages/postgres/src/stores/wallet-table.ts b/indexer/packages/postgres/src/stores/wallet-table.ts index 6e96b19dae..7d0cd41edb 100644 --- a/indexer/packages/postgres/src/stores/wallet-table.ts +++ b/indexer/packages/postgres/src/stores/wallet-table.ts @@ -15,6 +15,7 @@ import { WalletFromDatabase, WalletQueryConfig, WalletUpdateObject, + PersistentCacheKeys, } from '../types'; export async function findAll( @@ -161,7 +162,7 @@ export async function updateTotalVolume( -- Step 5: Upsert new totalVolumeUpdateTime to persistent_cache table INSERT INTO persistent_cache (key, value) - VALUES ('totalVolumeUpdateTime', '${windowEndTs}') + VALUES ('${PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME}', '${windowEndTs}') ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value; diff --git a/indexer/packages/postgres/src/types/affiliate-info-types.ts b/indexer/packages/postgres/src/types/affiliate-info-types.ts index 4c7f11d108..c10e8e0610 100644 --- a/indexer/packages/postgres/src/types/affiliate-info-types.ts +++ b/indexer/packages/postgres/src/types/affiliate-info-types.ts @@ -7,6 +7,7 @@ export interface AffiliateInfoCreateObject { totalReferredUsers: number, referredNetProtocolEarnings: string, firstReferralBlockHeight: string, + totalReferredVolume: string, } export enum AffiliateInfoColumns { @@ -18,4 +19,5 @@ export enum AffiliateInfoColumns { totalReferredUsers = 'totalReferredUsers', referredNetProtocolEarnings = 'referredNetProtocolEarnings', firstReferralBlockHeight = 'firstReferralBlockHeight', + totalReferredVolume = 'totalReferredVolume', } diff --git a/indexer/packages/postgres/src/types/db-model-types.ts b/indexer/packages/postgres/src/types/db-model-types.ts index 785511c596..0b4cbc8b84 100644 --- a/indexer/packages/postgres/src/types/db-model-types.ts +++ b/indexer/packages/postgres/src/types/db-model-types.ts @@ -287,6 +287,7 @@ export interface AffiliateInfoFromDatabase { totalReferredUsers: number, referredNetProtocolEarnings: string, firstReferralBlockHeight: string, + totalReferredVolume: string, } export interface AffiliateReferredUserFromDatabase { diff --git a/indexer/packages/postgres/src/types/persistent-cache-types.ts b/indexer/packages/postgres/src/types/persistent-cache-types.ts index 6022e6764d..0ae79ae7e1 100644 --- a/indexer/packages/postgres/src/types/persistent-cache-types.ts +++ b/indexer/packages/postgres/src/types/persistent-cache-types.ts @@ -7,3 +7,8 @@ export enum PersistentCacheColumns { key = 'key', value = 'value', } + +export enum PersistentCacheKeys { + TOTAL_VOLUME_UPDATE_TIME = 'totalVolumeUpdateTime', + AFFILIATE_INFO_UPDATE_TIME = 'affiliateInfoUpdateTime', +} diff --git a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts new file mode 100644 index 0000000000..6c6f3ebac7 --- /dev/null +++ b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts @@ -0,0 +1,244 @@ +import { + dbHelpers, + testConstants, + testMocks, + PersistentCacheTable, + FillTable, + OrderTable, + PersistentCacheKeys, + AffiliateReferredUsersTable, + AffiliateInfoFromDatabase, + AffiliateInfoTable, + Liquidity, +} from '@dydxprotocol-indexer/postgres'; +import affiliateInfoUpdateTask from '../../src/tasks/update-affiliate-info'; +import { DateTime } from 'luxon'; + +describe('update-affiliate-info', () => { + beforeAll(async () => { + await dbHelpers.migrate(); + await dbHelpers.clearData(); + }); + + beforeEach(async () => { + await testMocks.seedData(); + await OrderTable.create(testConstants.defaultOrder); + }); + + afterAll(async () => { + await dbHelpers.teardown(); + jest.resetAllMocks(); + }); + + afterEach(async () => { + await dbHelpers.clearData(); + }); + + it('Successfully updates affiliate info and persistent cache multiple times', async () => { + const startDt = DateTime.utc(); + + // Set persistent cache affiliateInfoUpdateTIme so task does not use backfill windows + await PersistentCacheTable.create({ + key: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + value: startDt.toISO(), + }); + + // First task run: add refereal w/o any fills + // defaultWallet2 will be affiliate and defaultWallet will be referee + await AffiliateReferredUsersTable.create({ + affiliateAddress: testConstants.defaultWallet2.address, + refereeAddress: testConstants.defaultWallet.address, + referredAtBlock: '1', + }); + await affiliateInfoUpdateTask(); + + let updatedInfo: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( + testConstants.defaultWallet2.address, + ); + let expectedAffiliateInfo: AffiliateInfoFromDatabase = { + address: testConstants.defaultWallet2.address, + affiliateEarnings: '0', + referredMakerTrades: 0, + referredTakerTrades: 0, + totalReferredFees: '0', + totalReferredUsers: 1, + referredNetProtocolEarnings: '0', + firstReferralBlockHeight: '1', + totalReferredVolume: '0', + }; + expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + + // Check that persistent cache updated + const lastUpdateTime1 = await getAffiliateInfoUpdateTime(); + if (lastUpdateTime1 !== undefined) { + expect(lastUpdateTime1.toMillis()) + .toBeGreaterThan(startDt.toMillis()); + } + + // Second task run: one new fill and one new referral + await FillTable.create({ + ...testConstants.defaultFill, + liquidity: Liquidity.TAKER, + createdAt: DateTime.utc().toISO(), + eventId: testConstants.defaultTendermintEventId, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }); + await AffiliateReferredUsersTable.create({ + affiliateAddress: testConstants.defaultWallet2.address, + refereeAddress: testConstants.defaultWallet3.address, + referredAtBlock: '2', + }); + + await affiliateInfoUpdateTask(); + + updatedInfo = await AffiliateInfoTable.findById( + testConstants.defaultWallet2.address, + ); + expectedAffiliateInfo = { + address: testConstants.defaultWallet2.address, + affiliateEarnings: '500', + referredMakerTrades: 0, + referredTakerTrades: 1, + totalReferredFees: '1000', + totalReferredUsers: 2, + referredNetProtocolEarnings: '500', + firstReferralBlockHeight: '1', + totalReferredVolume: '1', + }; + expect(updatedInfo).toEqual(expectedAffiliateInfo); + const lastUpdateTime2 = await getAffiliateInfoUpdateTime(); + if (lastUpdateTime2 !== undefined && lastUpdateTime1 !== undefined) { + expect(lastUpdateTime2.toMillis()) + .toBeGreaterThan(lastUpdateTime1.toMillis()); + } + }); + + it('Successfully backfills from past date', async () => { + const currentDt = DateTime.utc(); + + // Set persistent cache to 3 weeks ago to emulate backfill from 3 weeks. + await PersistentCacheTable.create({ + key: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + value: currentDt.minus({ weeks: 3 }).toISO(), + }); + + // defaultWallet2 will be affiliate and defaultWallet will be referee + await AffiliateReferredUsersTable.create({ + affiliateAddress: testConstants.defaultWallet2.address, + refereeAddress: testConstants.defaultWallet.address, + referredAtBlock: '1', + }); + + // Fills spannings 2 weeks + await FillTable.create({ + ...testConstants.defaultFill, + liquidity: Liquidity.TAKER, + createdAt: currentDt.minus({ weeks: 1 }).toISO(), + eventId: testConstants.defaultTendermintEventId, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }); + await FillTable.create({ + ...testConstants.defaultFill, + liquidity: Liquidity.TAKER, + createdAt: currentDt.minus({ weeks: 2 }).toISO(), + eventId: testConstants.defaultTendermintEventId2, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }); + + // Simulate backfill + let backfillTime = await getAffiliateInfoUpdateTime(); + while (backfillTime !== undefined && DateTime.fromISO(backfillTime.toISO()) < currentDt) { + await affiliateInfoUpdateTask(); + backfillTime = await getAffiliateInfoUpdateTime(); + } + + const expectedAffiliateInfo = { + address: testConstants.defaultWallet2.address, + affiliateEarnings: '1000', + referredMakerTrades: 0, + referredTakerTrades: 2, + totalReferredFees: '2000', + totalReferredUsers: 1, + referredNetProtocolEarnings: '1000', + firstReferralBlockHeight: '1', + totalReferredVolume: '2', + }; + const updatedInfo = await AffiliateInfoTable.findById(testConstants.defaultWallet2.address); + expect(updatedInfo).toEqual(expectedAffiliateInfo); + }); + + it('Successfully backfills on first run', async () => { + // Leave persistent cache affiliateInfoUpdateTime empty and create fills around + // `defaultLastUpdateTime` value to emulate backfilling from very beginning + expect(await getAffiliateInfoUpdateTime()).toBeUndefined(); + + const referenceDt = DateTime.fromISO('2020-01-01T00:00:00Z'); + + // defaultWallet2 will be affiliate and defaultWallet will be referee + await AffiliateReferredUsersTable.create({ + affiliateAddress: testConstants.defaultWallet2.address, + refereeAddress: testConstants.defaultWallet.address, + referredAtBlock: '1', + }); + + // Fills spannings 2 weeks after referenceDt + await FillTable.create({ + ...testConstants.defaultFill, + liquidity: Liquidity.TAKER, + createdAt: referenceDt.plus({ weeks: 1 }).toISO(), + eventId: testConstants.defaultTendermintEventId, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }); + await FillTable.create({ + ...testConstants.defaultFill, + liquidity: Liquidity.TAKER, + createdAt: referenceDt.plus({ weeks: 2 }).toISO(), + eventId: testConstants.defaultTendermintEventId2, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }); + + // Simulate 20 roundtable runs (this is enough to backfill all the fills) + for (let i = 0; i < 20; i++) { + await affiliateInfoUpdateTask(); + } + + const expectedAffiliateInfo = { + address: testConstants.defaultWallet2.address, + affiliateEarnings: '1000', + referredMakerTrades: 0, + referredTakerTrades: 2, + totalReferredFees: '2000', + totalReferredUsers: 1, + referredNetProtocolEarnings: '1000', + firstReferralBlockHeight: '1', + totalReferredVolume: '2', + }; + const updatedInfo = await AffiliateInfoTable.findById(testConstants.defaultWallet2.address); + expect(updatedInfo).toEqual(expectedAffiliateInfo); + }); +}); + +async function getAffiliateInfoUpdateTime(): Promise { + const persistentCache = await PersistentCacheTable.findById( + PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + ); + const lastUpdateTime = persistentCache?.value + ? DateTime.fromISO(persistentCache.value) + : undefined; + return lastUpdateTime; +} diff --git a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts index edb61608c0..16455b2b15 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts @@ -3,10 +3,10 @@ import { testConstants, testMocks, WalletTable, - SubaccountTable, PersistentCacheTable, FillTable, OrderTable, + PersistentCacheKeys, } from '@dydxprotocol-indexer/postgres'; import walletTotalVolumeUpdateTask from '../../src/tasks/update-wallet-total-volume'; import { DateTime } from 'luxon'; @@ -33,22 +33,16 @@ describe('update-wallet-total-volume', () => { }); it('Successfully updates totalVolume multiple times', async () => { - const defaultSubaccountId = await SubaccountTable.findAll( - { subaccountNumber: testConstants.defaultSubaccount.subaccountNumber }, - [], - {}, - ); // Set persistent cache totalVolumeUpdateTime so walletTotalVolumeUpdateTask() does not attempt // to backfill await PersistentCacheTable.create({ - key: 'totalVolumeUpdateTime', + key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, value: DateTime.utc().toISO(), }); // First task run: one new fill await FillTable.create({ ...testConstants.defaultFill, - subaccountId: defaultSubaccountId[0].id, createdAt: DateTime.utc().toISO(), eventId: testConstants.defaultTendermintEventId, price: '1', @@ -72,7 +66,6 @@ describe('update-wallet-total-volume', () => { // Third task run: one new fill await FillTable.create({ ...testConstants.defaultFill, - subaccountId: defaultSubaccountId[0].id, createdAt: DateTime.utc().toISO(), eventId: testConstants.defaultTendermintEventId2, price: '1', @@ -90,7 +83,7 @@ describe('update-wallet-total-volume', () => { // Set persistent cache totalVolumeUpdateTime so walletTotalVolumeUpdateTask() does not attempt // to backfill await PersistentCacheTable.create({ - key: 'totalVolumeUpdateTime', + key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, value: DateTime.utc().toISO(), }); @@ -110,16 +103,10 @@ describe('update-wallet-total-volume', () => { it('Successfully backfills from past date', async () => { const currentDt: DateTime = DateTime.utc(); - const defaultSubaccountId = await SubaccountTable.findAll( - { subaccountNumber: testConstants.defaultSubaccount.subaccountNumber }, - [], - {}, - ); // Create 3 fills spanning 2 weeks in the past await FillTable.create({ ...testConstants.defaultFill, - subaccountId: defaultSubaccountId[0].id, createdAt: currentDt.toISO(), eventId: testConstants.defaultTendermintEventId, price: '1', @@ -127,7 +114,6 @@ describe('update-wallet-total-volume', () => { }); await FillTable.create({ ...testConstants.defaultFill, - subaccountId: defaultSubaccountId[0].id, createdAt: currentDt.minus({ weeks: 1 }).toISO(), eventId: testConstants.defaultTendermintEventId2, price: '2', @@ -135,7 +121,6 @@ describe('update-wallet-total-volume', () => { }); await FillTable.create({ ...testConstants.defaultFill, - subaccountId: defaultSubaccountId[0].id, createdAt: currentDt.minus({ weeks: 2 }).toISO(), eventId: testConstants.defaultTendermintEventId3, price: '3', @@ -144,7 +129,7 @@ describe('update-wallet-total-volume', () => { // Set persistent cache totalVolumeUpdateTime to 3 weeks ago to emulate backfill from 3 weeks. await PersistentCacheTable.create({ - key: 'totalVolumeUpdateTime', + key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, value: currentDt.minus({ weeks: 3 }).toISO(), }); @@ -162,12 +147,6 @@ describe('update-wallet-total-volume', () => { }); it('Successfully backfills on first run', async () => { - const defaultSubaccountId = await SubaccountTable.findAll( - { subaccountNumber: testConstants.defaultSubaccount.subaccountNumber }, - [], - {}, - ); - // Leave persistent cache totalVolumeUpdateTime empty and create fills around // `defaultLastUpdateTime` value to emulate backfilling from very beginning expect(await getTotalVolumeUpdateTime()).toBeUndefined(); @@ -176,7 +155,6 @@ describe('update-wallet-total-volume', () => { await FillTable.create({ ...testConstants.defaultFill, - subaccountId: defaultSubaccountId[0].id, createdAt: referenceDt.plus({ days: 1 }).toISO(), eventId: testConstants.defaultTendermintEventId, price: '1', @@ -184,7 +162,6 @@ describe('update-wallet-total-volume', () => { }); await FillTable.create({ ...testConstants.defaultFill, - subaccountId: defaultSubaccountId[0].id, createdAt: referenceDt.plus({ days: 2 }).toISO(), eventId: testConstants.defaultTendermintEventId2, price: '2', @@ -192,7 +169,6 @@ describe('update-wallet-total-volume', () => { }); await FillTable.create({ ...testConstants.defaultFill, - subaccountId: defaultSubaccountId[0].id, createdAt: referenceDt.plus({ days: 3 }).toISO(), eventId: testConstants.defaultTendermintEventId3, price: '3', @@ -213,9 +189,11 @@ describe('update-wallet-total-volume', () => { }); async function getTotalVolumeUpdateTime(): Promise { - const persistentCache = await PersistentCacheTable.findById('totalVolumeUpdateTime'); - const lastUpdateTime1 = persistentCache?.value + const persistentCache = await PersistentCacheTable.findById( + PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, + ); + const lastUpdateTime = persistentCache?.value ? DateTime.fromISO(persistentCache.value) : undefined; - return lastUpdateTime1; + return lastUpdateTime; } diff --git a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts new file mode 100644 index 0000000000..4dff9e92f6 --- /dev/null +++ b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts @@ -0,0 +1,49 @@ +import { logger, stats } from '@dydxprotocol-indexer/base'; +import { PersistentCacheTable, AffiliateInfoTable, PersistentCacheKeys } from '@dydxprotocol-indexer/postgres'; +import { DateTime } from 'luxon'; + +import config from '../config'; + +const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; + +/** + * Update the affiliate info for all affiliate addresses. + */ +export default async function runTask(): Promise { + try { + const start = Date.now(); + const persistentCacheEntry = await PersistentCacheTable.findById( + PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + ); + + if (!persistentCacheEntry) { + logger.info({ + at: 'update-affiliate-info#runTask', + message: `No previous ${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME} found in persistent cache table. Will use default value: ${defaultLastUpdateTime}`, + }); + } + + const lastUpdateTime = DateTime.fromISO(persistentCacheEntry + ? persistentCacheEntry.value + : defaultLastUpdateTime); + let windowEndTime = DateTime.utc(); + + // During backfilling, we process one day at a time to reduce roundtable runtime. + if (windowEndTime > lastUpdateTime.plus({ days: 1 })) { + windowEndTime = lastUpdateTime.plus({ days: 1 }); + } + + await AffiliateInfoTable.updateInfo(lastUpdateTime.toISO(), windowEndTime.toISO()); + + stats.timing( + `${config.SERVICE_NAME}.update_affiliate_info_timing`, + Date.now() - start, + ); + } catch (error) { + logger.error({ + at: 'update-affiliate-info#runTask', + message: 'Error when updating affiliate info in affiliate_info table', + error, + }); + } +} diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index 5229d068dd..d14fa2fce8 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -1,42 +1,39 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; -import { PersistentCacheTable, WalletTable } from '@dydxprotocol-indexer/postgres'; +import { PersistentCacheTable, WalletTable, PersistentCacheKeys } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; import config from '../config'; const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; -const persistentCacheKey: string = 'totalVolumeUpdateTime'; /** - * Update the total volume for each address in the wallet table. + * Update the total volume for each addresses in the wallet table who filled recently. */ export default async function runTask(): Promise { try { const start = Date.now(); - const persistentCacheEntry = await PersistentCacheTable.findById(persistentCacheKey); + const persistentCacheEntry = await PersistentCacheTable.findById( + PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, + ); if (!persistentCacheEntry) { logger.info({ - at: 'update-address-total-volume#runTask', - message: `No previous totalVolumeUpdateTime found in persistent cache table. Will use default value: ${defaultLastUpdateTime}`, + at: 'update-wallet-total-volume#runTask', + message: `No previous ${PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME} found in persistent cache table. Will use default value: ${defaultLastUpdateTime}`, }); } const lastUpdateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); - let currentTime = DateTime.utc(); + let windowEndTime = DateTime.utc(); // During backfilling, we process one day at a time to reduce roundtable runtime. - if (currentTime > lastUpdateTime.plus({ days: 1 })) { - currentTime = lastUpdateTime.plus({ days: 1 }); + if (windowEndTime > lastUpdateTime.plus({ days: 1 })) { + windowEndTime = lastUpdateTime.plus({ days: 1 }); } - await WalletTable.updateTotalVolume(lastUpdateTime.toISO(), currentTime.toISO()); - await PersistentCacheTable.upsert({ - key: persistentCacheKey, - value: currentTime.toISO(), - }); + await WalletTable.updateTotalVolume(lastUpdateTime.toISO(), windowEndTime.toISO()); stats.timing( `${config.SERVICE_NAME}.update_wallet_total_volume_timing`, @@ -44,7 +41,7 @@ export default async function runTask(): Promise { ); } catch (error) { logger.error({ - at: 'update-address-total-volume#runTask', + at: 'update-wallet-total-volume#runTask', message: 'Error when updating totalVolume in wallets table', error, }); From 368742b41faa354ee1aeb0b4873df9a5cdbbaf96 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Wed, 11 Sep 2024 11:21:01 -0400 Subject: [PATCH 02/21] modify affiliate api stubs --- .../api/v4/affiliates-controller.test.ts | 4 +-- .../comlink/public/api-documentation.md | 28 +++++++++++-------- indexer/services/comlink/public/swagger.json | 19 ++++++++----- .../api/v4/affiliates-controller.ts | 15 +++++----- indexer/services/comlink/src/types.ts | 3 +- 5 files changed, 40 insertions(+), 29 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts index 09c0a3ba70..ff203417a5 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts @@ -3,12 +3,12 @@ import request from 'supertest'; import { sendRequest } from '../../../helpers/helpers'; describe('affiliates-controller#V4', () => { - describe('GET /referral_code', () => { + describe('GET /metadata', () => { it('should return referral code for a valid address string', async () => { const address = 'some_address'; const response: request.Response = await sendRequest({ type: RequestMethod.GET, - path: `/v4/affiliates/referral_code?address=${address}`, + path: `/v4/affiliates/metadata?address=${address}`, }); expect(response.status).toBe(200); diff --git a/indexer/services/comlink/public/api-documentation.md b/indexer/services/comlink/public/api-documentation.md index f5cf63ba61..4c56a11551 100644 --- a/indexer/services/comlink/public/api-documentation.md +++ b/indexer/services/comlink/public/api-documentation.md @@ -368,9 +368,9 @@ fetch(`${baseURL}/addresses/{address}/parentSubaccountNumber/{parentSubaccountNu This operation does not require authentication -## GetReferralCode +## GetMetadata - + > Code samples @@ -384,7 +384,7 @@ headers = { # baseURL = 'https://indexer.dydx.trade/v4' baseURL = 'https://dydx-testnet.imperator.co/v4' -r = requests.get(f'{baseURL}/affiliates/referral_code', params={ +r = requests.get(f'{baseURL}/affiliates/metadata', params={ 'address': 'string' }, headers = headers) @@ -402,7 +402,7 @@ const headers = { // const baseURL = 'https://indexer.dydx.trade/v4'; const baseURL = 'https://dydx-testnet.imperator.co/v4'; -fetch(`${baseURL}/affiliates/referral_code?address=string`, +fetch(`${baseURL}/affiliates/metadata?address=string`, { method: 'GET', @@ -416,7 +416,7 @@ fetch(`${baseURL}/affiliates/referral_code?address=string`, ``` -`GET /affiliates/referral_code` +`GET /affiliates/metadata` ### Parameters @@ -509,12 +509,13 @@ fetch(`${baseURL}/affiliates/snapshot`, "affiliateList": [ { "affiliateAddress": "string", - "affiliateEarnings": 0.1, "affiliateReferralCode": "string", + "affiliateEarnings": 0.1, "affiliateReferredTrades": 0.1, "affiliateTotalReferredFees": 0.1, "affiliateReferredUsers": 0.1, - "affiliateReferredNetProtocolEarnings": 0.1 + "affiliateReferredNetProtocolEarnings": 0.1, + "affiliateReferredTotalVolume": 0.1 } ], "total": 0.1, @@ -4069,12 +4070,13 @@ This operation does not require authentication ```json { "affiliateAddress": "string", - "affiliateEarnings": 0.1, "affiliateReferralCode": "string", + "affiliateEarnings": 0.1, "affiliateReferredTrades": 0.1, "affiliateTotalReferredFees": 0.1, "affiliateReferredUsers": 0.1, - "affiliateReferredNetProtocolEarnings": 0.1 + "affiliateReferredNetProtocolEarnings": 0.1, + "affiliateReferredTotalVolume": 0.1 } ``` @@ -4084,12 +4086,13 @@ This operation does not require authentication |Name|Type|Required|Restrictions|Description| |---|---|---|---|---| |affiliateAddress|string|true|none|none| -|affiliateEarnings|number(double)|true|none|none| |affiliateReferralCode|string|true|none|none| +|affiliateEarnings|number(double)|true|none|none| |affiliateReferredTrades|number(double)|true|none|none| |affiliateTotalReferredFees|number(double)|true|none|none| |affiliateReferredUsers|number(double)|true|none|none| |affiliateReferredNetProtocolEarnings|number(double)|true|none|none| +|affiliateReferredTotalVolume|number(double)|true|none|none| ## AffiliateSnapshotResponse @@ -4103,12 +4106,13 @@ This operation does not require authentication "affiliateList": [ { "affiliateAddress": "string", - "affiliateEarnings": 0.1, "affiliateReferralCode": "string", + "affiliateEarnings": 0.1, "affiliateReferredTrades": 0.1, "affiliateTotalReferredFees": 0.1, "affiliateReferredUsers": 0.1, - "affiliateReferredNetProtocolEarnings": 0.1 + "affiliateReferredNetProtocolEarnings": 0.1, + "affiliateReferredTotalVolume": 0.1 } ], "total": 0.1, diff --git a/indexer/services/comlink/public/swagger.json b/indexer/services/comlink/public/swagger.json index 1054683fcd..1113841e85 100644 --- a/indexer/services/comlink/public/swagger.json +++ b/indexer/services/comlink/public/swagger.json @@ -271,13 +271,13 @@ "affiliateAddress": { "type": "string" }, + "affiliateReferralCode": { + "type": "string" + }, "affiliateEarnings": { "type": "number", "format": "double" }, - "affiliateReferralCode": { - "type": "string" - }, "affiliateReferredTrades": { "type": "number", "format": "double" @@ -293,16 +293,21 @@ "affiliateReferredNetProtocolEarnings": { "type": "number", "format": "double" + }, + "affiliateReferredTotalVolume": { + "type": "number", + "format": "double" } }, "required": [ "affiliateAddress", - "affiliateEarnings", "affiliateReferralCode", + "affiliateEarnings", "affiliateReferredTrades", "affiliateTotalReferredFees", "affiliateReferredUsers", - "affiliateReferredNetProtocolEarnings" + "affiliateReferredNetProtocolEarnings", + "affiliateReferredTotalVolume" ], "type": "object", "additionalProperties": false @@ -1668,9 +1673,9 @@ ] } }, - "/affiliates/referral_code": { + "/affiliates/metadata": { "get": { - "operationId": "GetReferralCode", + "operationId": "GetMetadata", "responses": { "200": { "description": "Ok", diff --git a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts index f4ac198ba1..2d100df478 100644 --- a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts @@ -29,8 +29,8 @@ const controllerName: string = 'affiliates-controller'; // TODO(OTE-731): replace api stubs with real logic @Route('affiliates') class AffiliatesController extends Controller { - @Get('/referral_code') - async getReferralCode( + @Get('/metadata') + async getMetadata( @Query() address: string, // eslint-disable-line @typescript-eslint/no-unused-vars ): Promise { // simulate a delay @@ -67,12 +67,13 @@ class AffiliatesController extends Controller { const snapshot: AffiliateSnapshotResponseObject = { affiliateAddress: 'some_address', - affiliateEarnings: 100, affiliateReferralCode: 'TempCode123', + affiliateEarnings: 100, affiliateReferredTrades: 1000, affiliateTotalReferredFees: 100, affiliateReferredUsers: 10, affiliateReferredNetProtocolEarnings: 1000, + affiliateReferredTotalVolume: 1000000, }; const affiliateSnapshots: AffiliateSnapshotResponseObject[] = []; @@ -102,7 +103,7 @@ class AffiliatesController extends Controller { } router.get( - '/referral_code', + '/metadata', rateLimiterMiddleware(getReqRateLimiter), ...checkSchema({ address: { @@ -121,11 +122,11 @@ router.get( try { const controller: AffiliatesController = new AffiliatesController(); - const response: AffiliateReferralCodeResponse = await controller.getReferralCode(address); + const response: AffiliateReferralCodeResponse = await controller.getMetadata(address); return res.send(response); } catch (error) { return handleControllerError( - 'AffiliatesController GET /referral_code', + 'AffiliatesController GET /metadata', 'Affiliates referral code error', error, req, @@ -133,7 +134,7 @@ router.get( ); } finally { stats.timing( - `${config.SERVICE_NAME}.${controllerName}.get_referral_code.timing`, + `${config.SERVICE_NAME}.${controllerName}.get_metadata.timing`, Date.now() - start, ); } diff --git a/indexer/services/comlink/src/types.ts b/indexer/services/comlink/src/types.ts index 45622e56d5..894caba79a 100644 --- a/indexer/services/comlink/src/types.ts +++ b/indexer/services/comlink/src/types.ts @@ -703,12 +703,13 @@ export interface AffiliateSnapshotResponse { export interface AffiliateSnapshotResponseObject { affiliateAddress: string, - affiliateEarnings: number, affiliateReferralCode: string, + affiliateEarnings: number, affiliateReferredTrades: number, affiliateTotalReferredFees: number, affiliateReferredUsers: number, affiliateReferredNetProtocolEarnings: number, + affiliateReferredTotalVolume: number, } export interface AffiliateTotalVolumeResponse { From bc16518230b687c8eecb1ccb2e458a60a5899f17 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Wed, 11 Sep 2024 13:27:16 -0400 Subject: [PATCH 03/21] implement comlink GET affiliates/metadata --- indexer/services/comlink/src/config.ts | 3 ++ .../api/v4/affiliates-controller.ts | 51 +++++++++++++++++-- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/indexer/services/comlink/src/config.ts b/indexer/services/comlink/src/config.ts index eb3713bc14..e2c85ade2f 100644 --- a/indexer/services/comlink/src/config.ts +++ b/indexer/services/comlink/src/config.ts @@ -60,6 +60,9 @@ export const configSchema = { // vaults table is added. EXPERIMENT_VAULTS: parseString({ default: '' }), EXPERIMENT_VAULT_MARKETS: parseString({ default: '' }), + + // Affiliates config + VOLUME_ELIGIBILITY_THRESHOLD: parseInteger({ default: 10_000 }), }; //////////////////////////////////////////////////////////////////////////////// diff --git a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts index 2d100df478..768cffccd6 100644 --- a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts @@ -4,7 +4,13 @@ import { checkSchema, matchedData } from 'express-validator'; import { Controller, Get, Query, Route, } from 'tsoa'; - +import { + WalletTable, + AffiliateReferredUsersTable, + SubaccountTable, + SubaccountUsernamesTable, +} from '@dydxprotocol-indexer/postgres'; +import { NotFoundError, UnexpectedServerError } from '../../../lib/errors'; import { getReqRateLimiter } from '../../../caches/rate-limiters'; import config from '../../../config'; import { handleControllerError } from '../../../lib/helpers'; @@ -31,12 +37,47 @@ const controllerName: string = 'affiliates-controller'; class AffiliatesController extends Controller { @Get('/metadata') async getMetadata( - @Query() address: string, // eslint-disable-line @typescript-eslint/no-unused-vars + @Query() address: string, ): Promise { - // simulate a delay - await new Promise((resolve) => setTimeout(resolve, 100)); + // Check that the address exists + const walletRow = await WalletTable.findById(address); + if (!walletRow) { + throw new NotFoundError(`Wallet with address ${address} not found`); + } + const isVolumeEligible = Number(walletRow.totalVolume) >= config.VOLUME_ELIGIBILITY_THRESHOLD; + + // Check if the address is an affiliate (has referred users) + const referredUserRows = await AffiliateReferredUsersTable.findByAffiliateAddress(address); + const isAffiliate = referredUserRows != undefined ? referredUserRows.length > 0 : false; + + // Get referral code (subaccount 0 username) + const subaccountRows = await SubaccountTable.findAll( + { + address: address, + subaccountNumber: 0, + }, + [], + ) + // No need to check subaccountRows.length > 1 because subaccountNumber is unique for an address + if (subaccountRows.length === 0) { + throw new UnexpectedServerError(`Subaccount 0 not found for address ${address}`); + } + const subaccountId = subaccountRows[0].id; + const usernameRows = await SubaccountUsernamesTable.findAll( + { + subaccountId: [subaccountId], + }, + [], + ) + if (usernameRows.length === 0) { + throw new UnexpectedServerError(`Username not found for subaccount ${subaccountId}`); + } else if (usernameRows.length > 1) { + throw new UnexpectedServerError(`Found multiple usernames for subaccount ${subaccountId}`); + } + const referralCode = usernameRows[0].username; + return { - referralCode: 'TempCode123', + referralCode: referralCode, }; } From ac62c32c015dbe47726ef37aa3ce855e75f8fc6e Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 12 Sep 2024 12:42:26 -0400 Subject: [PATCH 04/21] tests --- .../api/v4/affiliates-controller.test.ts | 127 +++++++++++++++++- indexer/services/comlink/package.json | 2 +- .../api/v4/affiliates-controller.ts | 19 +-- 3 files changed, 126 insertions(+), 22 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts index 1f86ebf54e..7cf7847a79 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts @@ -1,22 +1,139 @@ +import { + dbHelpers, + testConstants, + testMocks, + SubaccountUsernamesTable, + WalletTable, + AffiliateReferredUsersTable, +} from '@dydxprotocol-indexer/postgres'; import { AffiliateSnapshotRequest, RequestMethod } from '../../../../src/types'; import request from 'supertest'; import { sendRequest } from '../../../helpers/helpers'; +import { defaultWallet, defaultWallet2 } from '@dydxprotocol-indexer/postgres/build/__tests__/helpers/constants'; describe('affiliates-controller#V4', () => { + beforeAll(async () => { + await dbHelpers.migrate(); + }); + + afterAll(async () => { + await dbHelpers.teardown(); + }); + describe('GET /metadata', () => { - it('should return referral code for a valid address string', async () => { - const address = 'some_address'; + beforeEach(async () => { + await testMocks.seedData(); + await SubaccountUsernamesTable.create(testConstants.defaultSubaccountUsername); + }); + + afterEach(async () => { + await dbHelpers.clearData(); + }); + + it('should return referral code for address with username', async () => { const response: request.Response = await sendRequest({ type: RequestMethod.GET, - path: `/v4/affiliates/metadata?address=${address}`, + path: `/v4/affiliates/metadata?address=${testConstants.defaultWallet.address}`, + expectedStatus: 200, // helper performs expect on status }); - expect(response.status).toBe(200); expect(response.body).toEqual({ - referralCode: 'TempCode123', + // username is the referral code + referralCode: testConstants.defaultSubaccountUsername.username, + isVolumeEligible: false, + isAffiliate: false, + }); + }); + + it('should fail if address does not exist', async () => { + const nonExistentAddress = 'adgsakhasgt' + const response: request.Response = await sendRequest({ + type: RequestMethod.GET, + path: `/v4/affiliates/metadata?address=${nonExistentAddress}`, + expectedStatus: 404, // helper performs expect on status + }); + }); + + it('should classify not volume eligible', async () => { + await WalletTable.update( + { + address: testConstants.defaultWallet.address, + totalVolume: "0", + totalTradingRewards: "0", + }, + ); + const response: request.Response = await sendRequest({ + type: RequestMethod.GET, + path: `/v4/affiliates/metadata?address=${testConstants.defaultWallet.address}`, + expectedStatus: 200, // helper performs expect on status + }); + expect(response.body).toEqual({ + referralCode: testConstants.defaultSubaccountUsername.username, + isVolumeEligible: false, + isAffiliate: false, + }); + }); + + it('should classify volume eligible', async () => { + await WalletTable.update( + { + address: testConstants.defaultWallet.address, + totalVolume: "100000", + totalTradingRewards: "0", + }, + ); + const response: request.Response = await sendRequest({ + type: RequestMethod.GET, + path: `/v4/affiliates/metadata?address=${testConstants.defaultWallet.address}`, + expectedStatus: 200, // helper performs expect on status + }); + expect(response.body).toEqual({ + referralCode: testConstants.defaultSubaccountUsername.username, isVolumeEligible: true, isAffiliate: false, }); + }); + + it('should classify is not affiliate', async () => { + // AffiliateReferredUsersTable is empty + const response: request.Response = await sendRequest({ + type: RequestMethod.GET, + path: `/v4/affiliates/metadata?address=${testConstants.defaultWallet.address}`, + expectedStatus: 200, // helper performs expect on status + }); + expect(response.body).toEqual({ + referralCode: testConstants.defaultSubaccountUsername.username, + isVolumeEligible: false, + isAffiliate: false, + }); + }); + + it('should classify is affiliate', async () => { + await AffiliateReferredUsersTable.create({ + affiliateAddress: defaultWallet.address, + refereeAddress: defaultWallet2.address, + referredAtBlock: '1', + }); + const response: request.Response = await sendRequest({ + type: RequestMethod.GET, + path: `/v4/affiliates/metadata?address=${testConstants.defaultWallet.address}`, + expectedStatus: 200, // helper performs expect on status + }); + expect(response.body).toEqual({ + referralCode: testConstants.defaultSubaccountUsername.username, + isVolumeEligible: false, + isAffiliate: true, + }); + }); + + it('should fail if subaccount username not found', async () => { + // create defaultWallet2 without subaccount username + WalletTable.create(testConstants.defaultWallet2); + const response: request.Response = await sendRequest({ + type: RequestMethod.GET, + path: `/v4/affiliates/metadata?address=${testConstants.defaultWallet2.address}`, + expectedStatus: 500, // helper performs expect on status + }); }); }); diff --git a/indexer/services/comlink/package.json b/indexer/services/comlink/package.json index ba12e7064b..d86c94d2c4 100644 --- a/indexer/services/comlink/package.json +++ b/indexer/services/comlink/package.json @@ -13,7 +13,7 @@ "coverage": "pnpm test -- --coverage", "lint": "eslint --ext .ts,.js .", "lint:fix": "eslint --ext .ts,.js . --fix", - "test": "NODE_ENV=test jest --runInBand --forceExit", + "test-comlink": "NODE_ENV=test jest --runInBand --forceExit", "swagger": "ts-node -r dotenv-flow/config src/scripts/generate-swagger.ts", "gen-markdown": "widdershins public/swagger.json -o public/api-documentation.md --omitHeader --language_tabs 'python:Python' 'javascript:Javascript'" }, diff --git a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts index 34be54dbad..78d51a9a32 100644 --- a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts @@ -37,9 +37,8 @@ const controllerName: string = 'affiliates-controller'; class AffiliatesController extends Controller { @Get('/metadata') async getMetadata( -<<<<<<< HEAD @Query() address: string, - ): Promise { + ): Promise { // Check that the address exists const walletRow = await WalletTable.findById(address); if (!walletRow) { @@ -79,16 +78,8 @@ class AffiliatesController extends Controller { return { referralCode: referralCode, -======= - @Query() address: string, // eslint-disable-line @typescript-eslint/no-unused-vars - ): Promise { - // simulate a delay - await new Promise((resolve) => setTimeout(resolve, 100)); - return { - referralCode: 'TempCode123', - isVolumeEligible: true, - isAffiliate: false, ->>>>>>> main + isVolumeEligible: isVolumeEligible, + isAffiliate: isAffiliate, }; } @@ -174,11 +165,7 @@ router.get( try { const controller: AffiliatesController = new AffiliatesController(); -<<<<<<< HEAD - const response: AffiliateReferralCodeResponse = await controller.getMetadata(address); -======= const response: AffiliateMetadataResponse = await controller.getMetadata(address); ->>>>>>> main return res.send(response); } catch (error) { return handleControllerError( From 0784cfa3b7de2e20133430e7f241fd6385a8bc96 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 12 Sep 2024 12:47:33 -0400 Subject: [PATCH 05/21] revert package.json --- indexer/services/comlink/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/services/comlink/package.json b/indexer/services/comlink/package.json index 5e8085911c..29ea1def60 100644 --- a/indexer/services/comlink/package.json +++ b/indexer/services/comlink/package.json @@ -13,7 +13,7 @@ "coverage": "pnpm test -- --coverage", "lint": "eslint --ext .ts,.js .", "lint:fix": "eslint --ext .ts,.js . --fix", - "test-comlink": "NODE_ENV=test jest --runInBand --forceExit", + "test": "NODE_ENV=test jest --runInBand --forceExit", "swagger": "ts-node -r dotenv-flow/config src/scripts/generate-swagger.ts", "gen-markdown": "widdershins public/swagger.json -o public/api-documentation.md --omitHeader --language_tabs 'python:Python' 'javascript:Javascript'" }, From 21481ecbb9b98843c8c437ed736168883ed224c1 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 12 Sep 2024 12:58:52 -0400 Subject: [PATCH 06/21] lint fix --- .../api/v4/affiliates-controller.test.ts | 28 ++++++++-------- .../api/v4/affiliates-controller.ts | 33 ++++++++++--------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts index 7cf7847a79..465865020d 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts @@ -25,7 +25,7 @@ describe('affiliates-controller#V4', () => { await testMocks.seedData(); await SubaccountUsernamesTable.create(testConstants.defaultSubaccountUsername); }); - + afterEach(async () => { await dbHelpers.clearData(); }); @@ -46,20 +46,20 @@ describe('affiliates-controller#V4', () => { }); it('should fail if address does not exist', async () => { - const nonExistentAddress = 'adgsakhasgt' - const response: request.Response = await sendRequest({ + const nonExistentAddress = 'adgsakhasgt'; + await sendRequest({ type: RequestMethod.GET, path: `/v4/affiliates/metadata?address=${nonExistentAddress}`, expectedStatus: 404, // helper performs expect on status }); }); - it('should classify not volume eligible', async () => { + it('should classify not volume eligible', async () => { await WalletTable.update( - { + { address: testConstants.defaultWallet.address, - totalVolume: "0", - totalTradingRewards: "0", + totalVolume: '0', + totalTradingRewards: '0', }, ); const response: request.Response = await sendRequest({ @@ -74,12 +74,12 @@ describe('affiliates-controller#V4', () => { }); }); - it('should classify volume eligible', async () => { + it('should classify volume eligible', async () => { await WalletTable.update( - { + { address: testConstants.defaultWallet.address, - totalVolume: "100000", - totalTradingRewards: "0", + totalVolume: '100000', + totalTradingRewards: '0', }, ); const response: request.Response = await sendRequest({ @@ -92,7 +92,7 @@ describe('affiliates-controller#V4', () => { isVolumeEligible: true, isAffiliate: false, }); - }); + }); it('should classify is not affiliate', async () => { // AffiliateReferredUsersTable is empty @@ -128,8 +128,8 @@ describe('affiliates-controller#V4', () => { it('should fail if subaccount username not found', async () => { // create defaultWallet2 without subaccount username - WalletTable.create(testConstants.defaultWallet2); - const response: request.Response = await sendRequest({ + await WalletTable.create(testConstants.defaultWallet2); + await sendRequest({ type: RequestMethod.GET, path: `/v4/affiliates/metadata?address=${testConstants.defaultWallet2.address}`, expectedStatus: 500, // helper performs expect on status diff --git a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts index 78d51a9a32..15ca181906 100644 --- a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts @@ -1,18 +1,19 @@ import { stats } from '@dydxprotocol-indexer/base'; -import express from 'express'; -import { checkSchema, matchedData } from 'express-validator'; -import { - Controller, Get, Query, Route, -} from 'tsoa'; import { WalletTable, AffiliateReferredUsersTable, SubaccountTable, SubaccountUsernamesTable, } from '@dydxprotocol-indexer/postgres'; -import { NotFoundError, UnexpectedServerError } from '../../../lib/errors'; +import express from 'express'; +import { checkSchema, matchedData } from 'express-validator'; +import { + Controller, Get, Query, Route, +} from 'tsoa'; + import { getReqRateLimiter } from '../../../caches/rate-limiters'; import config from '../../../config'; +import { NotFoundError, UnexpectedServerError } from '../../../lib/errors'; import { handleControllerError } from '../../../lib/helpers'; import { rateLimiterMiddleware } from '../../../lib/rate-limit'; import { handleValidationErrors } from '../../../request-helpers/error-handler'; @@ -48,38 +49,38 @@ class AffiliatesController extends Controller { // Check if the address is an affiliate (has referred users) const referredUserRows = await AffiliateReferredUsersTable.findByAffiliateAddress(address); - const isAffiliate = referredUserRows != undefined ? referredUserRows.length > 0 : false; + const isAffiliate = referredUserRows !== undefined ? referredUserRows.length > 0 : false; // Get referral code (subaccount 0 username) const subaccountRows = await SubaccountTable.findAll( { - address: address, + address, subaccountNumber: 0, }, [], - ) - // No need to check subaccountRows.length > 1 because subaccountNumber is unique for an address + ); + // No need to check subaccountRows.length > 1 as subaccountNumber is unique for an address if (subaccountRows.length === 0) { throw new UnexpectedServerError(`Subaccount 0 not found for address ${address}`); } const subaccountId = subaccountRows[0].id; + const usernameRows = await SubaccountUsernamesTable.findAll( { subaccountId: [subaccountId], }, [], - ) + ); + // No need to check usernameRows.length > 1 as subAccountId is unique (foreign key constraint) if (usernameRows.length === 0) { throw new UnexpectedServerError(`Username not found for subaccount ${subaccountId}`); - } else if (usernameRows.length > 1) { - throw new UnexpectedServerError(`Found multiple usernames for subaccount ${subaccountId}`); } const referralCode = usernameRows[0].username; return { - referralCode: referralCode, - isVolumeEligible: isVolumeEligible, - isAffiliate: isAffiliate, + referralCode, + isVolumeEligible, + isAffiliate, }; } From 3fa0437059f24c2b29a8d8df30c919497afdd88c Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 12 Sep 2024 16:01:09 -0400 Subject: [PATCH 07/21] replace affiliates address and total_volume stubs with real implementation --- .../api/v4/affiliates-controller.test.ts | 63 +++++++++++++++---- .../api/v4/affiliates-controller.ts | 34 +++++++--- indexer/services/comlink/src/types.ts | 2 +- 3 files changed, 78 insertions(+), 21 deletions(-) diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts index 465865020d..62ee29513e 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts @@ -138,16 +138,34 @@ describe('affiliates-controller#V4', () => { }); describe('GET /address', () => { - it('should return address for a valid referral code string', async () => { - const referralCode = 'TempCode123'; + beforeEach(async () => { + await testMocks.seedData(); + await SubaccountUsernamesTable.create(testConstants.defaultSubaccountUsername); + }); + + afterEach(async () => { + await dbHelpers.clearData(); + }); + + it('should return address for a valid referral code', async () => { + const referralCode = testConstants.defaultSubaccountUsername.username; const response: request.Response = await sendRequest({ type: RequestMethod.GET, path: `/v4/affiliates/address?referralCode=${referralCode}`, + expectedStatus: 200, // helper performs expect on status }); - expect(response.status).toBe(200); expect(response.body).toEqual({ - address: 'some_address', + address: testConstants.defaultWallet.address, + }); + }); + + it('should fail when referral code not found', async () => { + const nonExistentReferralCode = 'BadCode123'; + await sendRequest({ + type: RequestMethod.GET, + path: `/v4/affiliates/address?referralCode=${nonExistentReferralCode}`, + expectedStatus: 404, // helper performs expect on status }); }); }); @@ -157,11 +175,11 @@ describe('affiliates-controller#V4', () => { const req: AffiliateSnapshotRequest = { limit: 10, offset: 10, - sortByReferredFees: true, + sortByAffiliateEarning: true, }; const response: request.Response = await sendRequest({ type: RequestMethod.GET, - path: `/v4/affiliates/snapshot?limit=${req.limit}&offset=${req.offset}&sortByReferredFees=${req.sortByReferredFees}`, + path: `/v4/affiliates/snapshot?limit=${req.limit}&offset=${req.offset}&sortByReferredFees=${req.sortByAffiliateEarning}`, }); expect(response.status).toBe(200); @@ -184,16 +202,39 @@ describe('affiliates-controller#V4', () => { }); describe('GET /total_volume', () => { - it('should return total_volume for a valid address', async () => { - const address = 'some_address'; + beforeEach(async () => { + await testMocks.seedData(); + await WalletTable.update( + { + address: testConstants.defaultWallet.address, + totalVolume: '100000', + totalTradingRewards: '0', + }, + ); + }); + + afterEach(async () => { + await dbHelpers.clearData(); + }); + + it('should return total volume for a valid address', async () => { const response: request.Response = await sendRequest({ type: RequestMethod.GET, - path: `/v4/affiliates/total_volume?address=${address}`, + path: `/v4/affiliates/total_volume?address=${testConstants.defaultWallet.address}`, + expectedStatus: 200, // helper performs expect on status }); - expect(response.status).toBe(200); expect(response.body).toEqual({ - totalVolume: 111.1, + totalVolume: 100000, + }); + }); + + it('should fail if address does not exist', async () => { + const nonExistentAddress = 'adgsakhasgt'; + await sendRequest({ + type: RequestMethod.GET, + path: `/v4/affiliates/metadata?address=${nonExistentAddress}`, + expectedStatus: 404, // helper performs expect on status }); }); }); diff --git a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts index 15ca181906..03890bcde5 100644 --- a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts @@ -86,12 +86,24 @@ class AffiliatesController extends Controller { @Get('/address') async getAddress( - @Query() referralCode: string, // eslint-disable-line @typescript-eslint/no-unused-vars + @Query() referralCode: string, ): Promise { - // simulate a delay - await new Promise((resolve) => setTimeout(resolve, 100)); + const usernameRow = await SubaccountUsernamesTable.findByUsername(referralCode); + if (!usernameRow) { + throw new NotFoundError(`Referral code ${referralCode} does not exist`); + } + const subAccountId = usernameRow.subaccountId; + + const subaccountRow = await SubaccountTable.findById(subAccountId); + // subaccountRow should never be undefined because of foreign key constraint between subaccounts + // and subaccount_usernames tables + if (!subaccountRow) { + throw new UnexpectedServerError(`Subaccount ${subAccountId} not found`); + } + const address = subaccountRow.address; + return { - address: 'some_address', + address, }; } @@ -136,12 +148,16 @@ class AffiliatesController extends Controller { @Get('/total_volume') public async getTotalVolume( - @Query() address: string, // eslint-disable-line @typescript-eslint/no-unused-vars + @Query() address: string, ): Promise { - // simulate a delay - await new Promise((resolve) => setTimeout(resolve, 100)); + // Check that the address exists + const walletRow = await WalletTable.findById(address); + if (!walletRow) { + throw new NotFoundError(`Wallet with address ${address} not found`); + } + return { - totalVolume: 111.1, + totalVolume: Number(walletRow.totalVolume), }; } } @@ -257,7 +273,7 @@ router.get( const { offset, limit, - sortByReferredFees, + sortByAffiliateEarning: sortByReferredFees, }: AffiliateSnapshotRequest = matchedData(req) as AffiliateSnapshotRequest; try { diff --git a/indexer/services/comlink/src/types.ts b/indexer/services/comlink/src/types.ts index 2867e0827b..04542f0e5e 100644 --- a/indexer/services/comlink/src/types.ts +++ b/indexer/services/comlink/src/types.ts @@ -686,7 +686,7 @@ export interface AffiliateAddressRequest{ export interface AffiliateSnapshotRequest{ limit?: number, offset?: number, - sortByReferredFees?: boolean, + sortByAffiliateEarning?: boolean, } export interface AffiliateTotalVolumeRequest{ From 93d3f83e9a313c7eb50773fce23be299a297008c Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Mon, 16 Sep 2024 15:08:27 -0400 Subject: [PATCH 08/21] replace affiliates snapshot stub with real implementation --- .../postgres/__tests__/helpers/constants.ts | 56 ++++++-- .../stores/affiliate-info-table.test.ts | 104 ++++++++++++++- .../stores/subaccount-usernames-table.test.ts | 25 ++++ ...precision_and_add_total_referred_volume.ts | 22 +++ indexer/packages/postgres/src/index.ts | 2 + .../src/models/affiliate-info-model.ts | 5 + .../src/stores/affiliate-info-table.ts | 30 +++++ .../src/stores/subaccount-usernames-table.ts | 29 ++++ .../src/types/affiliate-info-types.ts | 2 + .../postgres/src/types/db-model-types.ts | 6 + .../api/v4/affiliates-controller.test.ts | 125 +++++++++++++++--- .../comlink/public/api-documentation.md | 3 +- indexer/services/comlink/public/swagger.json | 13 +- .../api/v4/affiliates-controller.ts | 106 +++++++++++---- indexer/services/comlink/src/types.ts | 3 +- 15 files changed, 467 insertions(+), 64 deletions(-) create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts diff --git a/indexer/packages/postgres/__tests__/helpers/constants.ts b/indexer/packages/postgres/__tests__/helpers/constants.ts index f306550591..cc7175be59 100644 --- a/indexer/packages/postgres/__tests__/helpers/constants.ts +++ b/indexer/packages/postgres/__tests__/helpers/constants.ts @@ -75,6 +75,7 @@ export const blockedAddress: string = 'dydx1f9k5qldwmqrnwy8hcgp4fw6heuvszt35egvt export const vaultAddress: string = 'dydx1c0m5x87llaunl5sgv3q5vd7j5uha26d2q2r2q0'; // ============== Subaccounts ============== +export const defaultWalletAddress: string = 'defaultWalletAddress'; export const defaultSubaccount: SubaccountCreateObject = { address: defaultAddress, @@ -97,6 +98,14 @@ export const defaultSubaccount3: SubaccountCreateObject = { updatedAtHeight: createdHeight, }; +// defaultWalletAddress belongs to defaultWallet2 and is different from defaultAddress +export const defaultSubaccountDefaultWalletAddress: SubaccountCreateObject = { + address: defaultWalletAddress, + subaccountNumber: 0, + updatedAt: createdDateTime.toISO(), + updatedAtHeight: createdHeight, +}; + export const defaultSubaccountWithAlternateAddress: SubaccountCreateObject = { address: defaultAddress2, subaccountNumber: 0, @@ -125,8 +134,6 @@ export const isolatedSubaccount2: SubaccountCreateObject = { updatedAtHeight: createdHeight, }; -export const defaultWalletAddress: string = 'defaultWalletAddress'; - export const defaultSubaccountId: string = SubaccountTable.uuid( defaultAddress, defaultSubaccount.subaccountNumber, @@ -139,6 +146,10 @@ export const defaultSubaccountId3: string = SubaccountTable.uuid( defaultAddress, defaultSubaccount3.subaccountNumber, ); +export const defaultSubaccountIdDefaultWalletAddress: string = SubaccountTable.uuid( + defaultWalletAddress, + defaultSubaccountDefaultWalletAddress.subaccountNumber, +); export const defaultSubaccountIdWithAlternateAddress: string = SubaccountTable.uuid( defaultAddress2, defaultSubaccountWithAlternateAddress.subaccountNumber, @@ -906,6 +917,17 @@ export const duplicatedSubaccountUsername: SubaccountUsernamesCreateObject = { subaccountId: defaultSubaccountId3, }; +// defaultWalletAddress belongs to defaultWallet2 and is different from defaultAddress +export const subaccountUsernameWithDefaultWalletAddress: SubaccountUsernamesCreateObject = { + username: 'EvilRaisin11', + subaccountId: defaultSubaccountIdDefaultWalletAddress, +}; + +export const subaccountUsernameWithAlternativeAddress: SubaccountUsernamesCreateObject = { + username: 'HonestRaisin32', + subaccountId: defaultSubaccountIdWithAlternateAddress, +}; + // ============== Leaderboard pnl Data ============== export const defaultLeaderboardPnlOneDay: LeaderboardPnlCreateObject = { @@ -963,24 +985,38 @@ export const defaultKV2: PersistentCacheCreateObject = { export const defaultAffiliateInfo: AffiliateInfoCreateObject = { address: defaultAddress, - affiliateEarnings: '10.00', + affiliateEarnings: '10', referredMakerTrades: 10, referredTakerTrades: 20, - totalReferredFees: '10.00', + totalReferredFees: '10', totalReferredUsers: 5, - referredNetProtocolEarnings: '20.00', + referredNetProtocolEarnings: '20', firstReferralBlockHeight: '1', + referredTotalVolume: '1000', }; -export const defaultAffiliateInfo1: AffiliateInfoCreateObject = { - address: defaultAddress2, - affiliateEarnings: '11.00', +export const defaultAffiliateInfo2: AffiliateInfoCreateObject = { + address: defaultWalletAddress, + affiliateEarnings: '11', referredMakerTrades: 11, referredTakerTrades: 21, - totalReferredFees: '11.00', + totalReferredFees: '11', totalReferredUsers: 5, - referredNetProtocolEarnings: '21.00', + referredNetProtocolEarnings: '21', firstReferralBlockHeight: '11', + referredTotalVolume: '1000', +}; + +export const defaultAffiliateInfo3: AffiliateInfoCreateObject = { + address: defaultAddress2, + affiliateEarnings: '12', + referredMakerTrades: 12, + referredTakerTrades: 22, + totalReferredFees: '12', + totalReferredUsers: 10, + referredNetProtocolEarnings: '22', + firstReferralBlockHeight: '12', + referredTotalVolume: '1111111', }; // ============== Tokens ============= diff --git a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts index 7a8ac32bd7..e5c5ec33b5 100644 --- a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts @@ -1,6 +1,6 @@ import { AffiliateInfoFromDatabase } from '../../src/types'; import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; -import { defaultAffiliateInfo, defaultAffiliateInfo1 } from '../helpers/constants'; +import { defaultAffiliateInfo, defaultAffiliateInfo2 } from '../helpers/constants'; import * as AffiliateInfoTable from '../../src/stores/affiliate-info-table'; describe('Affiliate info store', () => { @@ -32,15 +32,15 @@ describe('Affiliate info store', () => { ); expect(info).toEqual(expect.objectContaining(defaultAffiliateInfo)); - await AffiliateInfoTable.upsert(defaultAffiliateInfo1); - info = await AffiliateInfoTable.findById(defaultAffiliateInfo1.address); - expect(info).toEqual(expect.objectContaining(defaultAffiliateInfo1)); + await AffiliateInfoTable.upsert(defaultAffiliateInfo2); + info = await AffiliateInfoTable.findById(defaultAffiliateInfo2.address); + expect(info).toEqual(expect.objectContaining(defaultAffiliateInfo2)); }); it('Successfully finds all affiliate infos', async () => { await Promise.all([ AffiliateInfoTable.create(defaultAffiliateInfo), - AffiliateInfoTable.create(defaultAffiliateInfo1), + AffiliateInfoTable.create(defaultAffiliateInfo2), ]); const infos: AffiliateInfoFromDatabase[] = await AffiliateInfoTable.findAll( @@ -52,7 +52,7 @@ describe('Affiliate info store', () => { expect(infos.length).toEqual(2); expect(infos).toEqual(expect.arrayContaining([ expect.objectContaining(defaultAffiliateInfo), - expect.objectContaining(defaultAffiliateInfo1), + expect.objectContaining(defaultAffiliateInfo2), ])); }); @@ -65,4 +65,96 @@ describe('Affiliate info store', () => { expect(info).toEqual(expect.objectContaining(defaultAffiliateInfo)); }); + + describe('paginatedFindWithAddressFilter', () => { + beforeEach(async () => { + await migrate(); + for (let i = 0; i < 10; i++) { + await AffiliateInfoTable.create({ + ...defaultAffiliateInfo, + address: `address_${i}`, + affiliateEarnings: i.toString(), + }); + } + }); + + it('Successfully filters by address', async () => { + const infos = await AffiliateInfoTable.paginatedFindWithAddressFilter( + ['address_0'], + 0, + 10, + false, + ); + expect(infos).toBeDefined(); + expect(infos!.length).toEqual(1); + expect(infos![0]).toEqual(expect.objectContaining({ + ...defaultAffiliateInfo, + address: 'address_0', + affiliateEarnings: '0', + })); + }); + + it('Successfully sorts by affiliate earning', async () => { + const infos = await AffiliateInfoTable.paginatedFindWithAddressFilter( + [], + 0, + 10, + true, + ); + expect(infos).toBeDefined(); + expect(infos!.length).toEqual(10); + expect(infos![0]).toEqual(expect.objectContaining({ + ...defaultAffiliateInfo, + address: 'address_9', + affiliateEarnings: '9', + })); + expect(infos![9]).toEqual(expect.objectContaining({ + ...defaultAffiliateInfo, + address: 'address_0', + affiliateEarnings: '0', + })); + }); + + it('Successfully uses offset and limit', async () => { + const infos = await AffiliateInfoTable.paginatedFindWithAddressFilter( + [], + 5, + 2, + false, + ); + expect(infos).toBeDefined(); + expect(infos!.length).toEqual(2); + expect(infos![0]).toEqual(expect.objectContaining({ + ...defaultAffiliateInfo, + address: 'address_5', + affiliateEarnings: '5', + })); + expect(infos![1]).toEqual(expect.objectContaining({ + ...defaultAffiliateInfo, + address: 'address_6', + affiliateEarnings: '6', + })); + }); + + it('Successfully filters, sorts, offsets, and limits', async () => { + const infos = await AffiliateInfoTable.paginatedFindWithAddressFilter( + [], + 3, + 2, + true, + ); + expect(infos).toBeDefined(); + expect(infos!.length).toEqual(2); + expect(infos![0]).toEqual(expect.objectContaining({ + ...defaultAffiliateInfo, + address: 'address_6', + affiliateEarnings: '6', + })); + expect(infos![1]).toEqual(expect.objectContaining({ + ...defaultAffiliateInfo, + address: 'address_5', + affiliateEarnings: '5', + })); + }); + }); }); diff --git a/indexer/packages/postgres/__tests__/stores/subaccount-usernames-table.test.ts b/indexer/packages/postgres/__tests__/stores/subaccount-usernames-table.test.ts index 5df00a60ad..c069df9f59 100644 --- a/indexer/packages/postgres/__tests__/stores/subaccount-usernames-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/subaccount-usernames-table.test.ts @@ -1,11 +1,16 @@ import { SubaccountFromDatabase, SubaccountUsernamesFromDatabase, SubaccountsWithoutUsernamesResult } from '../../src/types'; import * as SubaccountUsernamesTable from '../../src/stores/subaccount-usernames-table'; +import * as WalletTable from '../../src/stores/wallet-table'; import * as SubaccountsTable from '../../src/stores/subaccount-table'; import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; import { defaultSubaccountUsername, defaultSubaccountUsername2, + defaultSubaccountWithAlternateAddress, + defaultWallet, + defaultWallet2, duplicatedSubaccountUsername, + subaccountUsernameWithAlternativeAddress, } from '../helpers/constants'; import { seedData } from '../helpers/mock-generators'; @@ -80,4 +85,24 @@ describe('SubaccountUsernames store', () => { SubaccountUsernamesTable.getSubaccountsWithoutUsernames(); expect(subaccountIds.length).toEqual(subaccountLength - 1); }); + + it('Get username using address', async () => { + await Promise.all([ + // Add two usernames for defaultWallet + SubaccountUsernamesTable.create(defaultSubaccountUsername), + SubaccountUsernamesTable.create(defaultSubaccountUsername2), + // Add one username for alternativeWallet + WalletTable.create(defaultWallet2), + SubaccountsTable.create(defaultSubaccountWithAlternateAddress), + SubaccountUsernamesTable.create(subaccountUsernameWithAlternativeAddress), + ]); + + // Should only get username for defaultWallet's subaccount 0 + const usernames = await SubaccountUsernamesTable.findByAddress([defaultWallet.address]); + expect(usernames.length).toEqual(1); + expect(usernames[0]).toEqual(expect.objectContaining({ + address: defaultWallet.address, + username: defaultSubaccountUsername.username, + })); + }); }); diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts new file mode 100644 index 0000000000..53f6d6d373 --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts @@ -0,0 +1,22 @@ +import * as Knex from 'knex'; + +export async function up(knex: Knex): Promise { + return knex.schema.alterTable('affiliate_info', (table) => { + // null indicates variable precision whereas not specifying will result in 8,2 precision,scale + table.decimal('affiliateEarnings', null).alter(); + table.decimal('totalReferredFees', null).alter(); + table.decimal('referredNetProtocolEarnings', null).alter(); + + table.decimal('referredTotalVolume', null).notNullable(); + }); +} + +export async function down(knex: Knex): Promise { + return knex.schema.alterTable('affiliate_info', (table) => { + table.decimal('affiliateEarnings').alter(); + table.decimal('totalReferredFees').alter(); + table.decimal('referredNetProtocolEarnings').alter(); + + table.dropColumn('referredTotalVolume'); + }); +} diff --git a/indexer/packages/postgres/src/index.ts b/indexer/packages/postgres/src/index.ts index 0081e3bca9..07be84f1b4 100644 --- a/indexer/packages/postgres/src/index.ts +++ b/indexer/packages/postgres/src/index.ts @@ -20,6 +20,7 @@ export { default as SubaccountUsernamesModel } from './models/subaccount-usernam export { default as LeaderboardPnlModel } from './models/leaderboard-pnl-model'; export { default as PersistentCacheModel } from './models/persistent-cache-model'; export { default as AffiliateReferredUsersModel } from './models/affiliate-referred-users-model'; +export { default as AffiliateInfoModel } from './models/affiliate-info-model'; export * as AssetTable from './stores/asset-table'; export * as AssetPositionTable from './stores/asset-position-table'; @@ -48,6 +49,7 @@ export * as SubaccountUsernamesTable from './stores/subaccount-usernames-table'; export * as PersistentCacheTable from './stores/persistent-cache-table'; export * as AffiliateReferredUsersTable from './stores/affiliate-referred-users-table'; export * as FirebaseNotificationTokenTable from './stores/firebase-notification-token-table'; +export * as AffiliateInfoTable from './stores/affiliate-info-table'; export * as perpetualMarketRefresher from './loops/perpetual-market-refresher'; export * as assetRefresher from './loops/asset-refresher'; diff --git a/indexer/packages/postgres/src/models/affiliate-info-model.ts b/indexer/packages/postgres/src/models/affiliate-info-model.ts index 1fe37b2b61..7fcbefac39 100644 --- a/indexer/packages/postgres/src/models/affiliate-info-model.ts +++ b/indexer/packages/postgres/src/models/affiliate-info-model.ts @@ -23,6 +23,7 @@ export default class AffiliateInfoModel extends BaseModel { 'totalReferredUsers', 'referredNetProtocolEarnings', 'firstReferralBlockHeight', + 'referredTotalVolume', ], properties: { address: { type: 'string' }, @@ -33,6 +34,7 @@ export default class AffiliateInfoModel extends BaseModel { totalReferredUsers: { type: 'int' }, referredNetProtocolEarnings: { type: 'string', pattern: NonNegativeNumericPattern }, firstReferralBlockHeight: { type: 'string', pattern: NonNegativeNumericPattern }, + referredTotalVolume: { type: 'string', pattern: NonNegativeNumericPattern }, }, }; } @@ -53,6 +55,7 @@ export default class AffiliateInfoModel extends BaseModel { totalReferredUsers: 'int', referredNetProtocolEarnings: 'string', firstReferralBlockHeight: 'string', + referredTotalVolume: 'string', }; } @@ -73,4 +76,6 @@ export default class AffiliateInfoModel extends BaseModel { referredNetProtocolEarnings!: string; firstReferralBlockHeight!: string; + + referredTotalVolume!: string; } diff --git a/indexer/packages/postgres/src/stores/affiliate-info-table.ts b/indexer/packages/postgres/src/stores/affiliate-info-table.ts index 3f6695592a..7972bfcef5 100644 --- a/indexer/packages/postgres/src/stores/affiliate-info-table.ts +++ b/indexer/packages/postgres/src/stores/affiliate-info-table.ts @@ -80,6 +80,7 @@ export async function upsert( // should only ever be one AffiliateInfo return AffiliateInfos[0]; } + export async function findById( address: string, options: Options = DEFAULT_POSTGRES_OPTIONS, @@ -92,3 +93,32 @@ export async function findById( .findById(address) .returning('*'); } + +export async function paginatedFindWithAddressFilter( + addressFilter: string[], + offset: number, + limit: number, + sortByAffiliateEarning: boolean, + options: Options = DEFAULT_POSTGRES_OPTIONS, +): Promise { + let baseQuery: QueryBuilder = setupBaseQuery( + AffiliateInfoModel, + options, + ); + + // Apply address filter if provided + if (addressFilter.length > 0) { + baseQuery = baseQuery.whereIn(AffiliateInfoColumns.address, addressFilter); + } + + // Sorting by affiliate earnings or default sorting by address + if (sortByAffiliateEarning) { + baseQuery = baseQuery.orderBy(AffiliateInfoColumns.affiliateEarnings, Ordering.DESC); + } + + // Apply pagination using offset and limit + baseQuery = baseQuery.offset(offset).limit(limit); + + // Returning all fields + return baseQuery.returning('*'); +} diff --git a/indexer/packages/postgres/src/stores/subaccount-usernames-table.ts b/indexer/packages/postgres/src/stores/subaccount-usernames-table.ts index a553ebc17f..56abec087d 100644 --- a/indexer/packages/postgres/src/stores/subaccount-usernames-table.ts +++ b/indexer/packages/postgres/src/stores/subaccount-usernames-table.ts @@ -1,6 +1,7 @@ import { QueryBuilder } from 'objection'; import { DEFAULT_POSTGRES_OPTIONS } from '../constants'; +import { knexReadReplica } from '../helpers/knex'; import { verifyAllRequiredFields, setupBaseQuery, @@ -18,6 +19,7 @@ import { Options, Ordering, QueryableField, + AddressUsernameFromDatabase, } from '../types'; export async function findAll( @@ -113,3 +115,30 @@ export async function getSubaccountsWithoutUsernames( return result.rows; } + +export async function findByAddress( + addresses: string[], +): Promise { + if (addresses.length === 0) { + return []; + } + + const result: { rows: AddressUsernameFromDatabase[] } = await knexReadReplica + .getConnection() + .raw( + ` + WITH subaccountIds AS ( + SELECT "id", "address" + FROM subaccounts + WHERE "address" = ANY(?) + AND "subaccountNumber" = 0 + ) + SELECT s."address", u."username" + FROM subaccountIds s + LEFT JOIN subaccount_usernames u ON u."subaccountId" = s."id" + `, + [addresses], + ); + + return result.rows; +} diff --git a/indexer/packages/postgres/src/types/affiliate-info-types.ts b/indexer/packages/postgres/src/types/affiliate-info-types.ts index 4c7f11d108..885de8b9b7 100644 --- a/indexer/packages/postgres/src/types/affiliate-info-types.ts +++ b/indexer/packages/postgres/src/types/affiliate-info-types.ts @@ -7,6 +7,7 @@ export interface AffiliateInfoCreateObject { totalReferredUsers: number, referredNetProtocolEarnings: string, firstReferralBlockHeight: string, + referredTotalVolume: string, } export enum AffiliateInfoColumns { @@ -18,4 +19,5 @@ export enum AffiliateInfoColumns { totalReferredUsers = 'totalReferredUsers', referredNetProtocolEarnings = 'referredNetProtocolEarnings', firstReferralBlockHeight = 'firstReferralBlockHeight', + referredTotalVolume = 'referredTotalVolume', } diff --git a/indexer/packages/postgres/src/types/db-model-types.ts b/indexer/packages/postgres/src/types/db-model-types.ts index 21441874ef..363676a45d 100644 --- a/indexer/packages/postgres/src/types/db-model-types.ts +++ b/indexer/packages/postgres/src/types/db-model-types.ts @@ -264,6 +264,11 @@ export interface SubaccountUsernamesFromDatabase { subaccountId: string, } +export interface AddressUsernameFromDatabase { + address: string, + username: string, +} + export interface LeaderboardPnlFromDatabase { address: string, timeSpan: string, @@ -286,6 +291,7 @@ export interface AffiliateInfoFromDatabase { totalReferredUsers: number, referredNetProtocolEarnings: string, firstReferralBlockHeight: string, + referredTotalVolume: string, } export interface AffiliateReferredUserFromDatabase { diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts index 56565269ed..a4c76dd0f4 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/affiliates-controller.test.ts @@ -2,14 +2,21 @@ import { dbHelpers, testConstants, testMocks, + SubaccountTable, SubaccountUsernamesTable, WalletTable, AffiliateReferredUsersTable, + AffiliateInfoTable, + AffiliateInfoCreateObject, } from '@dydxprotocol-indexer/postgres'; -import { AffiliateSnapshotRequest, RequestMethod } from '../../../../src/types'; +import { + AffiliateSnapshotRequest, + AffiliateSnapshotResponse, + RequestMethod, + AffiliateSnapshotResponseObject, +} from '../../../../src/types'; import request from 'supertest'; import { sendRequest } from '../../../helpers/helpers'; -import { defaultWallet, defaultWallet2 } from '@dydxprotocol-indexer/postgres/build/__tests__/helpers/constants'; describe('affiliates-controller#V4', () => { beforeAll(async () => { @@ -110,8 +117,8 @@ describe('affiliates-controller#V4', () => { it('should classify is affiliate', async () => { await AffiliateReferredUsersTable.create({ - affiliateAddress: defaultWallet.address, - refereeAddress: defaultWallet2.address, + affiliateAddress: testConstants.defaultWallet.address, + refereeAddress: testConstants.defaultWallet2.address, referredAtBlock: '1', }); const response: request.Response = await sendRequest({ @@ -171,21 +178,60 @@ describe('affiliates-controller#V4', () => { }); describe('GET /snapshot', () => { - it('should return snapshots when all params specified', async () => { + const defaultInfo = testConstants.defaultAffiliateInfo; + const defaultInfo2 = testConstants.defaultAffiliateInfo2; + const defaultInfo3 = testConstants.defaultAffiliateInfo3; + + beforeEach(async () => { + await testMocks.seedData(); + // Create username for defaultWallet + await SubaccountUsernamesTable.create(testConstants.defaultSubaccountUsername); + + // Create defaultWallet2, subaccount, and username + await WalletTable.create(testConstants.defaultWallet2); + await SubaccountTable.create(testConstants.defaultSubaccountDefaultWalletAddress); + await SubaccountUsernamesTable.create( + testConstants.subaccountUsernameWithDefaultWalletAddress, + ); + + // Create defaultWallet3, create subaccount, create username + await WalletTable.create(testConstants.defaultWallet3); + await SubaccountTable.create(testConstants.defaultSubaccountWithAlternateAddress); + await SubaccountUsernamesTable.create(testConstants.subaccountUsernameWithAlternativeAddress); + + // Create affiliate infos + await AffiliateInfoTable.create(defaultInfo); + await AffiliateInfoTable.create(defaultInfo2); + await AffiliateInfoTable.create(defaultInfo3); + }); + + afterEach(async () => { + await dbHelpers.clearData(); + }); + + it('should filter by address', async () => { const req: AffiliateSnapshotRequest = { - limit: 10, - offset: 10, - sortByReferredFees: true, + addressFilter: [testConstants.defaultWallet.address], }; const response: request.Response = await sendRequest({ type: RequestMethod.GET, - path: `/v4/affiliates/snapshot?limit=${req.limit}&offset=${req.offset}&sortByReferredFees=${req.sortByReferredFees}`, + path: `/v4/affiliates/snapshot?addressFilter=${req.addressFilter!.join(',')}`, + expectedStatus: 200, // helper performs expect on status, }); - expect(response.status).toBe(200); - expect(response.body.affiliateList).toHaveLength(10); - expect(response.body.currentOffset).toBe(10); - expect(response.body.total).toBe(10); + const expectedResponse: AffiliateSnapshotResponse = { + affiliateList: [ + affiliateInfoCreateToResponseObject( + defaultInfo, testConstants.defaultSubaccountUsername.username, + ), + ], + total: 1, + currentOffset: 0, + }; + expect(response.body.affiliateList).toHaveLength(1); + expect(response.body.affiliateList[0]).toEqual(expectedResponse.affiliateList[0]); + expect(response.body.currentOffset).toEqual(expectedResponse.currentOffset); + expect(response.body.total).toEqual(expectedResponse.total); }); it('should return snapshots when optional params not specified', async () => { @@ -195,9 +241,39 @@ describe('affiliates-controller#V4', () => { }); expect(response.status).toBe(200); - expect(response.body.affiliateList).toHaveLength(1000); - expect(response.body.currentOffset).toBe(0); - expect(response.body.total).toBe(1000); + expect(response.body.affiliateList).toHaveLength(3); + expect(response.body.currentOffset).toEqual(0); + expect(response.body.total).toEqual(3); + }); + + it('should return snapshots when all params specified', async () => { + const req: AffiliateSnapshotRequest = { + addressFilter: [testConstants.defaultWallet.address, testConstants.defaultWallet2.address], + sortByAffiliateEarning: true, + }; + const response: request.Response = await sendRequest({ + type: RequestMethod.GET, + path: `/v4/affiliates/snapshot?${req.addressFilter!.map((address) => `addressFilter[]=${address}`).join('&')}&offset=1&limit=1&sortByAffiliateEarning=${req.sortByAffiliateEarning}`, + expectedStatus: 200, // helper performs expect on status + }); + + // addressFilter removes defaultInfo3 + // sortorder -> [defaultInfo2, defaultInfo] + // offset=1 -> defaultInfo + const expectedResponse: AffiliateSnapshotResponse = { + affiliateList: [ + affiliateInfoCreateToResponseObject( + defaultInfo, testConstants.defaultSubaccountUsername.username, + ), + ], + total: 1, + currentOffset: 1, + }; + + expect(response.body.affiliateList).toHaveLength(1); + expect(response.body.currentOffset).toEqual(expectedResponse.currentOffset); + expect(response.body.total).toEqual(expectedResponse.total); + expect(response.body.affiliateList[0]).toEqual(expectedResponse.affiliateList[0]); }); }); @@ -239,3 +315,20 @@ describe('affiliates-controller#V4', () => { }); }); }); + +function affiliateInfoCreateToResponseObject( + info: AffiliateInfoCreateObject, + username: string, +): AffiliateSnapshotResponseObject { + return { + affiliateAddress: info.address, + affiliateReferralCode: username, + affiliateEarnings: Number(info.affiliateEarnings), + affiliateReferredTrades: + Number(info.referredTakerTrades) + Number(info.referredMakerTrades), + affiliateTotalReferredFees: Number(info.totalReferredFees), + affiliateReferredUsers: Number(info.totalReferredUsers), + affiliateReferredNetProtocolEarnings: Number(info.referredNetProtocolEarnings), + affiliateReferredTotalVolume: Number(info.referredTotalVolume), + }; +} diff --git a/indexer/services/comlink/public/api-documentation.md b/indexer/services/comlink/public/api-documentation.md index 0439c0e85f..ceb40bff96 100644 --- a/indexer/services/comlink/public/api-documentation.md +++ b/indexer/services/comlink/public/api-documentation.md @@ -633,9 +633,10 @@ fetch(`${baseURL}/affiliates/snapshot`, |Name|In|Type|Required|Description| |---|---|---|---|---| +|addressFilter|query|array[string]|false|none| |offset|query|number(double)|false|none| |limit|query|number(double)|false|none| -|sortByReferredFees|query|boolean|false|none| +|sortByAffiliateEarning|query|boolean|false|none| > Example responses diff --git a/indexer/services/comlink/public/swagger.json b/indexer/services/comlink/public/swagger.json index 5fc7a5cff8..ff388856a5 100644 --- a/indexer/services/comlink/public/swagger.json +++ b/indexer/services/comlink/public/swagger.json @@ -1816,6 +1816,17 @@ }, "security": [], "parameters": [ + { + "in": "query", + "name": "addressFilter", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, { "in": "query", "name": "offset", @@ -1836,7 +1847,7 @@ }, { "in": "query", - "name": "sortByReferredFees", + "name": "sortByAffiliateEarning", "required": false, "schema": { "type": "boolean" diff --git a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts index 841d37c300..0a64429f36 100644 --- a/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/affiliates-controller.ts @@ -1,6 +1,7 @@ -import { stats } from '@dydxprotocol-indexer/base'; +import { logger, stats } from '@dydxprotocol-indexer/base'; import { WalletTable, + AffiliateInfoTable, AffiliateReferredUsersTable, SubaccountTable, SubaccountUsernamesTable, @@ -116,38 +117,63 @@ class AffiliatesController extends Controller { @Get('/snapshot') async getSnapshot( - @Query() offset?: number, + @Query() addressFilter?: string[], + @Query() offset?: number, @Query() limit?: number, - @Query() sortByReferredFees?: boolean, + @Query() sortByAffiliateEarning?: boolean, ): Promise { + const finalAddressFilter = addressFilter ?? []; const finalOffset = offset ?? 0; const finalLimit = limit ?? 1000; - // eslint-disable-next-line - const finalSortByReferredFees = sortByReferredFees ?? false; + const finalsortByAffiliateEarning = sortByAffiliateEarning ?? false; - // simulate a delay - await new Promise((resolve) => setTimeout(resolve, 100)); + const infos = await AffiliateInfoTable.paginatedFindWithAddressFilter( + finalAddressFilter, + finalOffset, + finalLimit, + finalsortByAffiliateEarning, + ); - const snapshot: AffiliateSnapshotResponseObject = { - affiliateAddress: 'some_address', - affiliateReferralCode: 'TempCode123', - affiliateEarnings: 100, - affiliateReferredTrades: 1000, - affiliateTotalReferredFees: 100, - affiliateReferredUsers: 10, - affiliateReferredNetProtocolEarnings: 1000, - affiliateReferredTotalVolume: 1000000, - }; + // No results found + if (infos === undefined) { + return { + affiliateList: [], + total: finalLimit, + currentOffset: finalOffset, + }; + } - const affiliateSnapshots: AffiliateSnapshotResponseObject[] = []; - for (let i = 0; i < finalLimit; i++) { - affiliateSnapshots.push(snapshot); + // Get referral codes + const addressUsernames = await SubaccountUsernamesTable.findByAddress( + infos.map((info) => info.address), + ); + const addressUsernameMap: Record = {}; + addressUsernames.forEach((addressUsername) => { + addressUsernameMap[addressUsername.address] = addressUsername.username; + }); + if (addressUsernames.length !== infos.length) { + logger.warning({ + at: 'affiliates-controller#snapshot', + message: `Could not find referral code for following addresses: ${infos.map((info) => info.address).filter((address) => !(address in addressUsernameMap)).join(', ')}`, + }); } + const affiliateSnapshots: AffiliateSnapshotResponseObject[] = infos.map((info) => ({ + affiliateAddress: info.address, + affiliateReferralCode: + info.address in addressUsernameMap ? addressUsernameMap[info.address] : '', + affiliateEarnings: Number(info.affiliateEarnings), + affiliateReferredTrades: Number(info.referredMakerTrades) + Number(info.referredTakerTrades), + affiliateTotalReferredFees: Number(info.totalReferredFees), + affiliateReferredUsers: Number(info.totalReferredUsers), + affiliateReferredNetProtocolEarnings: Number(info.referredNetProtocolEarnings), + affiliateReferredTotalVolume: Number(info.referredTotalVolume), + })); + const response: AffiliateSnapshotResponse = { affiliateList: affiliateSnapshots, - total: finalLimit, currentOffset: finalOffset, + total: affiliateSnapshots.length, }; return response; @@ -251,26 +277,46 @@ router.get( '/snapshot', rateLimiterMiddleware(getReqRateLimiter), ...checkSchema({ + addressFilter: { + in: ['query'], + optional: true, + customSanitizer: { + options: (value) => { + // Split the comma-separated string into an array + return typeof value === 'string' ? value.split(',') : value; + }, + }, + custom: { + options: (values) => { + return Array.isArray(values) && values.length > 0 && values.every((val) => typeof val === 'string'); + }, + }, + errorMessage: 'addressFilter must be a non-empy array of comma separated strings', + }, offset: { in: ['query'], - isInt: true, + optional: true, // Make sure this is the first rule + isInt: { + options: { min: 0 }, + }, toInt: true, - optional: true, errorMessage: 'offset must be a valid integer', }, limit: { in: ['query'], - isInt: true, + optional: true, // Make sure this is the first rule + isInt: { + options: { min: 1 }, + }, toInt: true, - optional: true, errorMessage: 'limit must be a valid integer', }, - sortByReferredFees: { + sortByAffiliateEarning: { in: ['query'], isBoolean: true, toBoolean: true, optional: true, - errorMessage: 'sortByReferredFees must be a boolean', + errorMessage: 'sortByAffiliateEarning must be a boolean', }, }), handleValidationErrors, @@ -278,17 +324,19 @@ router.get( async (req: express.Request, res: express.Response) => { const start: number = Date.now(); const { + addressFilter, offset, limit, - sortByReferredFees, + sortByAffiliateEarning, }: AffiliateSnapshotRequest = matchedData(req) as AffiliateSnapshotRequest; try { const controller: AffiliatesController = new AffiliatesController(); const response: AffiliateSnapshotResponse = await controller.getSnapshot( + addressFilter, offset, limit, - sortByReferredFees, + sortByAffiliateEarning, ); return res.send(response); } catch (error) { diff --git a/indexer/services/comlink/src/types.ts b/indexer/services/comlink/src/types.ts index 2867e0827b..6269eecdc3 100644 --- a/indexer/services/comlink/src/types.ts +++ b/indexer/services/comlink/src/types.ts @@ -684,9 +684,10 @@ export interface AffiliateAddressRequest{ } export interface AffiliateSnapshotRequest{ + addressFilter?: string[], limit?: number, offset?: number, - sortByReferredFees?: boolean, + sortByAffiliateEarning?: boolean, } export interface AffiliateTotalVolumeRequest{ From 6f4447a318161235a9e0dedd7912c85d7c6a1194 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Wed, 18 Sep 2024 13:18:47 -0400 Subject: [PATCH 09/21] minor edits --- .../stores/affiliate-info-table.test.ts | 39 +++++++++---------- .../__tests__/stores/wallet-table.test.ts | 17 ++++---- ...l_precision_and_add_totalreferredvolume.ts | 4 +- .../src/stores/affiliate-info-table.ts | 6 +-- .../postgres/src/stores/wallet-table.ts | 4 +- .../tasks/update-affiliate-info.test.ts | 17 ++++---- .../src/tasks/update-affiliate-info.ts | 11 +++--- .../src/tasks/update-wallet-total-volume.ts | 9 ++--- 8 files changed, 52 insertions(+), 55 deletions(-) diff --git a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts index eeb324509a..7a85da185e 100644 --- a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts @@ -1,4 +1,4 @@ -import { PersistentCacheKeys, AffiliateInfoFromDatabase, Liquidity } from '../../src/types'; +import { PersistentCacheKeys, AffiliateInfoFromDatabase, Liquidity, PersistentCacheFromDatabase } from '../../src/types'; import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; import { defaultOrder, @@ -45,7 +45,7 @@ describe('Affiliate info store', () => { it('Can upsert affiliate info multiple times', async () => { await AffiliateInfoTable.upsert(defaultAffiliateInfo); - let info: AffiliateInfoFromDatabase = await AffiliateInfoTable.findById( + let info: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( defaultAffiliateInfo.address, ); expect(info).toEqual(expect.objectContaining(defaultAffiliateInfo)); @@ -77,7 +77,7 @@ describe('Affiliate info store', () => { it('Successfully finds affiliate info by Id', async () => { await AffiliateInfoTable.create(defaultAffiliateInfo); - const info: AffiliateInfoFromDatabase = await AffiliateInfoTable.findById( + const info: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( defaultAffiliateInfo.address, ); expect(info).toEqual(expect.objectContaining(defaultAffiliateInfo)); @@ -85,7 +85,7 @@ describe('Affiliate info store', () => { describe('Affiliate info .updateInfo()', () => { it('Successfully creates new affiliate info', async () => { - const referenceDt = await populateFillsAndReferrals(); + const referenceDt: DateTime = await populateFillsAndReferrals(); // Perform update await AffiliateInfoTable.updateInfo( @@ -110,11 +110,11 @@ describe('Affiliate info store', () => { referredTotalVolume: '2', }; - expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + expect(updatedInfo).toEqual(expectedAffiliateInfo); }); - it('Successfully updates/increments affiliate info, both stats and metadata', async () => { - const referenceDt = await populateFillsAndReferrals(); + it('Successfully updates/increments affiliate info for stats and new referrals', async () => { + const referenceDt: DateTime = await populateFillsAndReferrals(); // Perform update: catches first 2 fills await AffiliateInfoTable.updateInfo( @@ -136,7 +136,7 @@ describe('Affiliate info store', () => { firstReferralBlockHeight: '1', referredTotalVolume: '2', }; - expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + expect(updatedInfo).toEqual(expectedAffiliateInfo); // Perform update: catches next 2 fills await AffiliateInfoTable.updateInfo( @@ -158,7 +158,7 @@ describe('Affiliate info store', () => { firstReferralBlockHeight: '1', referredTotalVolume: '4', }; - expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + expect(updatedInfo).toEqual(expectedAffiliateInfo); // Perform update: catches no fills but new affiliate referral await AffiliateReferredUsersTable.create({ @@ -184,21 +184,20 @@ describe('Affiliate info store', () => { firstReferralBlockHeight: '1', referredTotalVolume: '4', }; - expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + expect(updatedInfo).toEqual(expectedAffiliateInfo); }); it('Successfully upserts persistent cache', async () => { - const referenceDt = await populateFillsAndReferrals(); + const referenceDt: DateTime = await populateFillsAndReferrals(); // First update sets persistent cache await AffiliateInfoTable.updateInfo( referenceDt.minus({ minutes: 2 }).toISO(), referenceDt.minus({ minutes: 1 }).toISO(), ); - let persistentCache = await PersistentCacheTable.findById( - PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, - ); - let lastUpdateTime = persistentCache?.value; + let persistentCache: PersistentCacheFromDatabase | undefined = await PersistentCacheTable + .findById(PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME); + let lastUpdateTime: string | undefined = persistentCache?.value; expect(lastUpdateTime).not.toBeUndefined(); if (lastUpdateTime !== undefined) { expect(lastUpdateTime).toEqual(referenceDt.minus({ minutes: 1 }).toISO()); @@ -220,7 +219,7 @@ describe('Affiliate info store', () => { }); it('Does not use fills from before referal block height', async () => { - const referenceDt = DateTime.utc(); + const referenceDt: DateTime = DateTime.utc(); await seedData(); await OrderTable.create(defaultOrder); @@ -249,11 +248,11 @@ describe('Affiliate info store', () => { referenceDt.toISO(), ); - const updatedInfo = await AffiliateInfoTable.findById( + const updatedInfo: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( defaultWallet2.address, ); // expect one referred user but no fill stats - const expectedAffiliateInfo = { + const expectedAffiliateInfo: AffiliateInfoFromDatabase = { address: defaultWallet2.address, affiliateEarnings: '0', referredMakerTrades: 0, @@ -262,9 +261,9 @@ describe('Affiliate info store', () => { totalReferredUsers: 1, referredNetProtocolEarnings: '0', firstReferralBlockHeight: '2', - totalReferredVolume: '0', + referredTotalVolume: '0', }; - expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + expect(updatedInfo).toEqual(expectedAffiliateInfo); }); }); diff --git a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts index ca2c3b9c06..620625f273 100644 --- a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts @@ -1,4 +1,4 @@ -import { WalletFromDatabase, PersistentCacheKeys } from '../../src/types'; +import { WalletFromDatabase, PersistentCacheKeys, PersistentCacheFromDatabase } from '../../src/types'; import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; import { DateTime } from 'luxon'; import { @@ -92,14 +92,14 @@ describe('Wallet store', () => { describe('Wallet .updateTotalVolume()', () => { it('Successfully updates totalVolume for time window multiple times', async () => { - const firstFillTime = await populateWalletSubaccountFill(); + const firstFillTime: DateTime = await populateWalletSubaccountFill(); // Update totalVolume for a time window that covers all fills await WalletTable.updateTotalVolume( firstFillTime.minus({ hours: 1 }).toISO(), // need to minus because left bound is exclusive firstFillTime.plus({ hours: 1 }).toISO(), ); - let wallet = await WalletTable.findById(defaultWallet.address); + let wallet: WalletFromDatabase | undefined = await WalletTable.findById(defaultWallet.address); expect(wallet).toEqual(expect.objectContaining({ ...defaultWallet, totalVolume: '103', @@ -123,15 +123,14 @@ describe('Wallet store', () => { const referenceDt = DateTime.utc(); // Sets initial persistent cache value - let leftBound = referenceDt.minus({ hours: 2 }); - let rightBound = referenceDt.minus({ hours: 1 }); + let leftBound: DateTime = referenceDt.minus({ hours: 2 }); + let rightBound: DateTime = referenceDt.minus({ hours: 1 }); await WalletTable.updateTotalVolume(leftBound.toISO(), rightBound.toISO()); - let persistentCache = await PersistentCacheTable.findById( - PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, - ); - let lastUpdateTime = persistentCache?.value; + let persistentCache: PersistentCacheFromDatabase | undefined = await PersistentCacheTable + .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); + let lastUpdateTime: string | undefined = persistentCache?.value; expect(lastUpdateTime).not.toBeUndefined(); if (lastUpdateTime !== undefined) { expect(lastUpdateTime).toEqual(rightBound.toISO()); diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_totalreferredvolume.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_totalreferredvolume.ts index 1aa4cd1ea0..9f656dfefd 100644 --- a/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_totalreferredvolume.ts +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_totalreferredvolume.ts @@ -9,7 +9,7 @@ export async function up(knex: Knex): Promise { table.decimal('totalReferredFees', null).notNullable().defaultTo(0).alter(); table.decimal('referredNetProtocolEarnings', null).notNullable().defaultTo(0).alter(); - table.decimal('totalReferredVolume', null).notNullable().defaultTo(0); + table.decimal('referredTotalVolume', null).notNullable().defaultTo(0); }); } @@ -19,6 +19,6 @@ export async function down(knex: Knex): Promise { table.decimal('totalReferredFees').notNullable().defaultTo(0).alter(); table.decimal('referredNetProtocolEarnings').notNullable().defaultTo(0).alter(); - table.dropColumn('totalReferredVolume'); + table.dropColumn('referredTotalVolume'); }); } diff --git a/indexer/packages/postgres/src/stores/affiliate-info-table.ts b/indexer/packages/postgres/src/stores/affiliate-info-table.ts index 7070d99d40..1fe9cd24ce 100644 --- a/indexer/packages/postgres/src/stores/affiliate-info-table.ts +++ b/indexer/packages/postgres/src/stores/affiliate-info-table.ts @@ -1,7 +1,7 @@ import { QueryBuilder } from 'objection'; import { DEFAULT_POSTGRES_OPTIONS } from '../constants'; -import { knexReadReplica } from '../helpers/knex'; +import { knexPrimary } from '../helpers/knex'; import { setupBaseQuery, verifyAllRequiredFields } from '../helpers/stores-helpers'; import Transaction from '../helpers/transaction'; import AffiliateInfoModel from '../models/affiliate-info-model'; @@ -87,7 +87,7 @@ export async function upsert( export async function findById( address: string, options: Options = DEFAULT_POSTGRES_OPTIONS, -): Promise { +): Promise { const baseQuery: QueryBuilder = setupBaseQuery( AffiliateInfoModel, options, @@ -102,7 +102,7 @@ export async function updateInfo( windowEndTs: string, // inclusive ) : Promise { - await knexReadReplica.getConnection().raw( + await knexPrimary.raw( ` BEGIN; diff --git a/indexer/packages/postgres/src/stores/wallet-table.ts b/indexer/packages/postgres/src/stores/wallet-table.ts index 840e1a10b1..8b459a2840 100644 --- a/indexer/packages/postgres/src/stores/wallet-table.ts +++ b/indexer/packages/postgres/src/stores/wallet-table.ts @@ -1,7 +1,7 @@ import { PartialModelObject, QueryBuilder } from 'objection'; import { DEFAULT_POSTGRES_OPTIONS } from '../constants'; -import { knexReadReplica } from '../helpers/knex'; +import { knexPrimary } from '../helpers/knex'; import { setupBaseQuery, verifyAllRequiredFields } from '../helpers/stores-helpers'; import Transaction from '../helpers/transaction'; import WalletModel from '../models/wallet-model'; @@ -124,7 +124,7 @@ export async function updateTotalVolume( windowEndTs: string, ) : Promise { - await knexReadReplica.getConnection().raw( + await knexPrimary.raw( ` BEGIN; diff --git a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts index 4f2247a178..c4f53233d2 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts @@ -66,7 +66,7 @@ describe('update-affiliate-info', () => { firstReferralBlockHeight: '1', referredTotalVolume: '0', }; - expect(updatedInfo).toEqual(expect.objectContaining(expectedAffiliateInfo)); + expect(updatedInfo).toEqual(expectedAffiliateInfo); // Check that persistent cache updated const lastUpdateTime1 = await getAffiliateInfoUpdateTime(); @@ -155,13 +155,13 @@ describe('update-affiliate-info', () => { }); // Simulate backfill - let backfillTime = await getAffiliateInfoUpdateTime(); + let backfillTime: DateTime | undefined = await getAffiliateInfoUpdateTime(); while (backfillTime !== undefined && DateTime.fromISO(backfillTime.toISO()) < currentDt) { await affiliateInfoUpdateTask(); backfillTime = await getAffiliateInfoUpdateTime(); } - const expectedAffiliateInfo = { + const expectedAffiliateInfo: AffiliateInfoFromDatabase = { address: testConstants.defaultWallet2.address, affiliateEarnings: '1000', referredMakerTrades: 0, @@ -170,9 +170,10 @@ describe('update-affiliate-info', () => { totalReferredUsers: 1, referredNetProtocolEarnings: '1000', firstReferralBlockHeight: '1', - totalReferredVolume: '2', + referredTotalVolume: '2', }; - const updatedInfo = await AffiliateInfoTable.findById(testConstants.defaultWallet2.address); + const updatedInfo: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable + .findById(testConstants.defaultWallet2.address); expect(updatedInfo).toEqual(expectedAffiliateInfo); }); @@ -181,7 +182,7 @@ describe('update-affiliate-info', () => { // `defaultLastUpdateTime` value to emulate backfilling from very beginning expect(await getAffiliateInfoUpdateTime()).toBeUndefined(); - const referenceDt = DateTime.fromISO('2020-01-01T00:00:00Z'); + const referenceDt = DateTime.fromISO('2023-10-26T00:00:00Z'); // defaultWallet2 will be affiliate and defaultWallet will be referee await AffiliateReferredUsersTable.create({ @@ -217,7 +218,7 @@ describe('update-affiliate-info', () => { await affiliateInfoUpdateTask(); } - const expectedAffiliateInfo = { + const expectedAffiliateInfo: AffiliateInfoFromDatabase = { address: testConstants.defaultWallet2.address, affiliateEarnings: '1000', referredMakerTrades: 0, @@ -226,7 +227,7 @@ describe('update-affiliate-info', () => { totalReferredUsers: 1, referredNetProtocolEarnings: '1000', firstReferralBlockHeight: '1', - totalReferredVolume: '2', + referredTotalVolume: '2', }; const updatedInfo = await AffiliateInfoTable.findById(testConstants.defaultWallet2.address); expect(updatedInfo).toEqual(expectedAffiliateInfo); diff --git a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts index 4dff9e92f6..20f2d20505 100644 --- a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts +++ b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts @@ -1,10 +1,10 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; -import { PersistentCacheTable, AffiliateInfoTable, PersistentCacheKeys } from '@dydxprotocol-indexer/postgres'; +import { PersistentCacheTable, AffiliateInfoTable, PersistentCacheKeys, PersistentCacheFromDatabase } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; import config from '../config'; -const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; +const defaultLastUpdateTime: string = '2023-10-26T00:00:00Z'; /** * Update the affiliate info for all affiliate addresses. @@ -12,9 +12,8 @@ const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; export default async function runTask(): Promise { try { const start = Date.now(); - const persistentCacheEntry = await PersistentCacheTable.findById( - PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, - ); + const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable + .findById(PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME); if (!persistentCacheEntry) { logger.info({ @@ -23,7 +22,7 @@ export default async function runTask(): Promise { }); } - const lastUpdateTime = DateTime.fromISO(persistentCacheEntry + const lastUpdateTime: DateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); let windowEndTime = DateTime.utc(); diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index d14fa2fce8..eef5c96cd4 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -1,5 +1,5 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; -import { PersistentCacheTable, WalletTable, PersistentCacheKeys } from '@dydxprotocol-indexer/postgres'; +import { PersistentCacheTable, WalletTable, PersistentCacheKeys, PersistentCacheFromDatabase } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; import config from '../config'; @@ -12,9 +12,8 @@ const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; export default async function runTask(): Promise { try { const start = Date.now(); - const persistentCacheEntry = await PersistentCacheTable.findById( - PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, - ); + const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable + .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); if (!persistentCacheEntry) { logger.info({ @@ -23,7 +22,7 @@ export default async function runTask(): Promise { }); } - const lastUpdateTime = DateTime.fromISO(persistentCacheEntry + const lastUpdateTime: DateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); let windowEndTime = DateTime.utc(); From d6fdbb4173468fea9c142c8298e485fed6aa7033 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Wed, 18 Sep 2024 13:41:35 -0400 Subject: [PATCH 10/21] minor edit --- .../stores/affiliate-info-table.test.ts | 27 ++++++++++--------- .../__tests__/stores/wallet-table.test.ts | 5 ++-- .../src/tasks/update-affiliate-info.ts | 4 ++- .../src/tasks/update-wallet-total-volume.ts | 6 +++-- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts index 7a85da185e..a8e893b191 100644 --- a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts @@ -1,4 +1,6 @@ -import { PersistentCacheKeys, AffiliateInfoFromDatabase, Liquidity, PersistentCacheFromDatabase } from '../../src/types'; +import { + PersistentCacheKeys, AffiliateInfoFromDatabase, Liquidity, PersistentCacheFromDatabase, +} from '../../src/types'; import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; import { defaultOrder, @@ -270,23 +272,24 @@ describe('Affiliate info store', () => { describe('paginatedFindWithAddressFilter', () => { beforeEach(async () => { await migrate(); - for (let i = 0; i < 10; i++) { - await AffiliateInfoTable.create({ + await Promise.all( + Array.from({ length: 10 }, (_, i) => AffiliateInfoTable.create({ ...defaultAffiliateInfo, address: `address_${i}`, affiliateEarnings: i.toString(), - }); - } + }), + ), + ); }); it('Successfully filters by address', async () => { - // eslint-disable-next-line max-len - const infos: AffiliateInfoFromDatabase[] | undefined = await AffiliateInfoTable.paginatedFindWithAddressFilter( - ['address_0'], - 0, - 10, - false, - ); + const infos: AffiliateInfoFromDatabase[] | undefined = await AffiliateInfoTable + .paginatedFindWithAddressFilter( + ['address_0'], + 0, + 10, + false, + ); expect(infos).toBeDefined(); expect(infos!.length).toEqual(1); expect(infos![0]).toEqual(expect.objectContaining({ diff --git a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts index 620625f273..5dc516c57a 100644 --- a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts @@ -99,7 +99,8 @@ describe('Wallet store', () => { firstFillTime.minus({ hours: 1 }).toISO(), // need to minus because left bound is exclusive firstFillTime.plus({ hours: 1 }).toISO(), ); - let wallet: WalletFromDatabase | undefined = await WalletTable.findById(defaultWallet.address); + let wallet: WalletFromDatabase | undefined = await WalletTable + .findById(defaultWallet.address); expect(wallet).toEqual(expect.objectContaining({ ...defaultWallet, totalVolume: '103', @@ -129,7 +130,7 @@ describe('Wallet store', () => { await WalletTable.updateTotalVolume(leftBound.toISO(), rightBound.toISO()); let persistentCache: PersistentCacheFromDatabase | undefined = await PersistentCacheTable - .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); + .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); let lastUpdateTime: string | undefined = persistentCache?.value; expect(lastUpdateTime).not.toBeUndefined(); if (lastUpdateTime !== undefined) { diff --git a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts index 20f2d20505..426fa2124a 100644 --- a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts +++ b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts @@ -1,5 +1,7 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; -import { PersistentCacheTable, AffiliateInfoTable, PersistentCacheKeys, PersistentCacheFromDatabase } from '@dydxprotocol-indexer/postgres'; +import { + PersistentCacheTable, AffiliateInfoTable, PersistentCacheKeys, PersistentCacheFromDatabase, +} from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; import config from '../config'; diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index eef5c96cd4..f205d9bcda 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -1,5 +1,7 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; -import { PersistentCacheTable, WalletTable, PersistentCacheKeys, PersistentCacheFromDatabase } from '@dydxprotocol-indexer/postgres'; +import { + PersistentCacheTable, WalletTable, PersistentCacheKeys, PersistentCacheFromDatabase, +} from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; import config from '../config'; @@ -13,7 +15,7 @@ export default async function runTask(): Promise { try { const start = Date.now(); const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable - .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); + .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); if (!persistentCacheEntry) { logger.info({ From b0c02f49918ee0b78d3ea736ad712fa6bae955b5 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Wed, 18 Sep 2024 13:41:35 -0400 Subject: [PATCH 11/21] minor edit --- .../stores/affiliate-info-table.test.ts | 36 ++++++++++++------- .../__tests__/stores/wallet-table.test.ts | 5 +-- .../src/tasks/update-affiliate-info.ts | 4 ++- .../src/tasks/update-wallet-total-volume.ts | 6 ++-- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts index 7a85da185e..bbab81e3e6 100644 --- a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts @@ -1,4 +1,6 @@ -import { PersistentCacheKeys, AffiliateInfoFromDatabase, Liquidity, PersistentCacheFromDatabase } from '../../src/types'; +import { + PersistentCacheKeys, AffiliateInfoFromDatabase, Liquidity, PersistentCacheFromDatabase, +} from '../../src/types'; import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; import { defaultOrder, @@ -83,6 +85,15 @@ describe('Affiliate info store', () => { expect(info).toEqual(expect.objectContaining(defaultAffiliateInfo)); }); + it('Returns undefined if affiliate info not found by Id', async () => { + await AffiliateInfoTable.create(defaultAffiliateInfo); + + const info: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( + 'non_existent_address', + ); + expect(info).toBeUndefined(); + }); + describe('Affiliate info .updateInfo()', () => { it('Successfully creates new affiliate info', async () => { const referenceDt: DateTime = await populateFillsAndReferrals(); @@ -270,23 +281,24 @@ describe('Affiliate info store', () => { describe('paginatedFindWithAddressFilter', () => { beforeEach(async () => { await migrate(); - for (let i = 0; i < 10; i++) { - await AffiliateInfoTable.create({ + await Promise.all( + Array.from({ length: 10 }, (_, i) => AffiliateInfoTable.create({ ...defaultAffiliateInfo, address: `address_${i}`, affiliateEarnings: i.toString(), - }); - } + }), + ), + ); }); it('Successfully filters by address', async () => { - // eslint-disable-next-line max-len - const infos: AffiliateInfoFromDatabase[] | undefined = await AffiliateInfoTable.paginatedFindWithAddressFilter( - ['address_0'], - 0, - 10, - false, - ); + const infos: AffiliateInfoFromDatabase[] | undefined = await AffiliateInfoTable + .paginatedFindWithAddressFilter( + ['address_0'], + 0, + 10, + false, + ); expect(infos).toBeDefined(); expect(infos!.length).toEqual(1); expect(infos![0]).toEqual(expect.objectContaining({ diff --git a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts index 620625f273..5dc516c57a 100644 --- a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts @@ -99,7 +99,8 @@ describe('Wallet store', () => { firstFillTime.minus({ hours: 1 }).toISO(), // need to minus because left bound is exclusive firstFillTime.plus({ hours: 1 }).toISO(), ); - let wallet: WalletFromDatabase | undefined = await WalletTable.findById(defaultWallet.address); + let wallet: WalletFromDatabase | undefined = await WalletTable + .findById(defaultWallet.address); expect(wallet).toEqual(expect.objectContaining({ ...defaultWallet, totalVolume: '103', @@ -129,7 +130,7 @@ describe('Wallet store', () => { await WalletTable.updateTotalVolume(leftBound.toISO(), rightBound.toISO()); let persistentCache: PersistentCacheFromDatabase | undefined = await PersistentCacheTable - .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); + .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); let lastUpdateTime: string | undefined = persistentCache?.value; expect(lastUpdateTime).not.toBeUndefined(); if (lastUpdateTime !== undefined) { diff --git a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts index 20f2d20505..426fa2124a 100644 --- a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts +++ b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts @@ -1,5 +1,7 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; -import { PersistentCacheTable, AffiliateInfoTable, PersistentCacheKeys, PersistentCacheFromDatabase } from '@dydxprotocol-indexer/postgres'; +import { + PersistentCacheTable, AffiliateInfoTable, PersistentCacheKeys, PersistentCacheFromDatabase, +} from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; import config from '../config'; diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index eef5c96cd4..f205d9bcda 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -1,5 +1,7 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; -import { PersistentCacheTable, WalletTable, PersistentCacheKeys, PersistentCacheFromDatabase } from '@dydxprotocol-indexer/postgres'; +import { + PersistentCacheTable, WalletTable, PersistentCacheKeys, PersistentCacheFromDatabase, +} from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; import config from '../config'; @@ -13,7 +15,7 @@ export default async function runTask(): Promise { try { const start = Date.now(); const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable - .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); + .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); if (!persistentCacheEntry) { logger.info({ From 38c7da38cce5b1b5569a0f3736866fd0cbe0ba4d Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Wed, 18 Sep 2024 13:41:35 -0400 Subject: [PATCH 12/21] minor edit --- .../__tests__/stores/affiliate-info-table.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts index a8e893b191..bbab81e3e6 100644 --- a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts @@ -85,6 +85,15 @@ describe('Affiliate info store', () => { expect(info).toEqual(expect.objectContaining(defaultAffiliateInfo)); }); + it('Returns undefined if affiliate info not found by Id', async () => { + await AffiliateInfoTable.create(defaultAffiliateInfo); + + const info: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( + 'non_existent_address', + ); + expect(info).toBeUndefined(); + }); + describe('Affiliate info .updateInfo()', () => { it('Successfully creates new affiliate info', async () => { const referenceDt: DateTime = await populateFillsAndReferrals(); From 6a681a0b8962071d712a1fec91e343a8639ac1f9 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 19 Sep 2024 11:12:46 -0400 Subject: [PATCH 13/21] pr revision --- ...e_info_decimal_precision_and_add_total_referred_volume.ts} | 0 .../roundtable/__tests__/tasks/update-affiliate-info.test.ts | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename indexer/packages/postgres/src/db/migrations/migration_files/{20240910101430_change_affiliate_info_decimal_precision_and_add_referredtotalvolume.ts => 20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts} (100%) diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_referredtotalvolume.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts similarity index 100% rename from indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_referredtotalvolume.ts rename to indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts diff --git a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts index c4f53233d2..7e38d52da5 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts @@ -109,7 +109,7 @@ describe('update-affiliate-info', () => { referredTotalVolume: '1', }; expect(updatedInfo).toEqual(expectedAffiliateInfo); - const lastUpdateTime2 = await getAffiliateInfoUpdateTime(); + const lastUpdateTime2: DateTime | undefined = await getAffiliateInfoUpdateTime(); if (lastUpdateTime2 !== undefined && lastUpdateTime1 !== undefined) { expect(lastUpdateTime2.toMillis()) .toBeGreaterThan(lastUpdateTime1.toMillis()); @@ -117,7 +117,7 @@ describe('update-affiliate-info', () => { }); it('Successfully backfills from past date', async () => { - const currentDt = DateTime.utc(); + const currentDt: DateTime = DateTime.utc(); // Set persistent cache to 3 weeks ago to emulate backfill from 3 weeks. await PersistentCacheTable.create({ From c4036b090efacb5d925c802d2afeed3cd2de3850 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 19 Sep 2024 13:16:20 -0400 Subject: [PATCH 14/21] address most PR comments --- ...precision_and_add_total_referred_volume.ts | 17 ++++---- ...157_change_affiliate_info_default_value.ts | 17 ++++++++ .../tasks/update-affiliate-info.test.ts | 39 +++++++++++-------- .../tasks/update-wallet-total-volume.test.ts | 8 ++-- indexer/services/roundtable/src/config.ts | 4 ++ indexer/services/roundtable/src/index.ts | 9 ++++- .../src/tasks/update-affiliate-info.ts | 20 +++++----- .../src/tasks/update-wallet-total-volume.ts | 18 ++++----- 8 files changed, 83 insertions(+), 49 deletions(-) create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20240919142157_change_affiliate_info_default_value.ts diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts index efa701960f..4edc686e19 100644 --- a/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts @@ -1,23 +1,22 @@ import * as Knex from 'knex'; -// No data has been stored added at time of commit export async function up(knex: Knex): Promise { return knex.schema.alterTable('affiliate_info', (table) => { // null indicates variable precision whereas not specifying will result in 8,2 precision,scale - table.decimal('affiliateEarnings', null).notNullable().defaultTo(0).alter(); - table.decimal('totalReferredFees', null).notNullable().defaultTo(0).alter(); - table.decimal('referredNetProtocolEarnings', null).notNullable().defaultTo(0).alter(); + table.decimal('affiliateEarnings', null).alter(); + table.decimal('totalReferredFees', null).alter(); + table.decimal('referredNetProtocolEarnings', null).alter(); - table.decimal('referredTotalVolume', null).notNullable().defaultTo(0); + table.decimal('referredTotalVolume', null).notNullable(); }); } export async function down(knex: Knex): Promise { return knex.schema.alterTable('affiliate_info', (table) => { - table.decimal('affiliateEarnings').notNullable().defaultTo(0).alter(); - table.decimal('totalReferredFees').notNullable().defaultTo(0).alter(); - table.decimal('referredNetProtocolEarnings').notNullable().defaultTo(0).alter(); + table.decimal('affiliateEarnings').alter(); + table.decimal('totalReferredFees').alter(); + table.decimal('referredNetProtocolEarnings').alter(); table.dropColumn('referredTotalVolume'); }); -} +} \ No newline at end of file diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240919142157_change_affiliate_info_default_value.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240919142157_change_affiliate_info_default_value.ts new file mode 100644 index 0000000000..9b19769be7 --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240919142157_change_affiliate_info_default_value.ts @@ -0,0 +1,17 @@ +import * as Knex from 'knex'; + +export async function up(knex: Knex): Promise { + return knex.schema.alterTable('affiliate_info', (table) => { + table.decimal('affiliateEarnings', null).notNullable().defaultTo(0).alter(); + table.decimal('totalReferredFees', null).notNullable().defaultTo(0).alter(); + table.decimal('referredNetProtocolEarnings', null).notNullable().defaultTo(0).alter(); + }); +} + +export async function down(knex: Knex): Promise { + return knex.schema.alterTable('affiliate_info', (table) => { + table.decimal('affiliateEarnings', null).alter(); + table.decimal('totalReferredFees', null).alter(); + table.decimal('referredNetProtocolEarnings', null).alter(); + }); +} diff --git a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts index 7e38d52da5..3baa574817 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts @@ -10,6 +10,7 @@ import { AffiliateInfoFromDatabase, AffiliateInfoTable, Liquidity, + PersistentCacheFromDatabase, } from '@dydxprotocol-indexer/postgres'; import affiliateInfoUpdateTask from '../../src/tasks/update-affiliate-info'; import { DateTime } from 'luxon'; @@ -37,19 +38,22 @@ describe('update-affiliate-info', () => { it('Successfully updates affiliate info and persistent cache multiple times', async () => { const startDt = DateTime.utc(); - // Set persistent cache affiliateInfoUpdateTIme so task does not use backfill windows - await PersistentCacheTable.create({ - key: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, - value: startDt.toISO(), - }); + await Promise.all([ + // Set persistent cache affiliateInfoUpdateTime so task does not use backfill windows + PersistentCacheTable.create({ + key: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + value: startDt.toISO(), + }), + + // First task run: add referral w/o any fills + // defaultWallet2 will be affiliate and defaultWallet will be referee + AffiliateReferredUsersTable.create({ + affiliateAddress: testConstants.defaultWallet2.address, + refereeAddress: testConstants.defaultWallet.address, + referredAtBlock: '1', + }), + ]); - // First task run: add refereal w/o any fills - // defaultWallet2 will be affiliate and defaultWallet will be referee - await AffiliateReferredUsersTable.create({ - affiliateAddress: testConstants.defaultWallet2.address, - refereeAddress: testConstants.defaultWallet.address, - referredAtBlock: '1', - }); await affiliateInfoUpdateTask(); let updatedInfo: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( @@ -182,7 +186,7 @@ describe('update-affiliate-info', () => { // `defaultLastUpdateTime` value to emulate backfilling from very beginning expect(await getAffiliateInfoUpdateTime()).toBeUndefined(); - const referenceDt = DateTime.fromISO('2023-10-26T00:00:00Z'); + const referenceDt: DateTime = DateTime.fromISO('2024-09-16T00:00:00Z'); // defaultWallet2 will be affiliate and defaultWallet will be referee await AffiliateReferredUsersTable.create({ @@ -235,10 +239,11 @@ describe('update-affiliate-info', () => { }); async function getAffiliateInfoUpdateTime(): Promise { - const persistentCache = await PersistentCacheTable.findById( - PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, - ); - const lastUpdateTime = persistentCache?.value + const persistentCache: PersistentCacheFromDatabase | undefined = await PersistentCacheTable + .findById( + PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + ); + const lastUpdateTime: DateTime | undefined = persistentCache?.value ? DateTime.fromISO(persistentCache.value) : undefined; return lastUpdateTime; diff --git a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts index 16455b2b15..a0a26fdad9 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts @@ -7,6 +7,7 @@ import { FillTable, OrderTable, PersistentCacheKeys, + PersistentCacheFromDatabase, } from '@dydxprotocol-indexer/postgres'; import walletTotalVolumeUpdateTask from '../../src/tasks/update-wallet-total-volume'; import { DateTime } from 'luxon'; @@ -189,9 +190,10 @@ describe('update-wallet-total-volume', () => { }); async function getTotalVolumeUpdateTime(): Promise { - const persistentCache = await PersistentCacheTable.findById( - PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, - ); + const persistentCache: PersistentCacheFromDatabase | undefined = await PersistentCacheTable + .findById( + PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, + ); const lastUpdateTime = persistentCache?.value ? DateTime.fromISO(persistentCache.value) : undefined; diff --git a/indexer/services/roundtable/src/config.ts b/indexer/services/roundtable/src/config.ts index 2e712c54ac..a8ec2cb87b 100644 --- a/indexer/services/roundtable/src/config.ts +++ b/indexer/services/roundtable/src/config.ts @@ -58,6 +58,7 @@ export const configSchema = { LOOPS_ENABLED_LEADERBOARD_PNL_MONTHLY: parseBoolean({ default: false }), LOOPS_ENABLED_LEADERBOARD_PNL_YEARLY: parseBoolean({ default: false }), LOOPS_ENABLED_UPDATE_WALLET_TOTAL_VOLUME: parseBoolean({ default: true }), + LOOPS_ENABLED_UPDATE_AFFILIATE_INFO: parseBoolean({ default: true }), LOOPS_ENABLED_DELETE_OLD_FIREBASE_NOTIFICATION_TOKENS: parseBoolean({ default: true }), // Loop Timing @@ -130,6 +131,9 @@ export const configSchema = { LOOPS_INTERVAL_MS_UPDATE_WALLET_TOTAL_VOLUME: parseInteger({ default: THIRTY_SECONDS_IN_MILLISECONDS, }), + LOOPS_INTERVAL_MS_UPDATE_AFFILIATE_INFO: parseInteger({ + default: THIRTY_SECONDS_IN_MILLISECONDS, + }), LOOPS_INTERVAL_MS_DELETE_FIREBASE_NOTIFICATION_TOKENS_MONTHLY: parseInteger({ default: 30 * ONE_DAY_IN_MILLISECONDS, }), diff --git a/indexer/services/roundtable/src/index.ts b/indexer/services/roundtable/src/index.ts index 09b83c5e77..f52903ac19 100644 --- a/indexer/services/roundtable/src/index.ts +++ b/indexer/services/roundtable/src/index.ts @@ -26,6 +26,7 @@ import subaccountUsernameGeneratorTask from './tasks/subaccount-username-generat import takeFastSyncSnapshotTask from './tasks/take-fast-sync-snapshot'; import trackLag from './tasks/track-lag'; import uncrossOrderbookTask from './tasks/uncross-orderbook'; +import updateAffiliateInfoTask from './tasks/update-affiliate-info'; import updateComplianceDataTask from './tasks/update-compliance-data'; import updateResearchEnvironmentTask from './tasks/update-research-environment'; import updateWalletTotalVolumeTask from './tasks/update-wallet-total-volume'; @@ -256,7 +257,13 @@ async function start(): Promise { config.LOOPS_INTERVAL_MS_UPDATE_WALLET_TOTAL_VOLUME, ); } - + if (config.LOOPS_ENABLED_UPDATE_AFFILIATE_INFO) { + startLoop( + updateAffiliateInfoTask, + 'update_affiliate_info', + config.LOOPS_INTERVAL_MS_UPDATE_AFFILIATE_INFO, + ); + } if (config.LOOPS_ENABLED_DELETE_OLD_FIREBASE_NOTIFICATION_TOKENS) { startLoop( deleteOldFirebaseNotificationTokensTask, diff --git a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts index 426fa2124a..66eb09c7a8 100644 --- a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts +++ b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts @@ -1,19 +1,18 @@ -import { logger, stats } from '@dydxprotocol-indexer/base'; +import { logger } from '@dydxprotocol-indexer/base'; import { PersistentCacheTable, AffiliateInfoTable, PersistentCacheKeys, PersistentCacheFromDatabase, + BlockFromDatabase, + BlockTable, } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; -import config from '../config'; - -const defaultLastUpdateTime: string = '2023-10-26T00:00:00Z'; +const defaultLastUpdateTime: string = '2024-09-16T00:00:00Z'; /** * Update the affiliate info for all affiliate addresses. */ export default async function runTask(): Promise { try { - const start = Date.now(); const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable .findById(PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME); @@ -27,7 +26,12 @@ export default async function runTask(): Promise { const lastUpdateTime: DateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); - let windowEndTime = DateTime.utc(); + + const latestBlock: BlockFromDatabase = await BlockTable.getLatest(); + if (latestBlock.time === null) { + throw Error('Failed to get latest block time'); + } + let windowEndTime = DateTime.fromISO(latestBlock.time); // During backfilling, we process one day at a time to reduce roundtable runtime. if (windowEndTime > lastUpdateTime.plus({ days: 1 })) { @@ -36,10 +40,6 @@ export default async function runTask(): Promise { await AffiliateInfoTable.updateInfo(lastUpdateTime.toISO(), windowEndTime.toISO()); - stats.timing( - `${config.SERVICE_NAME}.update_affiliate_info_timing`, - Date.now() - start, - ); } catch (error) { logger.error({ at: 'update-affiliate-info#runTask', diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index f205d9bcda..bd9a8fc6d9 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -1,11 +1,11 @@ -import { logger, stats } from '@dydxprotocol-indexer/base'; +import { logger } from '@dydxprotocol-indexer/base'; import { PersistentCacheTable, WalletTable, PersistentCacheKeys, PersistentCacheFromDatabase, + BlockFromDatabase, + BlockTable, } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; -import config from '../config'; - const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; /** @@ -13,7 +13,6 @@ const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; */ export default async function runTask(): Promise { try { - const start = Date.now(); const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); @@ -27,7 +26,12 @@ export default async function runTask(): Promise { const lastUpdateTime: DateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); - let windowEndTime = DateTime.utc(); + + const latestBlock: BlockFromDatabase = await BlockTable.getLatest(); + if (latestBlock.time === null) { + throw Error('Failed to get latest block time'); + } + let windowEndTime = DateTime.fromISO(latestBlock.time); // During backfilling, we process one day at a time to reduce roundtable runtime. if (windowEndTime > lastUpdateTime.plus({ days: 1 })) { @@ -36,10 +40,6 @@ export default async function runTask(): Promise { await WalletTable.updateTotalVolume(lastUpdateTime.toISO(), windowEndTime.toISO()); - stats.timing( - `${config.SERVICE_NAME}.update_wallet_total_volume_timing`, - Date.now() - start, - ); } catch (error) { logger.error({ at: 'update-wallet-total-volume#runTask', From dedb955b5500ae67c1af02a24883510e37d63256 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 19 Sep 2024 15:36:31 -0400 Subject: [PATCH 15/21] resolve race condition and revert wallet roundtable --- .../stores/affiliate-info-table.test.ts | 34 +----- ...precision_and_add_total_referred_volume.ts | 2 +- .../src/stores/affiliate-info-table.ts | 51 +++++--- .../tasks/update-affiliate-info.test.ts | 114 +++++++++++------- .../src/tasks/update-affiliate-info.ts | 39 ++++-- .../src/tasks/update-wallet-total-volume.ts | 17 +-- 6 files changed, 143 insertions(+), 114 deletions(-) diff --git a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts index bbab81e3e6..e068a4d59a 100644 --- a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts @@ -1,5 +1,5 @@ import { - PersistentCacheKeys, AffiliateInfoFromDatabase, Liquidity, PersistentCacheFromDatabase, + AffiliateInfoFromDatabase, Liquidity, } from '../../src/types'; import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; import { @@ -19,7 +19,6 @@ import * as AffiliateInfoTable from '../../src/stores/affiliate-info-table'; import * as OrderTable from '../../src/stores/order-table'; import * as AffiliateReferredUsersTable from '../../src/stores/affiliate-referred-users-table'; import * as FillTable from '../../src/stores/fill-table'; -import * as PersistentCacheTable from '../../src/stores/persistent-cache-table'; import { seedData } from '../helpers/mock-generators'; import { DateTime } from 'luxon'; @@ -198,37 +197,6 @@ describe('Affiliate info store', () => { expect(updatedInfo).toEqual(expectedAffiliateInfo); }); - it('Successfully upserts persistent cache', async () => { - const referenceDt: DateTime = await populateFillsAndReferrals(); - - // First update sets persistent cache - await AffiliateInfoTable.updateInfo( - referenceDt.minus({ minutes: 2 }).toISO(), - referenceDt.minus({ minutes: 1 }).toISO(), - ); - let persistentCache: PersistentCacheFromDatabase | undefined = await PersistentCacheTable - .findById(PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME); - let lastUpdateTime: string | undefined = persistentCache?.value; - expect(lastUpdateTime).not.toBeUndefined(); - if (lastUpdateTime !== undefined) { - expect(lastUpdateTime).toEqual(referenceDt.minus({ minutes: 1 }).toISO()); - } - - // Second update upserts persistent cache - await AffiliateInfoTable.updateInfo( - referenceDt.minus({ minutes: 1 }).toISO(), - referenceDt.toISO(), - ); - persistentCache = await PersistentCacheTable.findById( - PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, - ); - lastUpdateTime = persistentCache?.value; - expect(lastUpdateTime).not.toBeUndefined(); - if (lastUpdateTime !== undefined) { - expect(lastUpdateTime).toEqual(referenceDt.toISO()); - } - }); - it('Does not use fills from before referal block height', async () => { const referenceDt: DateTime = DateTime.utc(); diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts index 4edc686e19..53f6d6d373 100644 --- a/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20240913142157_change_affiliate_info_decimal_precision_and_add_total_referred_volume.ts @@ -19,4 +19,4 @@ export async function down(knex: Knex): Promise { table.dropColumn('referredTotalVolume'); }); -} \ No newline at end of file +} diff --git a/indexer/packages/postgres/src/stores/affiliate-info-table.ts b/indexer/packages/postgres/src/stores/affiliate-info-table.ts index 1fe9cd24ce..0c1ca1d120 100644 --- a/indexer/packages/postgres/src/stores/affiliate-info-table.ts +++ b/indexer/packages/postgres/src/stores/affiliate-info-table.ts @@ -15,8 +15,8 @@ import { AffiliateInfoFromDatabase, AffiliateInfoQueryConfig, Liquidity, - PersistentCacheKeys, } from '../types'; +import Knex from 'knex'; export async function findAll( { @@ -97,15 +97,27 @@ export async function findById( .returning('*'); } +/** + * Updates affiliate information in the database based on the provided time window. + * + * This function aggregates affiliate-related metadata and fill statistics + * from various tables. Then it upserts the aggregated data into the `affiliate_info` table. + * + * @async + * @function updateInfo + * @param {string} windowStartTs - The exclusive start timestamp for filtering fills. + * @param {string} windowEndTs - The inclusive end timestamp for filtering fill. + * @param {Options} [options={ txId: undefined }] - Optional transaction ID or additional options. + * @returns {Promise} + */ export async function updateInfo( windowStartTs: string, // exclusive windowEndTs: string, // inclusive + options: Options = { txId: undefined }, ) : Promise { - - await knexPrimary.raw( - ` -BEGIN; - + const transaction: Knex.Transaction | undefined = Transaction.get(options.txId); + + const query = ` -- Get metadata for all affiliates -- STEP 1: Aggregate affiliate_referred_users WITH affiliate_metadata AS ( @@ -245,17 +257,26 @@ DO UPDATE SET "totalReferredFees" = affiliate_info."totalReferredFees" + EXCLUDED."totalReferredFees", "referredNetProtocolEarnings" = affiliate_info."referredNetProtocolEarnings" + EXCLUDED."referredNetProtocolEarnings", "referredTotalVolume" = affiliate_info."referredTotalVolume" + EXCLUDED."referredTotalVolume"; + ` --- Step 5: Upsert new affiliateInfoLastUpdateTime to persistent_cache table -INSERT INTO persistent_cache (key, value) -VALUES ('${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME}', '${windowEndTs}') -ON CONFLICT (key) -DO UPDATE SET value = EXCLUDED.value; - -COMMIT; - `, - ); + return transaction + ? knexPrimary.raw(query).transacting(transaction) + : knexPrimary.raw(query); } + +/** + * Finds affiliate information from the database with optional address filtering, sorting, + * and offset based pagination. + * + * @async + * @function paginatedFindWithAddressFilter + * @param {string[]} addressFilter - An array of affiliate addresses to filter by. + * @param {number} offset - The offset for pagination. + * @param {number} limit - The maximum number of records to return. + * @param {boolean} sortByAffiliateEarning - Sort the results by affiliate earnings in desc order. + * @param {Options} [options=DEFAULT_POSTGRES_OPTIONS] - Optional config for database interaction. + * @returns {Promise} + */ export async function paginatedFindWithAddressFilter( addressFilter: string[], offset: number, diff --git a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts index 3baa574817..83460ce7e5 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts @@ -11,6 +11,7 @@ import { AffiliateInfoTable, Liquidity, PersistentCacheFromDatabase, + BlockTable, } from '@dydxprotocol-indexer/postgres'; import affiliateInfoUpdateTask from '../../src/tasks/update-affiliate-info'; import { DateTime } from 'luxon'; @@ -36,24 +37,29 @@ describe('update-affiliate-info', () => { }); it('Successfully updates affiliate info and persistent cache multiple times', async () => { - const startDt = DateTime.utc(); + const startDt: DateTime = DateTime.utc(); + // Set persistent cache affiliateInfoUpdateTime to now so task does not backfill + await PersistentCacheTable.create({ + key: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + value: startDt.toISO(), + }); - await Promise.all([ - // Set persistent cache affiliateInfoUpdateTime so task does not use backfill windows - PersistentCacheTable.create({ - key: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, - value: startDt.toISO(), - }), + // First task run: add referral w/o any fills + // defaultWallet2 will be affiliate and defaultWallet will be referee + await AffiliateReferredUsersTable.create({ + affiliateAddress: testConstants.defaultWallet2.address, + refereeAddress: testConstants.defaultWallet.address, + referredAtBlock: '1', + }); - // First task run: add referral w/o any fills - // defaultWallet2 will be affiliate and defaultWallet will be referee - AffiliateReferredUsersTable.create({ - affiliateAddress: testConstants.defaultWallet2.address, - refereeAddress: testConstants.defaultWallet.address, - referredAtBlock: '1', - }), - ]); + // Create block to simulate time passing + let updatedDt: DateTime = DateTime.utc(); + await BlockTable.create({ + blockHeight: '3', + time: updatedDt.toISO(), + }); + // Run task await affiliateInfoUpdateTask(); let updatedInfo: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( @@ -73,27 +79,34 @@ describe('update-affiliate-info', () => { expect(updatedInfo).toEqual(expectedAffiliateInfo); // Check that persistent cache updated - const lastUpdateTime1 = await getAffiliateInfoUpdateTime(); - if (lastUpdateTime1 !== undefined) { - expect(lastUpdateTime1.toMillis()) - .toBeGreaterThan(startDt.toMillis()); + let lastUpdateTime: DateTime | undefined = await getAffiliateInfoUpdateTime(); + if (lastUpdateTime !== undefined) { + expect(lastUpdateTime.toMillis()).toEqual(updatedDt.toMillis()); } // Second task run: one new fill and one new referral - await FillTable.create({ - ...testConstants.defaultFill, - liquidity: Liquidity.TAKER, - createdAt: DateTime.utc().toISO(), - eventId: testConstants.defaultTendermintEventId, - price: '1', - size: '1', - fee: '1000', - affiliateRevShare: '500', - }); - await AffiliateReferredUsersTable.create({ - affiliateAddress: testConstants.defaultWallet2.address, - refereeAddress: testConstants.defaultWallet3.address, - referredAtBlock: '2', + await Promise.all([ + FillTable.create({ + ...testConstants.defaultFill, + liquidity: Liquidity.TAKER, + createdAt: DateTime.utc().toISO(), + eventId: testConstants.defaultTendermintEventId, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }), + AffiliateReferredUsersTable.create({ + affiliateAddress: testConstants.defaultWallet2.address, + refereeAddress: testConstants.defaultWallet3.address, + referredAtBlock: '2', + }), + ]); + + updatedDt = DateTime.utc(); + await BlockTable.create({ + blockHeight: '4', + time: updatedDt.toISO(), }); await affiliateInfoUpdateTask(); @@ -113,10 +126,9 @@ describe('update-affiliate-info', () => { referredTotalVolume: '1', }; expect(updatedInfo).toEqual(expectedAffiliateInfo); - const lastUpdateTime2: DateTime | undefined = await getAffiliateInfoUpdateTime(); - if (lastUpdateTime2 !== undefined && lastUpdateTime1 !== undefined) { - expect(lastUpdateTime2.toMillis()) - .toBeGreaterThan(lastUpdateTime1.toMillis()); + lastUpdateTime = await getAffiliateInfoUpdateTime(); + if (lastUpdateTime !== undefined) { + expect(lastUpdateTime.toMillis()).toEqual(updatedDt.toMillis()); } }); @@ -158,6 +170,12 @@ describe('update-affiliate-info', () => { affiliateRevShare: '500', }); + // Create block at current time + await BlockTable.create({ + blockHeight: '3', + time: DateTime.utc().toISO(), + }); + // Simulate backfill let backfillTime: DateTime | undefined = await getAffiliateInfoUpdateTime(); while (backfillTime !== undefined && DateTime.fromISO(backfillTime.toISO()) < currentDt) { @@ -182,8 +200,10 @@ describe('update-affiliate-info', () => { }); it('Successfully backfills on first run', async () => { - // Leave persistent cache affiliateInfoUpdateTime empty and create fills around - // `defaultLastUpdateTime` value to emulate backfilling from very beginning + // We will simulate a 1 week backfill from the beginning time of + // `defaultLastUpdateTime`=2024-09-16T00:00:00Z. We do this by leaving persistent cache + // affiliateInfoUpdateTime empty and create fills around `defaultLastUpdateTime`. Then we run + // the backfill 7 times. expect(await getAffiliateInfoUpdateTime()).toBeUndefined(); const referenceDt: DateTime = DateTime.fromISO('2024-09-16T00:00:00Z'); @@ -195,11 +215,11 @@ describe('update-affiliate-info', () => { referredAtBlock: '1', }); - // Fills spannings 2 weeks after referenceDt + // Fills spannings 7 days after referenceDt await FillTable.create({ ...testConstants.defaultFill, liquidity: Liquidity.TAKER, - createdAt: referenceDt.plus({ weeks: 1 }).toISO(), + createdAt: referenceDt.plus({ days: 1 }).toISO(), eventId: testConstants.defaultTendermintEventId, price: '1', size: '1', @@ -209,7 +229,7 @@ describe('update-affiliate-info', () => { await FillTable.create({ ...testConstants.defaultFill, liquidity: Liquidity.TAKER, - createdAt: referenceDt.plus({ weeks: 2 }).toISO(), + createdAt: referenceDt.plus({ days: 7 }).toISO(), eventId: testConstants.defaultTendermintEventId2, price: '1', size: '1', @@ -217,8 +237,14 @@ describe('update-affiliate-info', () => { affiliateRevShare: '500', }); - // Simulate 20 roundtable runs (this is enough to backfill all the fills) - for (let i = 0; i < 20; i++) { + // Create block in the future relative to referenceDt + await BlockTable.create({ + blockHeight: '3', + time: referenceDt.plus({ days: 7 }).toISO(), + }); + + // Simulate roundtable runs + for (let i = 0; i < 7; i++) { await affiliateInfoUpdateTask(); } diff --git a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts index 66eb09c7a8..cecd0a2a7d 100644 --- a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts +++ b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts @@ -1,8 +1,13 @@ import { logger } from '@dydxprotocol-indexer/base'; import { - PersistentCacheTable, AffiliateInfoTable, PersistentCacheKeys, PersistentCacheFromDatabase, + PersistentCacheTable, + AffiliateInfoTable, + PersistentCacheKeys, + PersistentCacheFromDatabase, BlockFromDatabase, BlockTable, + Transaction, + IsolationLevel, } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; @@ -12,35 +17,43 @@ const defaultLastUpdateTime: string = '2024-09-16T00:00:00Z'; * Update the affiliate info for all affiliate addresses. */ export default async function runTask(): Promise { + const latestBlock: BlockFromDatabase = await BlockTable.getLatest(); + if (latestBlock.time === null) { + throw Error('Failed to get latest block time'); + } + + // Wrap getting cache, updating info, and setting cache in one transaction with row locking to + // prevent race condition on persistent cache rows between read and write. + const txId: number = await Transaction.start(); + await Transaction.setIsolationLevel(txId, IsolationLevel.REPEATABLE_READ); try { const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable - .findById(PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME); - + .findById(PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, { txId }); if (!persistentCacheEntry) { logger.info({ at: 'update-affiliate-info#runTask', message: `No previous ${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME} found in persistent cache table. Will use default value: ${defaultLastUpdateTime}`, }); } - - const lastUpdateTime: DateTime = DateTime.fromISO(persistentCacheEntry + const windowStartTime: DateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); - const latestBlock: BlockFromDatabase = await BlockTable.getLatest(); - if (latestBlock.time === null) { - throw Error('Failed to get latest block time'); - } let windowEndTime = DateTime.fromISO(latestBlock.time); - // During backfilling, we process one day at a time to reduce roundtable runtime. - if (windowEndTime > lastUpdateTime.plus({ days: 1 })) { - windowEndTime = lastUpdateTime.plus({ days: 1 }); + if (windowEndTime > windowStartTime.plus({ days: 1 })) { + windowEndTime = windowStartTime.plus({ days: 1 }); } - await AffiliateInfoTable.updateInfo(lastUpdateTime.toISO(), windowEndTime.toISO()); + await AffiliateInfoTable.updateInfo(windowStartTime.toISO(), windowEndTime.toISO(), { txId }); + await PersistentCacheTable.upsert({ + key: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + value: windowEndTime.toISO(), + }, { txId }); + await Transaction.commit(txId); } catch (error) { + await Transaction.rollback(txId); logger.error({ at: 'update-affiliate-info#runTask', message: 'Error when updating affiliate info in affiliate_info table', diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index bd9a8fc6d9..0ecc4a8fa5 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -1,11 +1,11 @@ -import { logger } from '@dydxprotocol-indexer/base'; +import { logger, stats } from '@dydxprotocol-indexer/base'; import { PersistentCacheTable, WalletTable, PersistentCacheKeys, PersistentCacheFromDatabase, - BlockFromDatabase, - BlockTable, } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; +import config from '../config'; + const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; /** @@ -13,6 +13,7 @@ const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; */ export default async function runTask(): Promise { try { + const start = Date.now(); const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); @@ -27,11 +28,7 @@ export default async function runTask(): Promise { ? persistentCacheEntry.value : defaultLastUpdateTime); - const latestBlock: BlockFromDatabase = await BlockTable.getLatest(); - if (latestBlock.time === null) { - throw Error('Failed to get latest block time'); - } - let windowEndTime = DateTime.fromISO(latestBlock.time); + let windowEndTime = DateTime.utc(); // During backfilling, we process one day at a time to reduce roundtable runtime. if (windowEndTime > lastUpdateTime.plus({ days: 1 })) { @@ -40,6 +37,10 @@ export default async function runTask(): Promise { await WalletTable.updateTotalVolume(lastUpdateTime.toISO(), windowEndTime.toISO()); + stats.timing( + `${config.SERVICE_NAME}.update_wallet_total_volume_timing`, + Date.now() - start, + ); } catch (error) { logger.error({ at: 'update-wallet-total-volume#runTask', From 436c4d26c988943aea0e5bc624bd1cbb54d1116d Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 19 Sep 2024 18:04:22 -0400 Subject: [PATCH 16/21] revision --- .../stores/affiliate-info-table.test.ts | 92 ++++++++-------- .../__tests__/stores/wallet-table.test.ts | 2 +- .../src/stores/affiliate-info-table.ts | 8 +- .../tasks/update-affiliate-info.test.ts | 100 +++++++++--------- .../src/tasks/update-affiliate-info.ts | 6 +- 5 files changed, 104 insertions(+), 104 deletions(-) diff --git a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts index e068a4d59a..68ccbe0cb7 100644 --- a/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/affiliate-info-table.test.ts @@ -260,7 +260,7 @@ describe('Affiliate info store', () => { }); it('Successfully filters by address', async () => { - const infos: AffiliateInfoFromDatabase[] | undefined = await AffiliateInfoTable + const infos: AffiliateInfoFromDatabase[] = await AffiliateInfoTable .paginatedFindWithAddressFilter( ['address_0'], 0, @@ -371,50 +371,52 @@ async function populateFillsAndReferrals(): Promise { // Create order and fils for defaultWallet (referee) await OrderTable.create(defaultOrder); - await FillTable.create({ - ...defaultFill, - liquidity: Liquidity.TAKER, - subaccountId: defaultOrder.subaccountId, - createdAt: referenceDt.minus({ minutes: 1 }).toISO(), - eventId: defaultTendermintEventId, - price: '1', - size: '1', - fee: '1000', - affiliateRevShare: '500', - }); - await FillTable.create({ - ...defaultFill, - liquidity: Liquidity.MAKER, - subaccountId: defaultOrder.subaccountId, - createdAt: referenceDt.minus({ minutes: 1 }).toISO(), - eventId: defaultTendermintEventId2, - price: '1', - size: '1', - fee: '1000', - affiliateRevShare: '500', - }); - await FillTable.create({ - ...defaultFill, - liquidity: Liquidity.MAKER, // use uneven number of maker/taker - subaccountId: defaultOrder.subaccountId, - createdAt: referenceDt.minus({ minutes: 2 }).toISO(), - eventId: defaultTendermintEventId3, - price: '1', - size: '1', - fee: '1000', - affiliateRevShare: '500', - }); - await FillTable.create({ - ...defaultFill, - liquidity: Liquidity.MAKER, - subaccountId: defaultOrder.subaccountId, - createdAt: referenceDt.minus({ minutes: 2 }).toISO(), - eventId: defaultTendermintEventId4, - price: '1', - size: '1', - fee: '1000', - affiliateRevShare: '500', - }); + await Promise.all([ + FillTable.create({ + ...defaultFill, + liquidity: Liquidity.TAKER, + subaccountId: defaultOrder.subaccountId, + createdAt: referenceDt.minus({ minutes: 1 }).toISO(), + eventId: defaultTendermintEventId, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }), + FillTable.create({ + ...defaultFill, + liquidity: Liquidity.MAKER, + subaccountId: defaultOrder.subaccountId, + createdAt: referenceDt.minus({ minutes: 1 }).toISO(), + eventId: defaultTendermintEventId2, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }), + FillTable.create({ + ...defaultFill, + liquidity: Liquidity.MAKER, // use uneven number of maker/taker + subaccountId: defaultOrder.subaccountId, + createdAt: referenceDt.minus({ minutes: 2 }).toISO(), + eventId: defaultTendermintEventId3, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }), + FillTable.create({ + ...defaultFill, + liquidity: Liquidity.MAKER, + subaccountId: defaultOrder.subaccountId, + createdAt: referenceDt.minus({ minutes: 2 }).toISO(), + eventId: defaultTendermintEventId4, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }), + ]); return referenceDt; } diff --git a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts index 5dc516c57a..a6baa18ae5 100644 --- a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts @@ -90,7 +90,7 @@ describe('Wallet store', () => { expect(wallet).toEqual(expect.objectContaining(defaultWallet2)); }); - describe('Wallet .updateTotalVolume()', () => { + describe('updateTotalVolume', () => { it('Successfully updates totalVolume for time window multiple times', async () => { const firstFillTime: DateTime = await populateWalletSubaccountFill(); diff --git a/indexer/packages/postgres/src/stores/affiliate-info-table.ts b/indexer/packages/postgres/src/stores/affiliate-info-table.ts index 0c1ca1d120..8230287d3f 100644 --- a/indexer/packages/postgres/src/stores/affiliate-info-table.ts +++ b/indexer/packages/postgres/src/stores/affiliate-info-table.ts @@ -1,3 +1,4 @@ +import Knex from 'knex'; import { QueryBuilder } from 'objection'; import { DEFAULT_POSTGRES_OPTIONS } from '../constants'; @@ -16,7 +17,6 @@ import { AffiliateInfoQueryConfig, Liquidity, } from '../types'; -import Knex from 'knex'; export async function findAll( { @@ -116,7 +116,7 @@ export async function updateInfo( options: Options = { txId: undefined }, ) : Promise { const transaction: Knex.Transaction | undefined = Transaction.get(options.txId); - + const query = ` -- Get metadata for all affiliates -- STEP 1: Aggregate affiliate_referred_users @@ -257,9 +257,9 @@ DO UPDATE SET "totalReferredFees" = affiliate_info."totalReferredFees" + EXCLUDED."totalReferredFees", "referredNetProtocolEarnings" = affiliate_info."referredNetProtocolEarnings" + EXCLUDED."referredNetProtocolEarnings", "referredTotalVolume" = affiliate_info."referredTotalVolume" + EXCLUDED."referredTotalVolume"; - ` + `; - return transaction + return transaction ? knexPrimary.raw(query).transacting(transaction) : knexPrimary.raw(query); } diff --git a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts index 83460ce7e5..e16ce16b72 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts @@ -53,19 +53,19 @@ describe('update-affiliate-info', () => { }); // Create block to simulate time passing - let updatedDt: DateTime = DateTime.utc(); + const updatedDt1: DateTime = DateTime.utc(); await BlockTable.create({ blockHeight: '3', - time: updatedDt.toISO(), + time: updatedDt1.toISO(), }); // Run task await affiliateInfoUpdateTask(); - let updatedInfo: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( + const updatedInfo1: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( testConstants.defaultWallet2.address, ); - let expectedAffiliateInfo: AffiliateInfoFromDatabase = { + const expectedAffiliateInfo1: AffiliateInfoFromDatabase = { address: testConstants.defaultWallet2.address, affiliateEarnings: '0', referredMakerTrades: 0, @@ -76,12 +76,12 @@ describe('update-affiliate-info', () => { firstReferralBlockHeight: '1', referredTotalVolume: '0', }; - expect(updatedInfo).toEqual(expectedAffiliateInfo); + expect(updatedInfo1).toEqual(expectedAffiliateInfo1); // Check that persistent cache updated - let lastUpdateTime: DateTime | undefined = await getAffiliateInfoUpdateTime(); - if (lastUpdateTime !== undefined) { - expect(lastUpdateTime.toMillis()).toEqual(updatedDt.toMillis()); + const lastUpdateTime1: DateTime | undefined = await getAffiliateInfoUpdateTime(); + if (lastUpdateTime1 !== undefined) { + expect(lastUpdateTime1.toMillis()).toEqual(updatedDt1.toMillis()); } // Second task run: one new fill and one new referral @@ -103,18 +103,18 @@ describe('update-affiliate-info', () => { }), ]); - updatedDt = DateTime.utc(); + const updatedDt2: DateTime = DateTime.utc(); await BlockTable.create({ blockHeight: '4', - time: updatedDt.toISO(), + time: updatedDt2.toISO(), }); await affiliateInfoUpdateTask(); - updatedInfo = await AffiliateInfoTable.findById( + const updatedInfo2: AffiliateInfoFromDatabase | undefined = await AffiliateInfoTable.findById( testConstants.defaultWallet2.address, ); - expectedAffiliateInfo = { + const expectedAffiliateInfo2: AffiliateInfoFromDatabase = { address: testConstants.defaultWallet2.address, affiliateEarnings: '500', referredMakerTrades: 0, @@ -125,10 +125,10 @@ describe('update-affiliate-info', () => { firstReferralBlockHeight: '1', referredTotalVolume: '1', }; - expect(updatedInfo).toEqual(expectedAffiliateInfo); - lastUpdateTime = await getAffiliateInfoUpdateTime(); - if (lastUpdateTime !== undefined) { - expect(lastUpdateTime.toMillis()).toEqual(updatedDt.toMillis()); + expect(updatedInfo2).toEqual(expectedAffiliateInfo2); + const lastUpdateTime2: DateTime | undefined = await getAffiliateInfoUpdateTime(); + if (lastUpdateTime2 !== undefined) { + expect(lastUpdateTime2.toMillis()).toEqual(updatedDt2.toMillis()); } }); @@ -141,40 +141,40 @@ describe('update-affiliate-info', () => { value: currentDt.minus({ weeks: 3 }).toISO(), }); - // defaultWallet2 will be affiliate and defaultWallet will be referee - await AffiliateReferredUsersTable.create({ - affiliateAddress: testConstants.defaultWallet2.address, - refereeAddress: testConstants.defaultWallet.address, - referredAtBlock: '1', - }); - - // Fills spannings 2 weeks - await FillTable.create({ - ...testConstants.defaultFill, - liquidity: Liquidity.TAKER, - createdAt: currentDt.minus({ weeks: 1 }).toISO(), - eventId: testConstants.defaultTendermintEventId, - price: '1', - size: '1', - fee: '1000', - affiliateRevShare: '500', - }); - await FillTable.create({ - ...testConstants.defaultFill, - liquidity: Liquidity.TAKER, - createdAt: currentDt.minus({ weeks: 2 }).toISO(), - eventId: testConstants.defaultTendermintEventId2, - price: '1', - size: '1', - fee: '1000', - affiliateRevShare: '500', - }); - - // Create block at current time - await BlockTable.create({ - blockHeight: '3', - time: DateTime.utc().toISO(), - }); + await Promise.all([ + // defaultWallet2 will be affiliate and defaultWallet will be referee + AffiliateReferredUsersTable.create({ + affiliateAddress: testConstants.defaultWallet2.address, + refereeAddress: testConstants.defaultWallet.address, + referredAtBlock: '1', + }), + // Fills spannings 2 weeks + FillTable.create({ + ...testConstants.defaultFill, + liquidity: Liquidity.TAKER, + createdAt: currentDt.minus({ weeks: 1 }).toISO(), + eventId: testConstants.defaultTendermintEventId, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }), + FillTable.create({ + ...testConstants.defaultFill, + liquidity: Liquidity.TAKER, + createdAt: currentDt.minus({ weeks: 2 }).toISO(), + eventId: testConstants.defaultTendermintEventId2, + price: '1', + size: '1', + fee: '1000', + affiliateRevShare: '500', + }), + // Create block at current time + await BlockTable.create({ + blockHeight: '3', + time: DateTime.utc().toISO(), + }), + ]); // Simulate backfill let backfillTime: DateTime | undefined = await getAffiliateInfoUpdateTime(); diff --git a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts index cecd0a2a7d..d2f9cbb89c 100644 --- a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts +++ b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts @@ -7,7 +7,6 @@ import { BlockFromDatabase, BlockTable, Transaction, - IsolationLevel, } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; @@ -22,10 +21,9 @@ export default async function runTask(): Promise { throw Error('Failed to get latest block time'); } - // Wrap getting cache, updating info, and setting cache in one transaction with row locking to - // prevent race condition on persistent cache rows between read and write. + // Wrap getting cache, updating info, and setting cache in one transaction so that persistent + // cache and affilitate info table are in sync. const txId: number = await Transaction.start(); - await Transaction.setIsolationLevel(txId, IsolationLevel.REPEATABLE_READ); try { const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable .findById(PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, { txId }); From bcb20ba2c93ab43df8089ccea48a6b389f138e80 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Thu, 19 Sep 2024 23:00:15 -0400 Subject: [PATCH 17/21] Wrap wallet total volume roundtable queries in transaction --- .../__tests__/stores/wallet-table.test.ts | 106 +++------ .../src/stores/affiliate-info-table.ts | 4 +- .../postgres/src/stores/wallet-table.ts | 23 +- .../tasks/update-wallet-total-volume.test.ts | 209 ++++++++++-------- .../src/tasks/update-wallet-total-volume.ts | 44 ++-- 5 files changed, 201 insertions(+), 185 deletions(-) diff --git a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts index a6baa18ae5..f90710d96a 100644 --- a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts @@ -4,7 +4,6 @@ import { DateTime } from 'luxon'; import { defaultFill, defaultOrder, - defaultSubaccount, defaultTendermintEventId, defaultTendermintEventId2, defaultTendermintEventId3, @@ -12,14 +11,14 @@ import { defaultWallet, defaultWallet2, isolatedMarketOrder, - isolatedSubaccount, + defaultSubaccountId, + isolatedSubaccountId, } from '../helpers/constants'; import * as FillTable from '../../src/stores/fill-table'; import * as OrderTable from '../../src/stores/order-table'; import * as WalletTable from '../../src/stores/wallet-table'; -import * as SubaccountTable from '../../src/stores/subaccount-table'; -import * as PersistentCacheTable from '../../src/stores/persistent-cache-table'; import { seedData } from '../helpers/mock-generators'; +import { testConstants } from 'packages/postgres/src'; describe('Wallet store', () => { beforeAll(async () => { @@ -99,9 +98,9 @@ describe('Wallet store', () => { firstFillTime.minus({ hours: 1 }).toISO(), // need to minus because left bound is exclusive firstFillTime.plus({ hours: 1 }).toISO(), ); - let wallet: WalletFromDatabase | undefined = await WalletTable + const wallet1: WalletFromDatabase | undefined = await WalletTable .findById(defaultWallet.address); - expect(wallet).toEqual(expect.objectContaining({ + expect(wallet1).toEqual(expect.objectContaining({ ...defaultWallet, totalVolume: '103', })); @@ -113,45 +112,12 @@ describe('Wallet store', () => { firstFillTime.toISO(), // exclusive -> filters out first fill from each subaccount firstFillTime.plus({ minutes: 2 }).toISO(), ); - wallet = await WalletTable.findById(defaultWallet.address); - expect(wallet).toEqual(expect.objectContaining({ + const wallet2 = await WalletTable.findById(defaultWallet.address); + expect(wallet2).toEqual(expect.objectContaining({ ...defaultWallet, totalVolume: '105', // 103 + 2 })); }); - - it('Successfully upserts persistent cache', async () => { - const referenceDt = DateTime.utc(); - - // Sets initial persistent cache value - let leftBound: DateTime = referenceDt.minus({ hours: 2 }); - let rightBound: DateTime = referenceDt.minus({ hours: 1 }); - - await WalletTable.updateTotalVolume(leftBound.toISO(), rightBound.toISO()); - - let persistentCache: PersistentCacheFromDatabase | undefined = await PersistentCacheTable - .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); - let lastUpdateTime: string | undefined = persistentCache?.value; - expect(lastUpdateTime).not.toBeUndefined(); - if (lastUpdateTime !== undefined) { - expect(lastUpdateTime).toEqual(rightBound.toISO()); - } - - // Updates persistent cache value - leftBound = referenceDt.minus({ hours: 1 }); - rightBound = referenceDt; - - await WalletTable.updateTotalVolume(leftBound.toISO(), rightBound.toISO()); - - persistentCache = await PersistentCacheTable.findById( - PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, - ); - lastUpdateTime = persistentCache?.value; - expect(lastUpdateTime).not.toBeUndefined(); - if (lastUpdateTime !== undefined) { - expect(lastUpdateTime).toEqual(rightBound.toISO()); - } - }); }); }); @@ -164,22 +130,12 @@ describe('Wallet store', () => { */ async function populateWalletSubaccountFill(): Promise { await seedData(); - await OrderTable.create(defaultOrder); - await OrderTable.create(isolatedMarketOrder); - - // seedData() creates defaultWallet with defaultSubaccount and isolatedSubaccount - const defaultSubaccountId = await SubaccountTable.findAll( - { subaccountNumber: defaultSubaccount.subaccountNumber }, - [], - {}, - ); - const isolatedSubaccountId = await SubaccountTable.findAll( - { subaccountNumber: isolatedSubaccount.subaccountNumber }, - [], - {}, - ); + await Promise.all([ + OrderTable.create(defaultOrder), + OrderTable.create(isolatedMarketOrder), + ]); - const referenceDt = DateTime.utc().minus({ hours: 1 }); + const referenceDt: DateTime = DateTime.utc().minus({ hours: 1 }); const eventIds = [ defaultTendermintEventId, defaultTendermintEventId2, @@ -188,27 +144,33 @@ async function populateWalletSubaccountFill(): Promise { ]; let eventIdx = 0; + const fillPromises: Promise[] = []; // Create 3 fills with 1 min increments for defaultSubaccount for (let i = 0; i < 3; i++) { - await FillTable.create({ - ...defaultFill, - subaccountId: defaultSubaccountId[0].id, - createdAt: referenceDt.plus({ minutes: i }).toISO(), - eventId: eventIds[eventIdx], - price: '1', - size: '1', - }); + fillPromises.push( + FillTable.create({ + ...defaultFill, + subaccountId: defaultSubaccountId, + createdAt: referenceDt.plus({ minutes: i }).toISO(), + eventId: eventIds[eventIdx], + price: '1', + size: '1', + }) + ); eventIdx += 1; } // Create 1 fill at referenceDt for isolatedSubaccount - await FillTable.create({ - ...defaultFill, - subaccountId: isolatedSubaccountId[0].id, - createdAt: referenceDt.toISO(), - eventId: eventIds[eventIdx], - price: '10', - size: '10', - }); + fillPromises.push( + FillTable.create({ + ...defaultFill, + subaccountId: isolatedSubaccountId, + createdAt: referenceDt.toISO(), + eventId: eventIds[eventIdx], + price: '10', + size: '10', + }) + ); + await Promise.all(fillPromises); return referenceDt; } diff --git a/indexer/packages/postgres/src/stores/affiliate-info-table.ts b/indexer/packages/postgres/src/stores/affiliate-info-table.ts index 8230287d3f..25f732cc24 100644 --- a/indexer/packages/postgres/src/stores/affiliate-info-table.ts +++ b/indexer/packages/postgres/src/stores/affiliate-info-table.ts @@ -111,8 +111,8 @@ export async function findById( * @returns {Promise} */ export async function updateInfo( - windowStartTs: string, // exclusive - windowEndTs: string, // inclusive + windowStartTs: string, + windowEndTs: string, options: Options = { txId: undefined }, ) : Promise { const transaction: Knex.Transaction | undefined = Transaction.get(options.txId); diff --git a/indexer/packages/postgres/src/stores/wallet-table.ts b/indexer/packages/postgres/src/stores/wallet-table.ts index 8b459a2840..1978ce6d42 100644 --- a/indexer/packages/postgres/src/stores/wallet-table.ts +++ b/indexer/packages/postgres/src/stores/wallet-table.ts @@ -17,6 +17,7 @@ import { WalletUpdateObject, PersistentCacheKeys, } from '../types'; +import Knex from 'knex'; export async function findAll( { @@ -116,18 +117,21 @@ export async function findById( * Calculates the total volume in a given time window for each address and adds the values to the * existing totalVolume values. * - * @param windowStartTs - The start timestamp of the time window (exclusive). - * @param windowEndTs - The end timestamp of the time window (inclusive). + * @async + * @function updateInfo + * @param {string} windowStartTs - The exclusive start timestamp for filtering fills. + * @param {string} windowEndTs - The inclusive end timestamp for filtering fill. + * @param {Options} [options={ txId: undefined }] - Optional transaction ID or additional options. + * @returns {Promise} */ export async function updateTotalVolume( windowStartTs: string, windowEndTs: string, + options: Options = { txId: undefined }, ) : Promise { + const transaction: Knex.Transaction | undefined = Transaction.get(options.txId); - await knexPrimary.raw( - ` - BEGIN; - + const query = ` WITH fills_total AS ( -- Step 1: Calculate total volume for each subaccountId SELECT "subaccountId", SUM("price" * "size") AS "totalVolume" @@ -161,6 +165,9 @@ export async function updateTotalVolume( DO UPDATE SET value = EXCLUDED.value; COMMIT; - `, - ); + `; + + return transaction + ? knexPrimary.raw(query).transacting(transaction) + : knexPrimary.raw(query); } diff --git a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts index a0a26fdad9..fbe76833e9 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts @@ -8,6 +8,8 @@ import { OrderTable, PersistentCacheKeys, PersistentCacheFromDatabase, + WalletFromDatabase, + BlockTable, } from '@dydxprotocol-indexer/postgres'; import walletTotalVolumeUpdateTask from '../../src/tasks/update-wallet-total-volume'; import { DateTime } from 'luxon'; @@ -33,9 +35,8 @@ describe('update-wallet-total-volume', () => { await dbHelpers.clearData(); }); - it('Successfully updates totalVolume multiple times', async () => { - // Set persistent cache totalVolumeUpdateTime so walletTotalVolumeUpdateTask() does not attempt - // to backfill + it('Successfully updates totalVolume and persistent cache multiple times', async () => { + // Set persistent cache totalVolumeUpdateTime to now so task does not backfill await PersistentCacheTable.create({ key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, value: DateTime.utc().toISO(), @@ -49,20 +50,49 @@ describe('update-wallet-total-volume', () => { price: '1', size: '1', }); + + // Create block to simulate time passing + const updatedDt1: DateTime = DateTime.utc(); + await BlockTable.create({ + blockHeight: '3', + time: updatedDt1.toISO(), + }); + + // Run task await walletTotalVolumeUpdateTask(); - let wallet = await WalletTable.findById(testConstants.defaultWallet.address); - expect(wallet).toEqual(expect.objectContaining({ + + // Check that wallet updated correctly + const wallet1: WalletFromDatabase | undefined = await WalletTable + .findById(testConstants.defaultWallet.address); + expect(wallet1).toEqual(expect.objectContaining({ ...testConstants.defaultWallet, totalVolume: '1', })); + // Check that persistent cache updated + const lastUpdateTime1: DateTime | undefined = await getTotalVolumeUpdateTime(); + if (lastUpdateTime1 !== undefined) { + expect(lastUpdateTime1.toMillis()).toEqual(updatedDt1.toMillis()); + } + // Second task run: no new fills + const updatedDt2: DateTime = DateTime.utc(); + await BlockTable.create({ + blockHeight: '4', + time: updatedDt2.toISO(), + }); await walletTotalVolumeUpdateTask(); - wallet = await WalletTable.findById(testConstants.defaultWallet.address); - expect(wallet).toEqual(expect.objectContaining({ + + const wallet2: WalletFromDatabase | undefined = await WalletTable + .findById(testConstants.defaultWallet.address); + expect(wallet2).toEqual(expect.objectContaining({ ...testConstants.defaultWallet, totalVolume: '1', })); + const lastUpdateTime2: DateTime | undefined = await getTotalVolumeUpdateTime(); + if (lastUpdateTime2 !== undefined) { + expect(lastUpdateTime2.toMillis()).toEqual(updatedDt2.toMillis()); + } // Third task run: one new fill await FillTable.create({ @@ -72,68 +102,64 @@ describe('update-wallet-total-volume', () => { price: '1', size: '1', }); + const updatedDt3: DateTime = DateTime.utc(); + await BlockTable.create({ + blockHeight: '5', + time: updatedDt3.toISO(), + }); await walletTotalVolumeUpdateTask(); - wallet = await WalletTable.findById(testConstants.defaultWallet.address); - expect(wallet).toEqual(expect.objectContaining({ + + const wallet3: WalletFromDatabase | undefined = await WalletTable + .findById(testConstants.defaultWallet.address); + expect(wallet3).toEqual(expect.objectContaining({ ...testConstants.defaultWallet, totalVolume: '2', })); - }); - - it('Successfully updates totalVolumeUpdateTime in persistent cache table', async () => { - // Set persistent cache totalVolumeUpdateTime so walletTotalVolumeUpdateTask() does not attempt - // to backfill - await PersistentCacheTable.create({ - key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, - value: DateTime.utc().toISO(), - }); - - await walletTotalVolumeUpdateTask(); - const lastUpdateTime1 = await getTotalVolumeUpdateTime(); - - await walletTotalVolumeUpdateTask(); - const lastUpdateTime2 = await getTotalVolumeUpdateTime(); - - expect(lastUpdateTime1).not.toBeUndefined(); - expect(lastUpdateTime2).not.toBeUndefined(); - if (lastUpdateTime1?.toMillis() !== undefined && lastUpdateTime2?.toMillis() !== undefined) { - expect(lastUpdateTime2.toMillis()) - .toBeGreaterThan(lastUpdateTime1.toMillis()); + const lastUpdateTime3: DateTime | undefined = await getTotalVolumeUpdateTime(); + if (lastUpdateTime3 !== undefined) { + expect(lastUpdateTime3.toMillis()).toEqual(updatedDt3.toMillis()); } }); it('Successfully backfills from past date', async () => { const currentDt: DateTime = DateTime.utc(); - // Create 3 fills spanning 2 weeks in the past - await FillTable.create({ - ...testConstants.defaultFill, - createdAt: currentDt.toISO(), - eventId: testConstants.defaultTendermintEventId, - price: '1', - size: '1', - }); - await FillTable.create({ - ...testConstants.defaultFill, - createdAt: currentDt.minus({ weeks: 1 }).toISO(), - eventId: testConstants.defaultTendermintEventId2, - price: '2', - size: '2', - }); - await FillTable.create({ - ...testConstants.defaultFill, - createdAt: currentDt.minus({ weeks: 2 }).toISO(), - eventId: testConstants.defaultTendermintEventId3, - price: '3', - size: '3', - }); - - // Set persistent cache totalVolumeUpdateTime to 3 weeks ago to emulate backfill from 3 weeks. - await PersistentCacheTable.create({ - key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, - value: currentDt.minus({ weeks: 3 }).toISO(), - }); - + await Promise.all([ + // Create 3 fills spanning 2 weeks in the past + FillTable.create({ + ...testConstants.defaultFill, + createdAt: currentDt.toISO(), + eventId: testConstants.defaultTendermintEventId, + price: '1', + size: '1', + }), + FillTable.create({ + ...testConstants.defaultFill, + createdAt: currentDt.minus({ weeks: 1 }).toISO(), + eventId: testConstants.defaultTendermintEventId2, + price: '2', + size: '2', + }), + FillTable.create({ + ...testConstants.defaultFill, + createdAt: currentDt.minus({ weeks: 2 }).toISO(), + eventId: testConstants.defaultTendermintEventId3, + price: '3', + size: '3', + }), + // Set persistent cache totalVolumeUpdateTime to 3 weeks ago to emulate backfill from 3 weeks. + PersistentCacheTable.create({ + key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, + value: currentDt.minus({ weeks: 3 }).toISO(), + }), + // Create block at current time + BlockTable.create({ + blockHeight: '3', + time: DateTime.utc().toISO(), + }), + ]); + + // Simulate backfill let backfillTime = await getTotalVolumeUpdateTime(); while (backfillTime !== undefined && DateTime.fromISO(backfillTime.toISO()) < currentDt) { await walletTotalVolumeUpdateTask(); @@ -148,36 +174,45 @@ describe('update-wallet-total-volume', () => { }); it('Successfully backfills on first run', async () => { - // Leave persistent cache totalVolumeUpdateTime empty and create fills around - // `defaultLastUpdateTime` value to emulate backfilling from very beginning + // We will simulate a 1 week backfill from the beginning time of + // `defaultLastUpdateTime`=2023-10-26T00:00:00Z. We do this by leaving persistent cache + // totalVolumeUpdateTime empty and create fills around `defaultLastUpdateTime`. Then we run + // the backfill 7 times. expect(await getTotalVolumeUpdateTime()).toBeUndefined(); - const referenceDt = DateTime.fromISO('2020-01-01T00:00:00Z'); - - await FillTable.create({ - ...testConstants.defaultFill, - createdAt: referenceDt.plus({ days: 1 }).toISO(), - eventId: testConstants.defaultTendermintEventId, - price: '1', - size: '1', - }); - await FillTable.create({ - ...testConstants.defaultFill, - createdAt: referenceDt.plus({ days: 2 }).toISO(), - eventId: testConstants.defaultTendermintEventId2, - price: '2', - size: '2', - }); - await FillTable.create({ - ...testConstants.defaultFill, - createdAt: referenceDt.plus({ days: 3 }).toISO(), - eventId: testConstants.defaultTendermintEventId3, - price: '3', - size: '3', - }); - - // Emulate 10 roundtable runs (this should backfill all the fills) - for (let i = 0; i < 10; i++) { + const referenceDt = DateTime.fromISO('2023-10-26T00:00:00Z'); + + Promise.all([ + FillTable.create({ + ...testConstants.defaultFill, + createdAt: referenceDt.plus({ days: 1 }).toISO(), + eventId: testConstants.defaultTendermintEventId, + price: '1', + size: '1', + }), + FillTable.create({ + ...testConstants.defaultFill, + createdAt: referenceDt.plus({ days: 4 }).toISO(), + eventId: testConstants.defaultTendermintEventId2, + price: '2', + size: '2', + }), + FillTable.create({ + ...testConstants.defaultFill, + createdAt: referenceDt.plus({ days: 7 }).toISO(), + eventId: testConstants.defaultTendermintEventId3, + price: '3', + size: '3', + }), + // Create block in the future relative to referenceDt + BlockTable.create({ + blockHeight: '3', + time: referenceDt.plus({ days: 7 }).toISO(), + }), + ]); + + // Simulate roundtable runs + for (let i = 0; i < 7; i++) { await walletTotalVolumeUpdateTask(); } diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index 0ecc4a8fa5..d4a8cbd78f 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -1,21 +1,32 @@ import { logger, stats } from '@dydxprotocol-indexer/base'; import { - PersistentCacheTable, WalletTable, PersistentCacheKeys, PersistentCacheFromDatabase, + PersistentCacheTable, + WalletTable, + PersistentCacheKeys, + PersistentCacheFromDatabase, + Transaction, + BlockFromDatabase, + BlockTable, } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; -import config from '../config'; - -const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; +const defaultLastUpdateTime: string = '2023-10-26T00:00:00Z'; /** * Update the total volume for each addresses in the wallet table who filled recently. */ export default async function runTask(): Promise { + // Wrap getting cache, updating info, and setting cache in one transaction so that persistent + // cache and affilitate info table are in sync. + const txId: number = await Transaction.start(); try { - const start = Date.now(); + const latestBlock: BlockFromDatabase = await BlockTable.getLatest(); + if (latestBlock.time === null) { + throw Error('Failed to get latest block time'); + } + const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable - .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); + .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, { txId }); if (!persistentCacheEntry) { logger.info({ @@ -24,24 +35,25 @@ export default async function runTask(): Promise { }); } - const lastUpdateTime: DateTime = DateTime.fromISO(persistentCacheEntry + const windowStartTime: DateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); - let windowEndTime = DateTime.utc(); - + let windowEndTime = DateTime.fromISO(latestBlock.time); // During backfilling, we process one day at a time to reduce roundtable runtime. - if (windowEndTime > lastUpdateTime.plus({ days: 1 })) { - windowEndTime = lastUpdateTime.plus({ days: 1 }); + if (windowEndTime > windowStartTime.plus({ days: 1 })) { + windowEndTime = windowStartTime.plus({ days: 1 }); } - await WalletTable.updateTotalVolume(lastUpdateTime.toISO(), windowEndTime.toISO()); + await WalletTable.updateTotalVolume(windowStartTime.toISO(), windowEndTime.toISO(), { txId }); + await PersistentCacheTable.upsert({ + key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, + value: windowEndTime.toISO(), + }, { txId }); - stats.timing( - `${config.SERVICE_NAME}.update_wallet_total_volume_timing`, - Date.now() - start, - ); + await Transaction.commit(txId); } catch (error) { + await Transaction.rollback(txId); logger.error({ at: 'update-wallet-total-volume#runTask', message: 'Error when updating totalVolume in wallets table', From 4db9ad7109760c5b3dfdea4bcfad69527fe62597 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Fri, 20 Sep 2024 11:44:28 -0400 Subject: [PATCH 18/21] minor edit --- .../__tests__/stores/wallet-table.test.ts | 7 +++---- .../src/stores/affiliate-info-table.ts | 2 +- .../postgres/src/stores/wallet-table.ts | 19 +++++-------------- .../tasks/update-wallet-total-volume.test.ts | 2 +- .../src/tasks/update-wallet-total-volume.ts | 14 +++++++++----- 5 files changed, 19 insertions(+), 25 deletions(-) diff --git a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts index f90710d96a..39381cc3b7 100644 --- a/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/wallet-table.test.ts @@ -1,4 +1,4 @@ -import { WalletFromDatabase, PersistentCacheKeys, PersistentCacheFromDatabase } from '../../src/types'; +import { WalletFromDatabase } from '../../src/types'; import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; import { DateTime } from 'luxon'; import { @@ -18,7 +18,6 @@ import * as FillTable from '../../src/stores/fill-table'; import * as OrderTable from '../../src/stores/order-table'; import * as WalletTable from '../../src/stores/wallet-table'; import { seedData } from '../helpers/mock-generators'; -import { testConstants } from 'packages/postgres/src'; describe('Wallet store', () => { beforeAll(async () => { @@ -155,7 +154,7 @@ async function populateWalletSubaccountFill(): Promise { eventId: eventIds[eventIdx], price: '1', size: '1', - }) + }), ); eventIdx += 1; } @@ -168,7 +167,7 @@ async function populateWalletSubaccountFill(): Promise { eventId: eventIds[eventIdx], price: '10', size: '10', - }) + }), ); await Promise.all(fillPromises); diff --git a/indexer/packages/postgres/src/stores/affiliate-info-table.ts b/indexer/packages/postgres/src/stores/affiliate-info-table.ts index b18568d1eb..6c2ab2adc4 100644 --- a/indexer/packages/postgres/src/stores/affiliate-info-table.ts +++ b/indexer/packages/postgres/src/stores/affiliate-info-table.ts @@ -107,7 +107,7 @@ export async function findById( * @function updateInfo * @param {string} windowStartTs - The exclusive start timestamp for filtering fills. * @param {string} windowEndTs - The inclusive end timestamp for filtering fill. - * @param {Options} [options={ txId: undefined }] - Optional transaction ID or additional options. + * @param {number} [txId] - Optional transaction ID. * @returns {Promise} */ export async function updateInfo( diff --git a/indexer/packages/postgres/src/stores/wallet-table.ts b/indexer/packages/postgres/src/stores/wallet-table.ts index 1978ce6d42..ce5e04a122 100644 --- a/indexer/packages/postgres/src/stores/wallet-table.ts +++ b/indexer/packages/postgres/src/stores/wallet-table.ts @@ -1,3 +1,4 @@ +import Knex from 'knex'; import { PartialModelObject, QueryBuilder } from 'objection'; import { DEFAULT_POSTGRES_OPTIONS } from '../constants'; @@ -15,9 +16,7 @@ import { WalletFromDatabase, WalletQueryConfig, WalletUpdateObject, - PersistentCacheKeys, } from '../types'; -import Knex from 'knex'; export async function findAll( { @@ -118,18 +117,18 @@ export async function findById( * existing totalVolume values. * * @async - * @function updateInfo + * @function updateTotalVolume * @param {string} windowStartTs - The exclusive start timestamp for filtering fills. * @param {string} windowEndTs - The inclusive end timestamp for filtering fill. - * @param {Options} [options={ txId: undefined }] - Optional transaction ID or additional options. + * @param {number} [txId] - Optional transaction ID. * @returns {Promise} */ export async function updateTotalVolume( windowStartTs: string, windowEndTs: string, - options: Options = { txId: undefined }, + txId: number | undefined = undefined, ) : Promise { - const transaction: Knex.Transaction | undefined = Transaction.get(options.txId); + const transaction: Knex.Transaction | undefined = Transaction.get(txId); const query = ` WITH fills_total AS ( @@ -157,14 +156,6 @@ export async function updateTotalVolume( SET "totalVolume" = COALESCE(wallets."totalVolume", 0) + av."totalVolume" FROM address_volume av WHERE wallets."address" = av."address"; - - -- Step 5: Upsert new totalVolumeUpdateTime to persistent_cache table - INSERT INTO persistent_cache (key, value) - VALUES ('${PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME}', '${windowEndTs}') - ON CONFLICT (key) - DO UPDATE SET value = EXCLUDED.value; - - COMMIT; `; return transaction diff --git a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts index fbe76833e9..77e4cfec40 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts @@ -182,7 +182,7 @@ describe('update-wallet-total-volume', () => { const referenceDt = DateTime.fromISO('2023-10-26T00:00:00Z'); - Promise.all([ + await Promise.all([ FillTable.create({ ...testConstants.defaultFill, createdAt: referenceDt.plus({ days: 1 }).toISO(), diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index d4a8cbd78f..7908c2bfc5 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -1,8 +1,8 @@ -import { logger, stats } from '@dydxprotocol-indexer/base'; +import { logger } from '@dydxprotocol-indexer/base'; import { - PersistentCacheTable, - WalletTable, - PersistentCacheKeys, + PersistentCacheTable, + WalletTable, + PersistentCacheKeys, PersistentCacheFromDatabase, Transaction, BlockFromDatabase, @@ -45,7 +45,11 @@ export default async function runTask(): Promise { windowEndTime = windowStartTime.plus({ days: 1 }); } - await WalletTable.updateTotalVolume(windowStartTime.toISO(), windowEndTime.toISO(), { txId }); + logger.info({ + at: 'update-wallet-total-volume#runTask', + message: `Updating wallet total volume from ${windowStartTime.toISO()} to ${windowEndTime.toISO()}`, + }); + await WalletTable.updateTotalVolume(windowStartTime.toISO(), windowEndTime.toISO(), txId); await PersistentCacheTable.upsert({ key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, value: windowEndTime.toISO(), From e62129d7e028fdd4a58c23a74f0fd50d443772ea Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Fri, 20 Sep 2024 14:30:11 -0400 Subject: [PATCH 19/21] add metrics to affiliate roundtable tasks --- ...l_precision_and_add_referredtotalvolume.ts | 23 ------------------- .../tasks/update-affiliate-info.test.ts | 20 ++++++++++++++-- .../tasks/update-wallet-total-volume.test.ts | 18 ++++++++++++++- .../src/tasks/update-affiliate-info.ts | 7 ++++++ .../src/tasks/update-wallet-total-volume.ts | 10 ++++++-- 5 files changed, 50 insertions(+), 28 deletions(-) delete mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_referredtotalvolume.ts diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_referredtotalvolume.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_referredtotalvolume.ts deleted file mode 100644 index efa701960f..0000000000 --- a/indexer/packages/postgres/src/db/migrations/migration_files/20240910101430_change_affiliate_info_decimal_precision_and_add_referredtotalvolume.ts +++ /dev/null @@ -1,23 +0,0 @@ -import * as Knex from 'knex'; - -// No data has been stored added at time of commit -export async function up(knex: Knex): Promise { - return knex.schema.alterTable('affiliate_info', (table) => { - // null indicates variable precision whereas not specifying will result in 8,2 precision,scale - table.decimal('affiliateEarnings', null).notNullable().defaultTo(0).alter(); - table.decimal('totalReferredFees', null).notNullable().defaultTo(0).alter(); - table.decimal('referredNetProtocolEarnings', null).notNullable().defaultTo(0).alter(); - - table.decimal('referredTotalVolume', null).notNullable().defaultTo(0); - }); -} - -export async function down(knex: Knex): Promise { - return knex.schema.alterTable('affiliate_info', (table) => { - table.decimal('affiliateEarnings').notNullable().defaultTo(0).alter(); - table.decimal('totalReferredFees').notNullable().defaultTo(0).alter(); - table.decimal('referredNetProtocolEarnings').notNullable().defaultTo(0).alter(); - - table.dropColumn('referredTotalVolume'); - }); -} diff --git a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts index c4f53233d2..3982c4d14d 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts @@ -1,3 +1,4 @@ +import { stats } from '@dydxprotocol-indexer/base'; import { dbHelpers, testConstants, @@ -109,7 +110,7 @@ describe('update-affiliate-info', () => { referredTotalVolume: '1', }; expect(updatedInfo).toEqual(expectedAffiliateInfo); - const lastUpdateTime2 = await getAffiliateInfoUpdateTime(); + const lastUpdateTime2: DateTime | undefined = await getAffiliateInfoUpdateTime(); if (lastUpdateTime2 !== undefined && lastUpdateTime1 !== undefined) { expect(lastUpdateTime2.toMillis()) .toBeGreaterThan(lastUpdateTime1.toMillis()); @@ -117,7 +118,7 @@ describe('update-affiliate-info', () => { }); it('Successfully backfills from past date', async () => { - const currentDt = DateTime.utc(); + const currentDt: DateTime = DateTime.utc(); // Set persistent cache to 3 weeks ago to emulate backfill from 3 weeks. await PersistentCacheTable.create({ @@ -232,6 +233,21 @@ describe('update-affiliate-info', () => { const updatedInfo = await AffiliateInfoTable.findById(testConstants.defaultWallet2.address); expect(updatedInfo).toEqual(expectedAffiliateInfo); }); + + it('Successfully records metrics', async () => { + jest.spyOn(stats, 'timing'); + jest.spyOn(stats, 'gauge'); + + await PersistentCacheTable.create({ + key: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, + value: DateTime.utc().toISO(), + }); + + await affiliateInfoUpdateTask(); + + expect(stats.gauge).toHaveBeenCalledWith(`roundtable.persistent_cache_${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME}_lag_seconds`, expect.any(Number)); + expect(stats.timing).toHaveBeenCalledWith(`roundtable.update_affiliate_info_timing`, expect.any(Number)); + }); }); async function getAffiliateInfoUpdateTime(): Promise { diff --git a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts index 16455b2b15..69daf29a28 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts @@ -1,3 +1,4 @@ +import { stats } from '@dydxprotocol-indexer/base'; import { dbHelpers, testConstants, @@ -151,7 +152,7 @@ describe('update-wallet-total-volume', () => { // `defaultLastUpdateTime` value to emulate backfilling from very beginning expect(await getTotalVolumeUpdateTime()).toBeUndefined(); - const referenceDt = DateTime.fromISO('2020-01-01T00:00:00Z'); + const referenceDt = DateTime.fromISO('2023-10-26T00:00:00Z'); await FillTable.create({ ...testConstants.defaultFill, @@ -186,6 +187,21 @@ describe('update-wallet-total-volume', () => { totalVolume: '14', // 1 + 4 + 9 })); }); + + it('Successfully records metrics', async () => { + jest.spyOn(stats, 'timing'); + jest.spyOn(stats, 'gauge'); + + await PersistentCacheTable.create({ + key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, + value: DateTime.utc().toISO(), + }); + + await walletTotalVolumeUpdateTask(); + + expect(stats.gauge).toHaveBeenCalledWith(`roundtable.persistent_cache_${PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME}_lag_seconds`, expect.any(Number)); + expect(stats.timing).toHaveBeenCalledWith(`roundtable.update_wallet_total_volume_timing`, expect.any(Number)); + }); }); async function getTotalVolumeUpdateTime(): Promise { diff --git a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts index 426fa2124a..aa361e8597 100644 --- a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts +++ b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts @@ -27,6 +27,13 @@ export default async function runTask(): Promise { const lastUpdateTime: DateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); + + // Track how long ago the last update time in persistent cache was. + stats.gauge( + `${config.SERVICE_NAME}.persistent_cache_${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME}_lag_seconds`, + Math.floor(start / 1000) - lastUpdateTime.toUnixInteger(), + ); + let windowEndTime = DateTime.utc(); // During backfilling, we process one day at a time to reduce roundtable runtime. diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index f205d9bcda..a37072dc1a 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -6,13 +6,13 @@ import { DateTime } from 'luxon'; import config from '../config'; -const defaultLastUpdateTime: string = '2020-01-01T00:00:00Z'; +const defaultLastUpdateTime: string = '2023-10-26T00:00:00Z'; /** * Update the total volume for each addresses in the wallet table who filled recently. */ export default async function runTask(): Promise { - try { + try { const start = Date.now(); const persistentCacheEntry: PersistentCacheFromDatabase | undefined = await PersistentCacheTable .findById(PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME); @@ -27,6 +27,12 @@ export default async function runTask(): Promise { const lastUpdateTime: DateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); + + stats.gauge( + `${config.SERVICE_NAME}.persistent_cache_${PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME}_lag_seconds`, + Math.floor(start / 1000) - lastUpdateTime.toUnixInteger(), + ); + let windowEndTime = DateTime.utc(); // During backfilling, we process one day at a time to reduce roundtable runtime. From 470358918652ef753713784e5fccdea7472cd02a Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Fri, 20 Sep 2024 15:26:03 -0400 Subject: [PATCH 20/21] add gauge --- .../__tests__/tasks/update-affiliate-info.test.ts | 6 ++---- .../__tests__/tasks/update-wallet-total-volume.test.ts | 6 ++---- .../services/roundtable/src/tasks/update-affiliate-info.ts | 5 +++-- .../roundtable/src/tasks/update-wallet-total-volume.ts | 5 +++-- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts index 871d3771cb..7e824eb9d7 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts @@ -266,18 +266,16 @@ describe('update-affiliate-info', () => { }); it('Successfully records metrics', async () => { - jest.spyOn(stats, 'timing'); jest.spyOn(stats, 'gauge'); await PersistentCacheTable.create({ key: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME, value: DateTime.utc().toISO(), }); - + await affiliateInfoUpdateTask(); - + expect(stats.gauge).toHaveBeenCalledWith(`roundtable.persistent_cache_${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME}_lag_seconds`, expect.any(Number)); - expect(stats.timing).toHaveBeenCalledWith(`roundtable.update_affiliate_info_timing`, expect.any(Number)); }); }); diff --git a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts index 228c5f1e67..6ef5dcec97 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts @@ -225,18 +225,16 @@ describe('update-wallet-total-volume', () => { }); it('Successfully records metrics', async () => { - jest.spyOn(stats, 'timing'); jest.spyOn(stats, 'gauge'); await PersistentCacheTable.create({ key: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME, value: DateTime.utc().toISO(), }); - + await walletTotalVolumeUpdateTask(); - + expect(stats.gauge).toHaveBeenCalledWith(`roundtable.persistent_cache_${PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME}_lag_seconds`, expect.any(Number)); - expect(stats.timing).toHaveBeenCalledWith(`roundtable.update_wallet_total_volume_timing`, expect.any(Number)); }); }); diff --git a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts index f7a6386811..929d1a3c35 100644 --- a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts +++ b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts @@ -1,4 +1,4 @@ -import { logger, stats} from '@dydxprotocol-indexer/base'; +import { logger, stats } from '@dydxprotocol-indexer/base'; import { PersistentCacheTable, AffiliateInfoTable, @@ -9,6 +9,7 @@ import { Transaction, } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; + import config from '../config'; const defaultLastUpdateTime: string = '2024-09-16T00:00:00Z'; @@ -36,7 +37,7 @@ export default async function runTask(): Promise { const windowStartTime: DateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); - + // Track how long ago the last update time (windowStartTime) in persistent cache was stats.gauge( `${config.SERVICE_NAME}.persistent_cache_${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME}_lag_seconds`, diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index 09ea9727b2..4cf52c4103 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -9,6 +9,7 @@ import { BlockTable, } from '@dydxprotocol-indexer/postgres'; import { DateTime } from 'luxon'; + import config from '../config'; const defaultLastUpdateTime: string = '2023-10-26T00:00:00Z'; @@ -39,10 +40,10 @@ export default async function runTask(): Promise { const windowStartTime: DateTime = DateTime.fromISO(persistentCacheEntry ? persistentCacheEntry.value : defaultLastUpdateTime); - + // Track how long ago the last update time (windowStartTime) in persistent cache was stats.gauge( - `${config.SERVICE_NAME}.persistent_cache_${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME}_lag_seconds`, + `${config.SERVICE_NAME}.persistent_cache_${PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME}_lag_seconds`, DateTime.utc().diff(windowStartTime).as('seconds'), ); From cade12002b190ad3314a1e1dc427c4b13ef861e5 Mon Sep 17 00:00:00 2001 From: Jerry Fan Date: Fri, 20 Sep 2024 15:48:39 -0400 Subject: [PATCH 21/21] revision --- .../__tests__/tasks/update-affiliate-info.test.ts | 6 +++++- .../__tests__/tasks/update-wallet-total-volume.test.ts | 6 +++++- .../services/roundtable/src/tasks/update-affiliate-info.ts | 1 + .../roundtable/src/tasks/update-wallet-total-volume.ts | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts index 7e824eb9d7..adc63c68e3 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-affiliate-info.test.ts @@ -275,7 +275,11 @@ describe('update-affiliate-info', () => { await affiliateInfoUpdateTask(); - expect(stats.gauge).toHaveBeenCalledWith(`roundtable.persistent_cache_${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME}_lag_seconds`, expect.any(Number)); + expect(stats.gauge).toHaveBeenCalledWith( + `roundtable.persistent_cache_${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME}_lag_seconds`, + expect.any(Number), + { cache: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME }, + ); }); }); diff --git a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts index 6ef5dcec97..9cc63afb81 100644 --- a/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts +++ b/indexer/services/roundtable/__tests__/tasks/update-wallet-total-volume.test.ts @@ -234,7 +234,11 @@ describe('update-wallet-total-volume', () => { await walletTotalVolumeUpdateTask(); - expect(stats.gauge).toHaveBeenCalledWith(`roundtable.persistent_cache_${PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME}_lag_seconds`, expect.any(Number)); + expect(stats.gauge).toHaveBeenCalledWith( + `roundtable.persistent_cache_${PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME}_lag_seconds`, + expect.any(Number), + { cache: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME }, + ); }); }); diff --git a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts index 929d1a3c35..e3546eeec9 100644 --- a/indexer/services/roundtable/src/tasks/update-affiliate-info.ts +++ b/indexer/services/roundtable/src/tasks/update-affiliate-info.ts @@ -42,6 +42,7 @@ export default async function runTask(): Promise { stats.gauge( `${config.SERVICE_NAME}.persistent_cache_${PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME}_lag_seconds`, DateTime.utc().diff(windowStartTime).as('seconds'), + { cache: PersistentCacheKeys.AFFILIATE_INFO_UPDATE_TIME }, ); let windowEndTime = DateTime.fromISO(latestBlock.time); diff --git a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts index 4cf52c4103..0286f1ef8f 100644 --- a/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts +++ b/indexer/services/roundtable/src/tasks/update-wallet-total-volume.ts @@ -45,6 +45,7 @@ export default async function runTask(): Promise { stats.gauge( `${config.SERVICE_NAME}.persistent_cache_${PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME}_lag_seconds`, DateTime.utc().diff(windowStartTime).as('seconds'), + { cache: PersistentCacheKeys.TOTAL_VOLUME_UPDATE_TIME }, ); let windowEndTime = DateTime.fromISO(latestBlock.time);