Skip to content

Commit f81fefd

Browse files
mistakiamistakia
committed
feat: add rollup-daily-balance-distribution script and updated rollup-daily table schema
1 parent 103eeab commit f81fefd

4 files changed

+223
-28
lines changed

‎db/schema.postgre.sql

+41-13
Original file line numberDiff line numberDiff line change
@@ -178,19 +178,47 @@ CREATE TABLE
178178
_00001_count integer DEFAULT 0,
179179
_000001_count integer DEFAULT 0,
180180
_000001_below_count integer DEFAULT 0,
181-
_1000000_total numeric(39) DEFAULT 0,
182-
_100000_total numeric(39) DEFAULT 0,
183-
_10000_total numeric(39) DEFAULT 0,
184-
_1000_total numeric(39) DEFAULT 0,
185-
_100_total numeric(39) DEFAULT 0,
186-
_10_total numeric(39) DEFAULT 0,
187-
_1_total numeric(39) DEFAULT 0,
188-
_01_total numeric(39) DEFAULT 0,
189-
_001_total numeric(39) DEFAULT 0,
190-
_0001_total numeric(39) DEFAULT 0,
191-
_00001_total numeric(39) DEFAULT 0,
192-
_000001_total numeric(39) DEFAULT 0,
193-
_000001_below_total numeric(39) DEFAULT 0,
181+
_1000000_total_amount_sent numeric(39) DEFAULT 0,
182+
_100000_total_amount_sent numeric(39) DEFAULT 0,
183+
_10000_total_amount_sent numeric(39) DEFAULT 0,
184+
_1000_total_amount_sent numeric(39) DEFAULT 0,
185+
_100_total_amount_sent numeric(39) DEFAULT 0,
186+
_10_total_amount_sent numeric(39) DEFAULT 0,
187+
_1_total_amount_sent numeric(39) DEFAULT 0,
188+
_01_total_amount_sent numeric(39) DEFAULT 0,
189+
_001_total_amount_sent numeric(39) DEFAULT 0,
190+
_0001_total_amount_sent numeric(39) DEFAULT 0,
191+
_00001_total_amount_sent numeric(39) DEFAULT 0,
192+
_000001_total_amount_sent numeric(39) DEFAULT 0,
193+
_000001_below_total_amount_sent numeric(39) DEFAULT 0,
194+
_1000000_total_balance numeric(39) DEFAULT 0,
195+
_100000_total_balance numeric(39) DEFAULT 0,
196+
_10000_total_balance numeric(39) DEFAULT 0,
197+
_1000_total_balance numeric(39) DEFAULT 0,
198+
_100_total_balance numeric(39) DEFAULT 0,
199+
_10_total_balance numeric(39) DEFAULT 0,
200+
_1_total_balance numeric(39) DEFAULT 0,
201+
_01_total_balance numeric(39) DEFAULT 0,
202+
_001_total_balance numeric(39) DEFAULT 0,
203+
_0001_total_balance numeric(39) DEFAULT 0,
204+
_00001_total_balance numeric(39) DEFAULT 0,
205+
_000001_total_balance numeric(39) DEFAULT 0,
206+
_000001_below_total_balance numeric(39) DEFAULT 0,
207+
_1000000_account_count integer DEFAULT 0,
208+
_1000000_account_count integer DEFAULT 0,
209+
_100000_account_count integer DEFAULT 0,
210+
_100000_account_count integer DEFAULT 0,
211+
_10000_account_count integer DEFAULT 0,
212+
_1000_account_count integer DEFAULT 0,
213+
_100_account_count integer DEFAULT 0,
214+
_10_account_count integer DEFAULT 0,
215+
_1_account_count integer DEFAULT 0,
216+
_01_account_count integer DEFAULT 0,
217+
_001_account_count integer DEFAULT 0,
218+
_0001_account_count integer DEFAULT 0,
219+
_00001_account_count integer DEFAULT 0,
220+
_000001_account_count integer DEFAULT 0,
221+
_000001_below_account_count integer DEFAULT 0,
194222
timestamp integer NOT NULL,
195223
timestamp_utc timestamp without time zone NOT NULL,
196224
CONSTRAINT unique_rollup_daily UNIQUE (timestamp)

‎db/schema.sql

+40-13
Original file line numberDiff line numberDiff line change
@@ -172,20 +172,47 @@ CREATE TABLE
172172
`_00001_count` int (11) DEFAULT 0,
173173
`_000001_count` int (11) DEFAULT 0,
174174
`_000001_below_count` int (11) DEFAULT 0,
175-
`_1000000_total` decimal(39, 0) DEFAULT 0,
176-
`_100000_total` decimal(39, 0) DEFAULT 0,
177-
`_10000_total` decimal(39, 0) DEFAULT 0,
178-
`_1000_total` decimal(39, 0) DEFAULT 0,
179-
`_100_total` decimal(39, 0) DEFAULT 0,
180-
`_10_total` decimal(39, 0) DEFAULT 0,
181-
`_1_total` decimal(39, 0) DEFAULT 0,
182-
`_01_total` decimal(39, 0) DEFAULT 0,
183-
`_001_total` decimal(39, 0) DEFAULT 0,
184-
`_0001_total` decimal(39, 0) DEFAULT 0,
185-
`_00001_total` decimal(39, 0) DEFAULT 0,
186-
`_000001_total` decimal(39, 0) DEFAULT 0,
187-
`_000001_below_total` decimal(39, 0) DEFAULT 0,
175+
`_1000000_total_amount_sent` decimal(39, 0) DEFAULT 0,
176+
`_100000_total_amount_sent` decimal(39, 0) DEFAULT 0,
177+
`_10000_total_amount_sent` decimal(39, 0) DEFAULT 0,
178+
`_1000_total_amount_sent` decimal(39, 0) DEFAULT 0,
179+
`_100_total_amount_sent` decimal(39, 0) DEFAULT 0,
180+
`_10_total_amount_sent` decimal(39, 0) DEFAULT 0,
181+
`_1_total_amount_sent` decimal(39, 0) DEFAULT 0,
182+
`_01_total_amount_sent` decimal(39, 0) DEFAULT 0,
183+
`_001_total_amount_sent` decimal(39, 0) DEFAULT 0,
184+
`_0001_total_amount_sent` decimal(39, 0) DEFAULT 0,
185+
`_00001_total_amount_sent` decimal(39, 0) DEFAULT 0,
186+
`_000001_total_amount_sent` decimal(39, 0) DEFAULT 0,
187+
`_000001_below_total_amount_sent` decimal(39, 0) DEFAULT 0,
188+
`_1000000_total_balance` decimal(39, 0) DEFAULT 0,
189+
`_100000_total_balance` decimal(39, 0) DEFAULT 0,
190+
`_10000_total_balance` decimal(39, 0) DEFAULT 0,
191+
`_1000_total_balance` decimal(39, 0) DEFAULT 0,
192+
`_100_total_balance` decimal(39, 0) DEFAULT 0,
193+
`_10_total_balance` decimal(39, 0) DEFAULT 0,
194+
`_1_total_balance` decimal(39, 0) DEFAULT 0,
195+
`_01_total_balance` decimal(39, 0) DEFAULT 0,
196+
`_001_total_balance` decimal(39, 0) DEFAULT 0,
197+
`_0001_total_balance` decimal(39, 0) DEFAULT 0,
198+
`_00001_total_balance` decimal(39, 0) DEFAULT 0,
199+
`_000001_total_balance` decimal(39, 0) DEFAULT 0,
200+
`_000001_below_total_balance` decimal(39, 0) DEFAULT 0,
201+
`_1000000_account_count` int (11) DEFAULT 0,
202+
`_100000_account_count` int (11) DEFAULT 0,
203+
`_10000_account_count` int (11) DEFAULT 0,
204+
`_1000_account_count` int (11) DEFAULT 0,
205+
`_100_account_count` int (11) DEFAULT 0,
206+
`_10_account_count` int (11) DEFAULT 0,
207+
`_1_account_count` int (11) DEFAULT 0,
208+
`_01_account_count` int (11) DEFAULT 0,
209+
`_001_account_count` int (11) DEFAULT 0,
210+
`_0001_account_count` int (11) DEFAULT 0,
211+
`_00001_account_count` int (11) DEFAULT 0,
212+
`_000001_account_count` int (11) DEFAULT 0,
213+
`_000001_below_account_count` int (11) DEFAULT 0,
188214
`timestamp` int (11) DEFAULT NULL,
215+
`timestamp_utc` datetime DEFAULT NULL,
189216
UNIQUE KEY `timestamp` (`timestamp`)
190217
) ENGINE = InnoDB;
191218

‎scripts/rollup-daily-balance-distribution.mjs

+140
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
import debug from 'debug'
2+
import dayjs from 'dayjs'
3+
import yargs from 'yargs'
4+
import { hideBin } from 'yargs/helpers'
5+
import utc from 'dayjs/plugin/utc.js'
6+
7+
import db from '#db'
8+
import { isMain } from '#common'
9+
10+
dayjs.extend(utc)
11+
12+
const argv = yargs(hideBin(process.argv)).argv
13+
const log = debug('rollup-daily-balance-distribution')
14+
debug.enable('rollup-daily-balance-distribution')
15+
16+
const first_timestamp = '1550832660' // earliest local_timestamp in blocks table
17+
18+
const rollup_daily_balance_distribution = async ({
19+
start_date = null,
20+
days = 1,
21+
full = false,
22+
end_date = null
23+
}) => {
24+
let time = start_date
25+
? dayjs(start_date).utc().startOf('day')
26+
: dayjs().utc().startOf('day')
27+
const end = end_date
28+
? dayjs(end_date).utc().startOf('day')
29+
: full
30+
? dayjs.unix(first_timestamp)
31+
: time.subtract(days, 'day')
32+
33+
do {
34+
const given_date = time.format('YYYY-MM-DD')
35+
const balance_ranges_query = db
36+
.with(
37+
'ranked_blocks',
38+
db.raw(
39+
`
40+
SELECT
41+
account,
42+
balance,
43+
rank() OVER (PARTITION BY account ORDER BY height DESC) AS rank
44+
FROM blocks
45+
WHERE local_timestamp < EXTRACT(EPOCH FROM TIMESTAMP '${given_date}'::date)
46+
`
47+
)
48+
)
49+
.with(
50+
'latest_balances',
51+
db.raw(`
52+
SELECT account, balance
53+
FROM ranked_blocks
54+
WHERE rank = 1
55+
`)
56+
)
57+
.with(
58+
'balance_ranges',
59+
db.raw(`
60+
SELECT
61+
CASE
62+
WHEN balance = 0 THEN '_zero'
63+
WHEN balance >= 1000000000000000000000000000000000000 THEN '_1000000'
64+
WHEN balance >= 100000000000000000000000000000000000 AND balance < 1000000000000000000000000000000000000 THEN '_100000'
65+
WHEN balance >= 10000000000000000000000000000000000 AND balance < 100000000000000000000000000000000000 THEN '_10000'
66+
WHEN balance >= 10000000000000000000000000000000 AND balance < 1000000000000000000000000000000000 THEN '_1000'
67+
WHEN balance >= 1000000000000000000000000000000 AND balance < 10000000000000000000000000000000 THEN '_100'
68+
WHEN balance >= 100000000000000000000000000000 AND balance < 100000000000000000000000000000 THEN '_10'
69+
WHEN balance >= 10000000000000000000000000000 AND balance < 100000000000000000000000000000 THEN '_1'
70+
WHEN balance >= 10000000000000000000000000000 AND balance < 100000000000000000000000000000 THEN '_01'
71+
WHEN balance >= 1000000000000000000000000000 AND balance < 1000000000000000000000000000 THEN '_001'
72+
WHEN balance >= 10000000000000000000000000 AND balance < 100000000000000000000000000 THEN '_0001'
73+
WHEN balance >= 1000000000000000000000000 AND balance < 10000000000000000000000000 THEN '_00001'
74+
WHEN balance >= 100000000000000000000000 AND balance < 100000000000000000000000 THEN '_000001'
75+
ELSE '_000001_below'
76+
END AS balance_range,
77+
SUM(balance) AS total_balance_raw,
78+
COUNT(account) AS account_count
79+
FROM latest_balances
80+
GROUP BY balance_range
81+
`)
82+
)
83+
.select('balance_range', 'total_balance_raw', 'account_count')
84+
.from('balance_ranges')
85+
86+
const result = await balance_ranges_query
87+
88+
const insert_object = {
89+
timestamp: time.unix(),
90+
timestamp_utc: time.format('YYYY-MM-DD HH:mm:ss')
91+
}
92+
93+
if (result && result.length > 0) {
94+
result.forEach((row) => {
95+
insert_object[`${row.balance_range}_account_count`] = row.account_count
96+
insert_object[`${row.balance_range}_total_balance`] =
97+
row.total_balance_raw
98+
})
99+
100+
await db('rollup_daily')
101+
.insert(insert_object)
102+
.onConflict('timestamp')
103+
.merge()
104+
}
105+
106+
log(`processed ${time.format('MM/DD/YYYY')}`)
107+
108+
time = time.subtract(1, 'day')
109+
} while (time.isAfter(end))
110+
}
111+
112+
const main = async () => {
113+
let error
114+
try {
115+
await rollup_daily_balance_distribution({
116+
start_date: argv.start_date,
117+
days: argv.days,
118+
full: argv.full,
119+
end_date: argv.end_date
120+
})
121+
} catch (err) {
122+
error = err
123+
log(error)
124+
}
125+
126+
// await db('jobs').insert({
127+
// type: constants.jobs.EXAMPLE,
128+
// succ: error ? 0 : 1,
129+
// reason: error ? error.message : null,
130+
// timestamp: Math.round(Date.now() / 1000)
131+
// })
132+
133+
process.exit()
134+
}
135+
136+
if (isMain(import.meta.url)) {
137+
main()
138+
}
139+
140+
export default rollup_daily_balance_distribution

‎scripts/rollup-daily.mjs

+2-2
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ const get_daily_stats = async (time) => {
157157
receive_volume: receive_volume.toFixed(),
158158

159159
_000001_below_count: amount_bottom_range_counter,
160-
_000001_below_total: amount_bottom_range_total.toFixed(),
160+
_000001_below_total_amount_sent: amount_bottom_range_total.toFixed(),
161161

162162
...counters
163163
}
@@ -166,7 +166,7 @@ const get_daily_stats = async (time) => {
166166
for (let i = 0; i < amounts.size; i++) {
167167
const key = amounts_iterator.next().value
168168
insert[`${key}_count`] = amount_range_counters[i]
169-
insert[`${key}_total`] = amount_range_totals[i].toFixed()
169+
insert[`${key}_total_amount_sent`] = amount_range_totals[i].toFixed()
170170
}
171171

172172
return insert

0 commit comments

Comments
 (0)