diff --git a/.env b/.env index 75b985bf3c9..5c9705335c0 100644 --- a/.env +++ b/.env @@ -137,7 +137,12 @@ CHAOS_TEST_KILL_INTERVAL = 3m # Regardless, until we have verification pools for dev testnets, this value isn't used except for testing. SMS_RETRIEVER_HASH_CODE=aBQdOa/7QJh +# Below are the variables needed to access +LEADERBOARD_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-testnet/celo-monorepo" +LEADERBOARD_DOCKER_IMAGE_TAG="leaderboard-" +LEADERBOARD_SHEET="1HCs1LZv1BOB1v2bVlH4qNPnxVRlYVhQ7CKhkMibE4EY" + # Attestation Bot variables ATTESTATION_BOT_INITIAL_WAIT_SECONDS=3 ATTESTATION_BOT_IN_BETWEEN_WAIT_SECONDS=10 -ATTESTATION_BOT_MAX_ATTESTATIONS=3 \ No newline at end of file +ATTESTATION_BOT_MAX_ATTESTATIONS=3 diff --git a/README.md b/README.md index 444d04bbb3b..31b1ed474b4 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,9 @@ See [Developer's Guide](https://docs.celo.org/) for full details about the desig See the [issue backlog](https://github.com/celo-org/celo-monorepo/issues) for a list of active or proposed tasks. Feel free to create new issues to report bugs and/or request features. Please add labels to your issues, tagging the appropriate package/area. - ### ✍️ Contributing -Feel free to jump on the Celo 🚂🚋🚋🚋. Improvements and contributions are highly encouraged! 🙏👊 +Feel free to jump on the Celo 🚂🚋🚋🚋. Improvements and contributions are highly encouraged! 🙏👊 See the [contributing guide](CONTRIBUTING.md) for details on how to participate. [![GitHub issues by-label](https://img.shields.io/github/issues/celo-org/celo-monorepo/1%20hour%20tasks)](https://github.com/celo-org/celo-monorepo/issues?q=is%3Aopen+is%3Aissue+label%3A%221+hour+tasks%22) @@ -72,6 +71,7 @@ twitter intent generator - http://tech.cymi.org/tweet-intents --> ### 💬 Ask questions, find answers, and get in touch: + - [Website](https://celo.org/) - [Docs](https://docs.celo.org/) - [Blog](https://medium.com/celohq) 📖 @@ -82,7 +82,6 @@ twitter intent generator - http://tech.cymi.org/tweet-intents - [Reddit](https://www.reddit.com/r/CeloHQ/) - [Community Events](https://celo.org/community) 🎉 - ### 📜 License All packages are licensed under the terms of the Apache 2.0 License unless otherwise specified in the LICENSE file at package's root. diff --git a/cloudbuild.yaml b/cloudbuild.yaml index dbe3613ba80..2278cfdeab3 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -41,4 +41,13 @@ steps: ] waitFor: ['-'] + - id: "docker:leaderboard" + name: gcr.io/kaniko-project/executor:latest + args: [ + "--dockerfile=dockerfiles/leaderboard/Dockerfile", + "--cache=true", + "--destination=gcr.io/$PROJECT_ID/celo-monorepo:leaderboard-$COMMIT_SHA" + ] + waitFor: ['-'] + timeout: 3000s diff --git a/dockerfiles/leaderboard/Dockerfile b/dockerfiles/leaderboard/Dockerfile new file mode 100644 index 00000000000..b081cca2988 --- /dev/null +++ b/dockerfiles/leaderboard/Dockerfile @@ -0,0 +1,31 @@ +FROM node:10 +WORKDIR /celo-monorepo + +# ensure yarn.lock is evaluated by kaniko cache diff +COPY lerna.json package.json yarn.lock ./ +COPY scripts/ scripts/ + +# Copy only pkg.json +COPY packages/typescript/package.json packages/typescript/ +COPY packages/utils/package.json packages/utils/ +COPY packages/dev-utils/package.json packages/dev-utils/ +COPY packages/protocol/package.json packages/protocol/ +COPY packages/contractkit/package.json packages/contractkit/ +COPY packages/leaderboard/package.json packages/leaderboard/ + +RUN yarn install --frozen-lockfile --network-timeout 100000 && yarn cache clean + +# Copy the rest +COPY packages/typescript packages/typescript/ +COPY packages/utils packages/utils/ +COPY packages/dev-utils packages/dev-utils/ +COPY packages/protocol packages/protocol/ +COPY packages/contractkit packages/contractkit/ +COPY packages/leaderboard packages/leaderboard/ + +# build all +RUN yarn build + +WORKDIR /celo-monorepo/packages/leaderboard + +CMD ["yarn run ts-node src/board.ts"] diff --git a/packages/celotool/src/cmds/deploy/destroy/leaderboard.ts b/packages/celotool/src/cmds/deploy/destroy/leaderboard.ts new file mode 100644 index 00000000000..2b9aba5b7e1 --- /dev/null +++ b/packages/celotool/src/cmds/deploy/destroy/leaderboard.ts @@ -0,0 +1,16 @@ +import { createClusterIfNotExists, switchToClusterFromEnv } from 'src/lib/cluster' +import { removeHelmRelease } from 'src/lib/leaderboard' +import { DestroyArgv } from '../../deploy/destroy' + +export const command = 'leaderboard' + +export const describe = 'destroy the leaderboard package' + +export const builder = {} + +export const handler = async (argv: DestroyArgv) => { + await createClusterIfNotExists() + await switchToClusterFromEnv() + + await removeHelmRelease(argv.celoEnv) +} diff --git a/packages/celotool/src/cmds/deploy/initial/leaderboard.ts b/packages/celotool/src/cmds/deploy/initial/leaderboard.ts new file mode 100644 index 00000000000..ffdcb0964c3 --- /dev/null +++ b/packages/celotool/src/cmds/deploy/initial/leaderboard.ts @@ -0,0 +1,17 @@ +import { InitialArgv } from 'src/cmds/deploy/initial' +import { switchToClusterFromEnv } from 'src/lib/cluster' +import { installHelmChart } from 'src/lib/leaderboard' +import yargs from 'yargs' + +export const command = 'leaderboard' + +export const describe = 'deploy the leaderboard for the specified network' + +export const builder = (argv: yargs.Argv) => { + return argv +} + +export const handler = async (argv: InitialArgv) => { + await switchToClusterFromEnv() + await installHelmChart(argv.celoEnv) +} diff --git a/packages/celotool/src/cmds/deploy/upgrade/leaderboard.ts b/packages/celotool/src/cmds/deploy/upgrade/leaderboard.ts new file mode 100644 index 00000000000..059eea67fd0 --- /dev/null +++ b/packages/celotool/src/cmds/deploy/upgrade/leaderboard.ts @@ -0,0 +1,32 @@ +import { UpgradeArgv } from 'src/cmds/deploy/upgrade' +import { createClusterIfNotExists, switchToClusterFromEnv } from 'src/lib/cluster' +import { installHelmChart, removeHelmRelease, upgradeHelmChart } from 'src/lib/leaderboard' +import yargs from 'yargs' + +export const command = 'leaderboard' + +export const describe = 'upgrade the leaderboard package' + +type LeaderboardArgv = UpgradeArgv & { + reset: boolean +} + +export const builder = (argv: yargs.Argv) => { + return argv.option('reset', { + description: 'Destroy & redeploy the leaderboard package', + default: false, + type: 'boolean', + }) +} + +export const handler = async (argv: LeaderboardArgv) => { + await createClusterIfNotExists() + await switchToClusterFromEnv() + + if (argv.reset) { + await removeHelmRelease(argv.celoEnv) + await installHelmChart(argv.celoEnv) + } else { + await upgradeHelmChart(argv.celoEnv) + } +} diff --git a/packages/celotool/src/lib/env-utils.ts b/packages/celotool/src/lib/env-utils.ts index 1c8ffa24b6f..10d9dbb81f8 100644 --- a/packages/celotool/src/lib/env-utils.ts +++ b/packages/celotool/src/lib/env-utils.ts @@ -61,6 +61,9 @@ export enum envVar { IN_MEMORY_DISCOVERY_TABLE = 'IN_MEMORY_DISCOVERY_TABLE', KUBERNETES_CLUSTER_NAME = 'KUBERNETES_CLUSTER_NAME', KUBERNETES_CLUSTER_ZONE = 'KUBERNETES_CLUSTER_ZONE', + LEADERBOARD_DOCKER_IMAGE_REPOSITORY = 'LEADERBOARD_DOCKER_IMAGE_REPOSITORY', + LEADERBOARD_DOCKER_IMAGE_TAG = 'LEADERBOARD_DOCKER_IMAGE_TAG', + LEADERBOARD_SHEET = 'LEADERBOARD_SHEET', LOAD_TEST_CLIENTS = 'LOAD_TEST_CLIENTS', LOAD_TEST_GENESIS_BALANCE = 'LOAD_TEST_GENESIS_BALANCE', LOAD_TEST_TX_DELAY_MS = 'LOAD_TEST_TX_DELAY_MS', diff --git a/packages/celotool/src/lib/leaderboard.ts b/packages/celotool/src/lib/leaderboard.ts new file mode 100644 index 00000000000..43998aa0794 --- /dev/null +++ b/packages/celotool/src/lib/leaderboard.ts @@ -0,0 +1,56 @@ +import yaml from 'js-yaml' +import { envVar, fetchEnv } from 'src/lib/env-utils' +import { installGenericHelmChart, removeGenericHelmChart } from 'src/lib/helm_deploy' +import { execCmd, execCmdWithExitOnFailure } from 'src/lib/utils' + +const helmChartPath = '../helm-charts/leaderboard' + +export async function installHelmChart(celoEnv: string) { + return installGenericHelmChart( + celoEnv, + releaseName(celoEnv), + helmChartPath, + await helmParameters(celoEnv) + ) +} + +export async function removeHelmRelease(celoEnv: string) { + await removeGenericHelmChart(releaseName(celoEnv)) +} + +export async function upgradeHelmChart(celoEnv: string) { + console.info(`Upgrading helm release ${releaseName(celoEnv)}`) + + const params = (await helmParameters(celoEnv)).join(' ') + + const upgradeCmdArgs = `${releaseName(celoEnv)} ${helmChartPath} --namespace ${celoEnv} ${params}` + + if (process.env.CELOTOOL_VERBOSE === 'true') { + await execCmdWithExitOnFailure(`helm upgrade --debug --dry-run ${upgradeCmdArgs}`) + } + await execCmdWithExitOnFailure(`helm upgrade ${upgradeCmdArgs}`) + console.info(`Helm release ${releaseName(celoEnv)} upgrade successful`) +} + +export async function helmParameters(celoEnv: string) { + const dbValues = await getBlockscoutHelmValues(celoEnv) + return [ + `--set leaderboard.db.connection_name=${dbValues.connection_name}`, + `--set leaderboard.db.username=${dbValues.username}`, + `--set leaderboard.db.password=${dbValues.password}`, + `--set leaderboard.image.repository=${fetchEnv(envVar.LEADERBOARD_DOCKER_IMAGE_REPOSITORY)}`, + `--set leaderboard.image.tag=${fetchEnv(envVar.LEADERBOARD_DOCKER_IMAGE_TAG)}`, + `--set leaderboard.sheet=${fetchEnv(envVar.LEADERBOARD_SHEET)}`, + `--set leaderboard.web3=https://${celoEnv}-forno.${fetchEnv(envVar.CLUSTER_DOMAIN_NAME)}.org`, + ] +} + +function releaseName(celoEnv: string) { + return `${celoEnv}-leaderboard` +} + +export async function getBlockscoutHelmValues(celoEnv: string) { + const [output] = await execCmd(`helm get values ${celoEnv}-blockscout`) + const blockscoutValues: any = yaml.safeLoad(output) + return blockscoutValues.blockscout.db +} diff --git a/packages/contractkit/src/identity/claims/verify.ts b/packages/contractkit/src/identity/claims/verify.ts index 83d61396aae..878c96be638 100644 --- a/packages/contractkit/src/identity/claims/verify.ts +++ b/packages/contractkit/src/identity/claims/verify.ts @@ -59,7 +59,7 @@ export const verifyAccountClaim = async ( const accountClaims = metadata.filterClaims(ClaimTypes.ACCOUNT) - if (accountClaims.find((x) => x.address === address) === undefined) { + if (accountClaims.find((x) => x.address.toLowerCase() === address.toLowerCase()) === undefined) { return `${claim.address} did not claim ${address}` } diff --git a/packages/docs/getting-started/running-a-validator.md b/packages/docs/getting-started/running-a-validator.md index 0d45fc29a3d..dece94d9fb1 100644 --- a/packages/docs/getting-started/running-a-validator.md +++ b/packages/docs/getting-started/running-a-validator.md @@ -274,7 +274,7 @@ The `mine` flag does not mean the node starts mining blocks, but rather starts t The `networkid` parameter value of `12219` indicates we are connecting to the Baklava network, Stake Off Phase 1. -Note that if you are running the validator and the proxy on the same machine, then you should set the validator's listening port to something other than `30303`. E.g. you could use the flag `--port 30313` and set the docker port forwarding rules accordingly (e.g. use the flags `-p 30313:30313` and `-p 30313:30313/udp`). +Note that if you are running the validator and the proxy on the same machine, then you should set the validator's listening port to something other than `30303`. E.g. you could use the flag `--port 30313` and set the docker port forwarding rules accordingly (e.g. use the flags `-p 30313:30313` and `-p 30313:30313/udp`). ### Register the Accounts diff --git a/packages/helm-charts/leaderboard/.helmignore b/packages/helm-charts/leaderboard/.helmignore new file mode 100644 index 00000000000..50af0317254 --- /dev/null +++ b/packages/helm-charts/leaderboard/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packages/helm-charts/leaderboard/Chart.yaml b/packages/helm-charts/leaderboard/Chart.yaml new file mode 100644 index 00000000000..b605f97dc66 --- /dev/null +++ b/packages/helm-charts/leaderboard/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: leaderboard +version: 0.1.0 diff --git a/packages/helm-charts/leaderboard/templates/leaderboard.cronjob.yaml b/packages/helm-charts/leaderboard/templates/leaderboard.cronjob.yaml new file mode 100644 index 00000000000..2b2ebb4f443 --- /dev/null +++ b/packages/helm-charts/leaderboard/templates/leaderboard.cronjob.yaml @@ -0,0 +1,80 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ .Release.Name }} + labels: + app: leaderboard + chart: leaderboard + release: {{ .Release.Service }} + component: leaderboard +spec: + schedule: "{{ .Values.leaderboard.schedule }}" + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + spec: + containers: + - name: cloudsql-proxy + image: gcr.io/cloudsql-docker/gce-proxy:1.11 + command: + - /bin/sh + args: + - -c + - | + /cloud_sql_proxy \ + -instances={{ .Values.leaderboard.db.connection_name }}=tcp:5432 \ + -credential_file=/secrets/cloudsql/credentials.json & + CHILD_PID=$! + (while true; do if [[ -f "/tmp/pod/main-terminated" ]]; then kill $CHILD_PID; fi; sleep 1; done) & + wait $CHILD_PID + if [[ -f "/tmp/pod/main-terminated" ]]; then exit 0; fi + securityContext: + runAsUser: 2 # non-root user + allowPrivilegeEscalation: false + volumeMounts: + - name: blockscout-cloudsql-credentials + mountPath: /secrets/cloudsql + readOnly: true + - mountPath: /tmp/pod + name: tmp-pod + readOnly: true + - name: update-sheet + image: {{ .Values.leaderboard.image.repository }}:{{ .Values.leaderboard.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + args: + - -c + - | + trap "touch /tmp/pod/main-terminated" EXIT + yarn run ts-node src/board.ts + env: + - name: LEADERBOARD_DATABASE + value: {{ .Values.leaderboard.database }} + - name: LEADERBOARD_WEB3 + value: {{ .Values.leaderboard.web3 }} + - name: LEADERBOARD_SHEET + value: {{ .Values.leaderboard.sheet }} + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ .Release.Namespace }}-leaderboard + key: DATABASE_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ .Release.Namespace }}-leaderboard + key: DATABASE_PASSWORD + volumeMounts: + - mountPath: /tmp/pod + name: tmp-pod + restartPolicy: Never + volumes: + - name: blockscout-cloudsql-credentials + secret: + defaultMode: 420 + secretName: blockscout-cloudsql-credentials + - name: tmp-pod + emptyDir: {} diff --git a/packages/helm-charts/leaderboard/templates/leaderboard.secret.yaml b/packages/helm-charts/leaderboard/templates/leaderboard.secret.yaml new file mode 100644 index 00000000000..29ec97390bd --- /dev/null +++ b/packages/helm-charts/leaderboard/templates/leaderboard.secret.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Namespace }}-leaderboard + labels: + app: leaderboard + chart: leaderboard + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +data: + DATABASE_USER: {{ .Values.leaderboard.db.username | b64enc | quote }} + DATABASE_PASSWORD: {{ .Values.leaderboard.db.password | b64enc | quote }} diff --git a/packages/helm-charts/leaderboard/values.yaml b/packages/helm-charts/leaderboard/values.yaml new file mode 100644 index 00000000000..0d9d2115224 --- /dev/null +++ b/packages/helm-charts/leaderboard/values.yaml @@ -0,0 +1,16 @@ +nodeSelector: {} + +# Expects cron schedule syntax +leaderboard: + schedule: "*/1 * * * *" + database: "blockscout" + image: + repository: jcortejoso/leaderboard + tag: latest + database: blockscout + web3: https://baklavastaging-forno.celo-testnet.org/ + sheet: 1HCs1LZv1BOB1v2bVlH4qNPnxVRlYVhQ7CKhkMibE4EY + db: + username: ls880fd3 + password: dg7mTjVB2khgkDFd + connection_name: celo-testnet:us-west1:baklavastaging0 diff --git a/packages/leaderboard/.gitignore b/packages/leaderboard/.gitignore new file mode 100644 index 00000000000..56f90842899 --- /dev/null +++ b/packages/leaderboard/.gitignore @@ -0,0 +1,2 @@ +credentials.json +token.json diff --git a/packages/leaderboard/README.md b/packages/leaderboard/README.md new file mode 100644 index 00000000000..69f6b4c4417 --- /dev/null +++ b/packages/leaderboard/README.md @@ -0,0 +1,56 @@ +## What does it do? + +Reads from Google sheet (template https://docs.google.com/spreadsheets/d/1Me56YkCHYmsN23gSMgDb1hZ_ezN0sTjNW4kyGbAO9vc/edit#gid=0) + +Reads and verifies account claims. + +Reads the exchange rate into DB. + +This script reads data directly into the DB that Blockscout uses, so it needs to be ran on the same machine as Blockscout. + +## Environment variables + +For Postgres: `PG_USER` and `PG_PASSWORD` + +Other variables: + +- `LEADERBOARD_DATABASE`: which database to connect into. Defaults to `blockscout` +- `LEADERBOARD_WEB3`: Geth RPC address. Default: `http://localhost:8545` +- `LEADERBOARD_SHEET`: Google spreadsheet identifier. Default: `1HCs1LZv1BOB1v2bVlH4qNPnxVRlYVhQ7CKhkMibE4EY` + +## Setup and running + +Make sure Postgres is up. + +Setting up env variables: + +``` +export PGUSER=postgres +export PGPASSWORD=1234 +``` + +Make sure geth is running. + +Make sure that public can access the spreadsheet: File/Publish to the Web + +``` +ts-node src/board.ts +``` + +## Account claims + +Using ganache and local web server + +``` +export NO_SYNCCHECK=true +export ACCOUNT1=0x5409ED021D9299bf6814279A6A1411A7e866A631 +export ACCOUNT2=0x6Ecbe1DB9EF729CBe972C83Fb886247691Fb6beb +yarn run celocli account:register --name "account 1" --from $ACCOUNT1 +yarn run celocli account:register --name "account 2" --from $ACCOUNT2 +yarn run celocli account:create-metadata --from $ACCOUNT1 /var/www/html/samples/metadata2.json +yarn run celocli account:create-metadata --from $ACCOUNT2 /var/www/html/samples/metadata1.json +yarn run celocli account:register-metadata --from $ACCOUNT1 --url http://localhost/samples/metadata2.json +yarn run celocli account:register-metadata --from $ACCOUNT2 --url http://localhost/samples/metadata1.json +yarn run celocli account:claim-account /var/www/html/samples/metadata2.json --address=$ACCOUNT2 --from=$ACCOUNT1 +yarn run celocli account:claim-account /var/www/html/samples/metadata1.json --address=$ACCOUNT1 --from=$ACCOUNT2 +``` diff --git a/packages/leaderboard/package.json b/packages/leaderboard/package.json new file mode 100644 index 00000000000..f7dec3d9418 --- /dev/null +++ b/packages/leaderboard/package.json @@ -0,0 +1,20 @@ +{ + "name": "@celo/leaderboard", + "version": "1.0.0", + "description": "Baklava leaderboard script", + "main": "index.js", + "author": "Celo", + "license": "Apache-2.0", + "dependencies": { + "@celo/contractkit": "^0.2.8-dev", + "@types/pg": "^7.11.2", + "google-spreadsheet": "^2.0.8", + "pg": "^7.12.1", + "ts-node": "^8.5.4" + }, + "devDependencies": {}, + "scripts": { + "build": "tsc -b ." + }, + "private": true +} diff --git a/packages/leaderboard/src/board.ts b/packages/leaderboard/src/board.ts new file mode 100644 index 00000000000..74c1d9d1070 --- /dev/null +++ b/packages/leaderboard/src/board.ts @@ -0,0 +1,164 @@ +import { ContractKit, newKitFromWeb3 } from '@celo/contractkit' +import { AccountsWrapper } from '@celo/contractkit/lib/wrappers/Accounts' +import Web3 from 'web3' +import { Client } from 'pg' +import { ClaimTypes, IdentityMetadataWrapper } from '@celo/contractkit/lib/identity' +import { verifyAccountClaim } from '@celo/contractkit/lib/identity/claims/verify' + +const GoogleSpreadsheet = require('google-spreadsheet') + +function addressToBinary(a: string) { + try { + if (a.substr(0, 2) == '0x') return a.substr(2) + else return a + } catch (_err) { + return a + } +} + +process.on('unhandledRejection', (reason, _promise) => { + console.log('Unhandled Rejection at:', reason.stack || reason) + process.exit(0) +}) + +const LEADERBOARD_DATABASE = process.env['LEADERBOARD_DATABASE'] || 'blockscout' +const LEADERBOARD_SHEET = + process.env['LEADERBOARD_SHEET'] || '1HCs1LZv1BOB1v2bVlH4qNPnxVRlYVhQ7CKhkMibE4EY' +const LEADERBOARD_WEB3 = process.env['LEADERBOARD_WEB3'] || 'http://localhost:8545' + +function readSheet() { + // spreadsheet key is the long id in the sheets URL + const doc = new GoogleSpreadsheet(LEADERBOARD_SHEET) + + doc.getInfo(function(_err: any, info: any) { + let sheet = info.worksheets[0] + sheet.getCells( + { + 'min-row': 3, + 'max-row': 500, + 'min-col': 1, + 'max-col': 3, + 'return-empty': true, + }, + function(err: any, cells: any) { + console.log(err) + let arr: any = {} + for (let e of cells) { + // console.log(e) + arr[e.row] = arr[e.row] || {} + if (e.col == 1) { + arr[e.row].address = addressToBinary(e.value) + } + if (e.col == 3) { + arr[e.row].multiplier = e.numericValue + } + } + let lst = Object.values(arr) + updateDB( + lst.filter((a: any) => !!a.address && a.multiplier !== 0), + lst.filter((a: any) => !!a.address && a.multiplier === 0) + ) + } + ) + }) +} + +async function updateDB(lst: any[], remove: any[]) { + console.log('Adding', lst) + const client = new Client({ database: LEADERBOARD_DATABASE }) + await client.connect() + await client.query( + 'INSERT INTO competitors (address, multiplier)' + + " SELECT decode(m.address, 'hex') AS address, m.multiplier FROM json_populate_recordset(null::json_type, $1) AS m" + + ' ON CONFLICT (address) DO UPDATE SET multiplier = EXCLUDED.multiplier RETURNING *', + [JSON.stringify(lst)] + ) + console.log('Removing', remove) + for (let elem of remove) { + await client.query( + "DELETE FROM competitors WHERE address = '\\x" + elem.address.toString() + "'" + ) + } + await client.end() + await readAssoc(lst.map((a: any) => a.address.toString())) +} + +function dedup(lst: string[]): string[] { + return [...new Set(lst)] +} + +async function getClaims( + kit: ContractKit, + address: string, + data: IdentityMetadataWrapper +): Promise { + if (address.substr(0, 2) === '0x') { + address = address.substr(2) + } + const res = [address] + const accounts = await kit.contracts.getAccounts() + for (const claim of data.claims) { + switch (claim.type) { + case ClaimTypes.KEYBASE: + break + case ClaimTypes.ACCOUNT: + try { + const status = await verifyAccountClaim(claim, '0x' + address, accounts.getMetadataURL) + if (status) console.error('Cannot verify claim:', status) + else { + console.log('Claim success', address, claim.address) + res.push(claim.address) + } + } catch (err) { + console.error('Cannot fetch metadata', err) + } + default: + break + } + } + return dedup(res) +} + +async function processClaims(kit: ContractKit, address: string, info: IdentityMetadataWrapper) { + try { + const lst: string[] = await getClaims(kit, address, info) + const client = new Client({ database: LEADERBOARD_DATABASE }) + await client.connect() + await client.query( + 'INSERT INTO claims (address, claimed_address)' + + " SELECT decode(m.address,'hex'), decode(m.claimed_address,'hex') FROM json_populate_recordset(null::json_assoc, $1) AS m" + + ' ON CONFLICT (address, claimed_address) DO NOTHING RETURNING *', + [ + JSON.stringify( + lst.map((a) => { + const res = { address: addressToBinary(address), claimed_address: addressToBinary(a) } + return res + }) + ), + ] + ) + await client.end() + } catch (err) { + console.error('Cannot process claims', err) + } +} + +async function readAssoc(lst: string[]) { + const web3 = new Web3(LEADERBOARD_WEB3) + const kit: ContractKit = newKitFromWeb3(web3) + const accounts: AccountsWrapper = await kit.contracts.getAccounts() + lst.forEach(async (a) => { + try { + const url = await accounts.getMetadataURL(a) + console.log(a, 'has url', url) + let metadata: IdentityMetadataWrapper + if (url == '') metadata = IdentityMetadataWrapper.fromEmpty(a) + else metadata = await IdentityMetadataWrapper.fetchFromURL(url) + processClaims(kit, a, metadata) + } catch (err) { + console.error('Bad address', a, err.toString()) + } + }) +} + +readSheet() diff --git a/packages/leaderboard/tsconfig.json b/packages/leaderboard/tsconfig.json new file mode 100644 index 00000000000..f1e65a88714 --- /dev/null +++ b/packages/leaderboard/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../typescript/tsconfig.library.json", + "compilerOptions": { + "outDir": "lib", + "rootDir": "src", + "baseUrl": ".", + "lib": ["es7", "es2017"], + "target": "es6", + "resolveJsonModule": true, + "noEmit": true, + "esModuleInterop": true, + "paths": { + "@google-cloud/monitoring": ["types/monitoring"] + } + }, + "include": ["src", "../contractkit/types"], + "exclude": ["node_modules/"], + "references": [{ "path": "../utils" }, { "path": "../contractkit" }] +} diff --git a/yarn.lock b/yarn.lock index de1078b6e3f..0f03ead51e7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5285,6 +5285,19 @@ resolved "https://registry.yarnpkg.com/@types/p-defer/-/p-defer-1.0.3.tgz#786ce79c86f779fcd9e9bec4f1fbd1167aeac064" integrity sha512-0CK39nXek0mSZL/lnGYjhcR1QLAxg9N0/5S1BvU+MQwjlP4Jd2ebbEkJ/bEUqYMAvKLMZcGd4sJE13dnUKlDnQ== +"@types/pg-types@*": + version "1.11.5" + resolved "https://registry.yarnpkg.com/@types/pg-types/-/pg-types-1.11.5.tgz#1eebbe62b6772fcc75c18957a90f933d155e005b" + integrity sha512-L8ogeT6vDzT1vxlW3KITTCt+BVXXVkLXfZ/XNm6UqbcJgxf+KPO7yjWx7dQQE8RW07KopL10x2gNMs41+IkMGQ== + +"@types/pg@^7.11.2": + version "7.11.2" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-7.11.2.tgz#199dec09426c9359574dedede37313805ba3fca2" + integrity sha512-4+rj7fnidA77jFURNanuPPc1HrQv+RkhI6s+K18G9zOKbOUUpChA/rbNMqFukNuZ89LoIt/I9dAlxf329TjCNw== + dependencies: + "@types/node" "*" + "@types/pg-types" "*" + "@types/prettier@1.15.2": version "1.15.2" resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-1.15.2.tgz#91594ea7cb6f3b1f7ea69f32621246654c7cc231" @@ -16590,6 +16603,17 @@ google-proto-files@^0.20.0: protobufjs "^6.8.0" walkdir "^0.3.0" +google-spreadsheet@^2.0.8: + version "2.0.8" + resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-2.0.8.tgz#4c52c13bec439010010d9123ebc3cd06f43a51c0" + integrity sha512-mOcYls5JlJN/nVcqVLrdvQ9XV6Z2OnAPuh8t8FhirAYYVCXv3QuLple8kknQ6gO+8i88T+rxrpJZ8PLjx6BvuQ== + dependencies: + async "^1.3.0" + google-auth-library "^0.10.0" + lodash "^4.17.15" + request "^2.69.0" + xml2js "~0.4.0" + got@7.1.0, got@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/got/-/got-7.1.0.tgz#05450fd84094e6bbea56f451a43a9c289166385a" @@ -25045,7 +25069,7 @@ pg-int8@1.0.1: resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== -pg-pool@^2.0.4: +pg-pool@^2.0.4, pg-pool@^2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-2.0.7.tgz#f14ecab83507941062c313df23f6adcd9fd0ce54" integrity sha512-UiJyO5B9zZpu32GSlP0tXy8J2NsJ9EFGFfz5v6PSbdz/1hBLX1rNiiy5+mAm5iJJYwfCv4A0EBcQLGWwjbpzZw== @@ -25074,6 +25098,19 @@ pg@7.12.1: pgpass "1.x" semver "4.3.2" +pg@^7.12.1: + version "7.14.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-7.14.0.tgz#f46727845ad19c2670a7e8151063a670338b6057" + integrity sha512-TLsdOWKFu44vHdejml4Uoo8h0EwCjdIj9Z9kpz7pA5i8iQxOTwVb1+Fy+X86kW5AXKxQpYpYDs4j/qPDbro/lg== + dependencies: + buffer-writer "2.0.0" + packet-reader "1.0.0" + pg-connection-string "0.1.3" + pg-pool "^2.0.7" + pg-types "^2.1.0" + pgpass "1.x" + semver "4.3.2" + pgpass@1.x: version "1.0.2" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" @@ -27867,7 +27904,7 @@ request-promise@^4.2.2: stealthy-require "^1.1.1" tough-cookie "^2.3.3" -request@2.88.0, request@^2.45.0, request@^2.67.0, request@^2.85.0, request@^2.87.0, request@^2.88.0: +request@2.88.0, request@^2.45.0, request@^2.67.0, request@^2.69.0, request@^2.85.0, request@^2.87.0, request@^2.88.0: version "2.88.0" resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg== @@ -31740,6 +31777,17 @@ ts-node@^8.4.1: source-map-support "^0.5.6" yn "^3.0.0" +ts-node@^8.5.4: + version "8.5.4" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-8.5.4.tgz#a152add11fa19c221d0b48962c210cf467262ab2" + integrity sha512-izbVCRV68EasEPQ8MSIGBNK9dc/4sYJJKYA+IarMQct1RtEot6Xp0bXuClsbUSnKpg50ho+aOAx8en5c+y4OFw== + dependencies: + arg "^4.1.0" + diff "^4.0.1" + make-error "^1.1.1" + source-map-support "^0.5.6" + yn "^3.0.0" + tsc-watch@^1.0.31: version "1.0.31" resolved "https://registry.yarnpkg.com/tsc-watch/-/tsc-watch-1.0.31.tgz#639fc7866a86abcdd3cf0836d5f1cd1d64dcd5e0" @@ -32644,7 +32692,7 @@ util-promisify@^2.1.0: dependencies: object.getownpropertydescriptors "^2.0.3" -util.promisify@^1.0.0: +util.promisify@^1.0.0, util.promisify@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/util.promisify/-/util.promisify-1.0.0.tgz#440f7165a459c9a16dc145eb8e72f35687097030" integrity sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA== @@ -34677,6 +34725,15 @@ xml2js@0.4.19, xml2js@^0.4.17: sax ">=0.6.0" xmlbuilder "~9.0.1" +xml2js@~0.4.0: + version "0.4.22" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.22.tgz#4fa2d846ec803237de86f30aa9b5f70b6600de02" + integrity sha512-MWTbxAQqclRSTnehWWe5nMKzI3VmJ8ltiJEco8akcC6j3miOhjjfzKum5sId+CWhfxdOs/1xauYr8/ZDBtQiRw== + dependencies: + sax ">=0.6.0" + util.promisify "~1.0.0" + xmlbuilder "~11.0.0" + xml@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" @@ -34692,6 +34749,11 @@ xmlbuilder@^9.0.7, xmlbuilder@~9.0.1: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= +xmlbuilder@~11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" + integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + xmldoc@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/xmldoc/-/xmldoc-1.1.2.tgz#6666e029fe25470d599cd30e23ff0d1ed50466d7"