Script for reading leaderboard data into database

.env

@@ -137,7 +137,12 @@ CHAOS_TEST_KILL_INTERVAL = 3m
 # Regardless, until we have verification pools for dev testnets, this value isn't used except for testing.
 SMS_RETRIEVER_HASH_CODE=aBQdOa/7QJh
 
+# Below are the variables needed to access
+LEADERBOARD_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-testnet/celo-monorepo"
+LEADERBOARD_DOCKER_IMAGE_TAG="leaderboard-"
+LEADERBOARD_SHEET="1HCs1LZv1BOB1v2bVlH4qNPnxVRlYVhQ7CKhkMibE4EY"
+
 # Attestation Bot variables
 ATTESTATION_BOT_INITIAL_WAIT_SECONDS=3
 ATTESTATION_BOT_IN_BETWEEN_WAIT_SECONDS=10
-ATTESTATION_BOT_MAX_ATTESTATIONS=3
+ATTESTATION_BOT_MAX_ATTESTATIONS=3

README.md

@@ -56,10 +56,9 @@ See [Developer's Guide](https://docs.celo.org/) for full details about the desig
 
 See the [issue backlog](https://github.com/celo-org/celo-monorepo/issues) for a list of active or proposed tasks. Feel free to create new issues to report bugs and/or request features. Please add labels to your issues, tagging the appropriate package/area.
 
-
 ### ✍️ Contributing
 
-Feel free to jump on the Celo 🚂🚋🚋🚋.  Improvements and contributions are highly encouraged! 🙏👊
+Feel free to jump on the Celo 🚂🚋🚋🚋. Improvements and contributions are highly encouraged! 🙏👊
 
 See the [contributing guide](CONTRIBUTING.md) for details on how to participate.
 [![GitHub issues by-label](https://img.shields.io/github/issues/celo-org/celo-monorepo/1%20hour%20tasks)](https://github.com/celo-org/celo-monorepo/issues?q=is%3Aopen+is%3Aissue+label%3A%221+hour+tasks%22)
@@ -72,6 +71,7 @@ twitter intent generator - http://tech.cymi.org/tweet-intents
 -->
 
 ### 💬 Ask questions, find answers, and get in touch:
+
 - [Website](https://celo.org/)
 - [Docs](https://docs.celo.org/)
 - [Blog](https://medium.com/celohq) 📖
@@ -82,7 +82,6 @@ twitter intent generator - http://tech.cymi.org/tweet-intents
 - [Reddit](https://www.reddit.com/r/CeloHQ/)
 - [Community Events](https://celo.org/community) 🎉
 
-
 ### 📜 License
 
 All packages are licensed under the terms of the Apache 2.0 License unless otherwise specified in the LICENSE file at package's root.

cloudbuild.yaml

@@ -41,4 +41,13 @@ steps:
   ]
   waitFor: ['-']
 
+  - id: "docker:leaderboard"
+  name: gcr.io/kaniko-project/executor:latest
+  args: [
+    "--dockerfile=dockerfiles/leaderboard/Dockerfile",
+    "--cache=true",
+    "--destination=gcr.io/$PROJECT_ID/celo-monorepo:leaderboard-$COMMIT_SHA"
+  ]
+  waitFor: ['-']
+
 timeout: 3000s

dockerfiles/leaderboard/Dockerfile

@@ -0,0 +1,31 @@
+FROM node:10
+WORKDIR /celo-monorepo
+
+# ensure yarn.lock is evaluated by kaniko cache diff
+COPY lerna.json package.json yarn.lock ./
+COPY scripts/ scripts/
+
+# Copy only pkg.json
+COPY packages/typescript/package.json packages/typescript/
+COPY packages/utils/package.json packages/utils/
+COPY packages/dev-utils/package.json packages/dev-utils/
+COPY packages/protocol/package.json packages/protocol/
+COPY packages/contractkit/package.json packages/contractkit/
+COPY packages/leaderboard/package.json packages/leaderboard/
+
+RUN yarn install --frozen-lockfile --network-timeout 100000 && yarn cache clean
+
+# Copy the rest
+COPY packages/typescript packages/typescript/
+COPY packages/utils packages/utils/
+COPY packages/dev-utils packages/dev-utils/
+COPY packages/protocol packages/protocol/
+COPY packages/contractkit packages/contractkit/
+COPY packages/leaderboard packages/leaderboard/
+
+# build all
+RUN yarn build
+
+WORKDIR /celo-monorepo/packages/leaderboard
+
+CMD ["yarn run ts-node src/board.ts"]

packages/celotool/src/cmds/deploy/destroy/leaderboard.ts

@@ -0,0 +1,16 @@
+import { createClusterIfNotExists, switchToClusterFromEnv } from 'src/lib/cluster'
+import { removeHelmRelease } from 'src/lib/leaderboard'
+import { DestroyArgv } from '../../deploy/destroy'
+
+export const command = 'leaderboard'
+
+export const describe = 'destroy the leaderboard package'
+
+export const builder = {}
+
+export const handler = async (argv: DestroyArgv) => {
+  await createClusterIfNotExists()
+  await switchToClusterFromEnv()
+
+  await removeHelmRelease(argv.celoEnv)
+}

packages/celotool/src/cmds/deploy/initial/leaderboard.ts

@@ -0,0 +1,17 @@
+import { InitialArgv } from 'src/cmds/deploy/initial'
+import { switchToClusterFromEnv } from 'src/lib/cluster'
+import { installHelmChart } from 'src/lib/leaderboard'
+import yargs from 'yargs'
+
+export const command = 'leaderboard'
+
+export const describe = 'deploy the leaderboard for the specified network'
+
+export const builder = (argv: yargs.Argv) => {
+  return argv
+}
+
+export const handler = async (argv: InitialArgv) => {
+  await switchToClusterFromEnv()
+  await installHelmChart(argv.celoEnv)
+}

packages/celotool/src/cmds/deploy/upgrade/leaderboard.ts

@@ -0,0 +1,32 @@
+import { UpgradeArgv } from 'src/cmds/deploy/upgrade'
+import { createClusterIfNotExists, switchToClusterFromEnv } from 'src/lib/cluster'
+import { installHelmChart, removeHelmRelease, upgradeHelmChart } from 'src/lib/leaderboard'
+import yargs from 'yargs'
+
+export const command = 'leaderboard'
+
+export const describe = 'upgrade the leaderboard package'
+
+type LeaderboardArgv = UpgradeArgv & {
+  reset: boolean
+}
+
+export const builder = (argv: yargs.Argv) => {
+  return argv.option('reset', {
+    description: 'Destroy & redeploy the leaderboard package',
+    default: false,
+    type: 'boolean',
+  })
+}
+
+export const handler = async (argv: LeaderboardArgv) => {
+  await createClusterIfNotExists()
+  await switchToClusterFromEnv()
+
+  if (argv.reset) {
+    await removeHelmRelease(argv.celoEnv)
+    await installHelmChart(argv.celoEnv)
+  } else {
+    await upgradeHelmChart(argv.celoEnv)
+  }
+}

packages/celotool/src/lib/env-utils.ts

@@ -61,6 +61,9 @@ export enum envVar {
   IN_MEMORY_DISCOVERY_TABLE = 'IN_MEMORY_DISCOVERY_TABLE',
   KUBERNETES_CLUSTER_NAME = 'KUBERNETES_CLUSTER_NAME',
   KUBERNETES_CLUSTER_ZONE = 'KUBERNETES_CLUSTER_ZONE',
+  LEADERBOARD_DOCKER_IMAGE_REPOSITORY = 'LEADERBOARD_DOCKER_IMAGE_REPOSITORY',
+  LEADERBOARD_DOCKER_IMAGE_TAG = 'LEADERBOARD_DOCKER_IMAGE_TAG',
+  LEADERBOARD_SHEET = 'LEADERBOARD_SHEET',
   LOAD_TEST_CLIENTS = 'LOAD_TEST_CLIENTS',
   LOAD_TEST_GENESIS_BALANCE = 'LOAD_TEST_GENESIS_BALANCE',
   LOAD_TEST_TX_DELAY_MS = 'LOAD_TEST_TX_DELAY_MS',

packages/celotool/src/lib/leaderboard.ts

@@ -0,0 +1,56 @@
+import yaml from 'js-yaml'
+import { envVar, fetchEnv } from 'src/lib/env-utils'
+import { installGenericHelmChart, removeGenericHelmChart } from 'src/lib/helm_deploy'
+import { execCmd, execCmdWithExitOnFailure } from 'src/lib/utils'
+
+const helmChartPath = '../helm-charts/leaderboard'
+
+export async function installHelmChart(celoEnv: string) {
+  return installGenericHelmChart(
+    celoEnv,
+    releaseName(celoEnv),
+    helmChartPath,
+    await helmParameters(celoEnv)
+  )
+}
+
+export async function removeHelmRelease(celoEnv: string) {
+  await removeGenericHelmChart(releaseName(celoEnv))
+}
+
+export async function upgradeHelmChart(celoEnv: string) {
+  console.info(`Upgrading helm release ${releaseName(celoEnv)}`)
+
+  const params = (await helmParameters(celoEnv)).join(' ')
+
+  const upgradeCmdArgs = `${releaseName(celoEnv)} ${helmChartPath} --namespace ${celoEnv} ${params}`
+
+  if (process.env.CELOTOOL_VERBOSE === 'true') {
+    await execCmdWithExitOnFailure(`helm upgrade --debug --dry-run ${upgradeCmdArgs}`)
+  }
+  await execCmdWithExitOnFailure(`helm upgrade ${upgradeCmdArgs}`)
+  console.info(`Helm release ${releaseName(celoEnv)} upgrade successful`)
+}
+
+export async function helmParameters(celoEnv: string) {
+  const dbValues = await getBlockscoutHelmValues(celoEnv)
+  return [
+    `--set leaderboard.db.connection_name=${dbValues.connection_name}`,
+    `--set leaderboard.db.username=${dbValues.username}`,
+    `--set leaderboard.db.password=${dbValues.password}`,
+    `--set leaderboard.image.repository=${fetchEnv(envVar.LEADERBOARD_DOCKER_IMAGE_REPOSITORY)}`,
+    `--set leaderboard.image.tag=${fetchEnv(envVar.LEADERBOARD_DOCKER_IMAGE_TAG)}`,
+    `--set leaderboard.sheet=${fetchEnv(envVar.LEADERBOARD_SHEET)}`,
+    `--set leaderboard.web3=https://${celoEnv}-forno.${fetchEnv(envVar.CLUSTER_DOMAIN_NAME)}.org`,
+  ]
+}
+
+function releaseName(celoEnv: string) {
+  return `${celoEnv}-leaderboard`
+}
+
+export async function getBlockscoutHelmValues(celoEnv: string) {
+  const [output] = await execCmd(`helm get values ${celoEnv}-blockscout`)
+  const blockscoutValues: any = yaml.safeLoad(output)
+  return blockscoutValues.blockscout.db
+}

packages/contractkit/src/identity/claims/verify.ts

@@ -59,7 +59,7 @@ export const verifyAccountClaim = async (
 
   const accountClaims = metadata.filterClaims(ClaimTypes.ACCOUNT)
 
-  if (accountClaims.find((x) => x.address === address) === undefined) {
+  if (accountClaims.find((x) => x.address.toLowerCase() === address.toLowerCase()) === undefined) {
     return `${claim.address} did not claim ${address}`
   }
 

packages/docs/getting-started/running-a-validator.md

@@ -274,7 +274,7 @@ The `mine` flag does not mean the node starts mining blocks, but rather starts t
 
 The `networkid` parameter value of `12219` indicates we are connecting to the Baklava network, Stake Off Phase 1.
 
-Note that if you are running the validator and the proxy on the same machine, then you should set the validator's listening port to something other than `30303`.  E.g. you could use the flag `--port 30313` and set the docker port forwarding rules accordingly (e.g. use the flags `-p 30313:30313` and `-p 30313:30313/udp`).
+Note that if you are running the validator and the proxy on the same machine, then you should set the validator's listening port to something other than `30303`. E.g. you could use the flag `--port 30313` and set the docker port forwarding rules accordingly (e.g. use the flags `-p 30313:30313` and `-p 30313:30313/udp`).
 
 ### Register the Accounts
 

packages/helm-charts/leaderboard/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

packages/helm-charts/leaderboard/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: leaderboard
+version: 0.1.0

packages/helm-charts/leaderboard/templates/leaderboard.cronjob.yaml

@@ -0,0 +1,80 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: {{ .Release.Name }}
+  labels:
+    app: leaderboard
+    chart: leaderboard
+    release: {{ .Release.Service }}
+    component: leaderboard
+spec:
+  schedule: "{{ .Values.leaderboard.schedule }}"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        spec:
+          containers:
+          - name: cloudsql-proxy
+            image: gcr.io/cloudsql-docker/gce-proxy:1.11
+            command:
+            - /bin/sh
+            args:
+            - -c
+            - |
+              /cloud_sql_proxy \
+              -instances={{ .Values.leaderboard.db.connection_name }}=tcp:5432 \
+              -credential_file=/secrets/cloudsql/credentials.json &
+              CHILD_PID=$!
+              (while true; do if [[ -f "/tmp/pod/main-terminated" ]]; then kill $CHILD_PID; fi; sleep 1; done) &
+              wait $CHILD_PID
+              if [[ -f "/tmp/pod/main-terminated" ]]; then exit 0; fi
+            securityContext:
+            runAsUser: 2  # non-root user
+            allowPrivilegeEscalation: false
+            volumeMounts:
+            - name: blockscout-cloudsql-credentials
+              mountPath: /secrets/cloudsql
+              readOnly: true
+            - mountPath: /tmp/pod
+              name: tmp-pod
+              readOnly: true
+          - name: update-sheet
+            image: {{ .Values.leaderboard.image.repository }}:{{ .Values.leaderboard.image.tag }}
+            imagePullPolicy: IfNotPresent
+            command:
+            - /bin/sh
+            args:
+            - -c
+            - |
+              trap "touch /tmp/pod/main-terminated" EXIT
+              yarn run ts-node src/board.ts
+            env:
+            - name: LEADERBOARD_DATABASE
+              value: {{ .Values.leaderboard.database }}
+            - name: LEADERBOARD_WEB3
+              value: {{ .Values.leaderboard.web3 }}
+            - name: LEADERBOARD_SHEET
+              value: {{ .Values.leaderboard.sheet }}
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Namespace }}-leaderboard
+                  key: DATABASE_USER
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Namespace }}-leaderboard
+                  key: DATABASE_PASSWORD
+            volumeMounts:
+            - mountPath: /tmp/pod
+              name: tmp-pod
+          restartPolicy: Never
+          volumes:
+          - name: blockscout-cloudsql-credentials
+            secret:
+              defaultMode: 420
+              secretName: blockscout-cloudsql-credentials
+          - name: tmp-pod
+            emptyDir: {}

packages/helm-charts/leaderboard/templates/leaderboard.secret.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Namespace }}-leaderboard
+  labels:
+    app: leaderboard
+    chart: leaderboard
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+type: Opaque
+data:
+  DATABASE_USER: {{ .Values.leaderboard.db.username | b64enc | quote }}
+  DATABASE_PASSWORD: {{ .Values.leaderboard.db.password | b64enc | quote }}

packages/helm-charts/leaderboard/values.yaml

@@ -0,0 +1,16 @@
+nodeSelector: {}
+
+# Expects cron schedule syntax
+leaderboard:
+  schedule: "*/1 * * * *"
+  database: "blockscout"
+  image:
+    repository: jcortejoso/leaderboard
+    tag: latest
+  database: blockscout
+  web3: https://baklavastaging-forno.celo-testnet.org/
+  sheet: 1HCs1LZv1BOB1v2bVlH4qNPnxVRlYVhQ7CKhkMibE4EY
+  db:
+    username: ls880fd3
+    password: dg7mTjVB2khgkDFd
+    connection_name: celo-testnet:us-west1:baklavastaging0

packages/leaderboard/.gitignore

@@ -0,0 +1,2 @@
+credentials.json
+token.json