20221012 Integrate progress until sprint 13 (#20)

* 74 index loanentity data types (#86)

* feat: create loanService

* refactor: improved service getters

* feat: increase outstandingDebt on borrowings

* feat: basics for handle loan prices

* feat: save priced loans

* feat: activate loan when priced

* feat: handle writeoffs

* feat: handle loan closed and write offs

* fix: loan.repay() should increase totalRepaid

* fix: invest execution decimals and avoid empty transactions (#87)

* 82 add prices to all investortransaction types (#88)

* chore: update chain data

* feat: extend InvestorTransaction decoration and refactor

* feat: use rpc prices from trancheTokenPrices() when investor transactions are processed (#89)

* feat: improved loans and borrower transactions (#90)

* 63 convert outflows to dai (#91)

* feat: add currency value for redeem fields

* fix: currency conversion should support null prices

* 98 failing poolservicegettranchetokenprices causes period to be skipped for remaining pools (#99)

* ci: fix deployment version

* fix: handle missing price updates from RPC

* fix: throw errors on missing prices

* feat: aggregate total investments and repayments for epoches (#100)

* feat: index accounts and tranche balances (#101)

* docs: add deployment status badge

* Update README.md

docs: fix badge link to deployment workflow

* feat: tracking loans history with state snapshot (#102)

* 56 handle updates to pool information poolsupdated event (#106)

* chore: upgraded packages

* feat: track pool updates

* 103 totalinvested  and totalredeemed  are missing in poolsnapshot (#107)

* fix: add missing properties to snapshot

* deploy: update node and query versions

* fix: pool investment and redemptions totals

* feat: indexing missing values (#108)

* ci: updated deployment

* ci: upgraded packages

* feat: paginated getters for snapshots and models (#109)

* chore: upgrade packages

* 104 track currency balances for accounts (#111)

* feat: add data model for currency balances

* feat: track endowments withdrawals and deposits of currency

* feat: get initial balance on currency balance init

* ci: revert subql/node version

Co-authored-by: Timon <timon@embrio.tech>

.github/workflows/deploy.yml

@@ -63,5 +63,5 @@ jobs:
           --projectName="$SUBQL_PROJ_NAME" \
           --ipfsCID="$IPFSCID" \
           --type=primary \
-          --indexerVersion="v1.9.1" \
-          --queryVersion="v1.5.0"
+          --indexerVersion="v1.9.2" \
+          --queryVersion="v1.6.0"

README.md

@@ -8,6 +8,8 @@ A GraphQL API to query the Centrifuge Parachain on Polkadot.
 
 ## :seedling: Staging
 
+[![Pipeline](https://github.com/embrio-tech/centrifuge-subql/actions/workflows/deploy.yml/badge.svg)](https://github.com/embrio-tech/centrifuge-subql/actions/workflows/deploy.yml)
+
 Use GraphQL query endpoint at [https://api.subquery.network/sq/embrio-tech/centrifuge-subql](https://explorer.subquery.network/subquery/embrio-tech/centrifuge-subql) or try the API with the [SubQuery Playground](https://explorer.subquery.network/subquery/embrio-tech/centrifuge-subql).
 
 ## :construction_worker_man: Development

docker-compose.yml

@@ -2,7 +2,9 @@ version: '3'
 
 services:
   postgres:
-    image: postgres:12-alpine
+    build:
+      context: .
+      dockerfile: ./docker/pg-Dockerfile
     ports:
       - 5432:5432
     volumes:

docker/load-extensions.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<EOF
+CREATE EXTENSION IF NOT EXISTS btree_gist;
+EOF

docker/pg-Dockerfile

@@ -0,0 +1,9 @@
+FROM postgres:12-alpine
+
+# Variables needed at runtime to configure postgres and run the initdb scripts
+ENV POSTGRES_DB 'postgres'
+ENV POSTGRES_USER 'postgres'
+ENV POSTGRES_PASSWORD 'postgres'
+
+# Copy in the load-extensions script
+COPY docker/load-extensions.sh /docker-entrypoint-initdb.d/

package.json

@@ -4,8 +4,8 @@
   "description": "Subquery for the pools functionality",
   "main": "dist/index.js",
   "scripts": {
-    "build": "tsc -b",
-    "prepack": "rm -rf dist && npm build",
+    "build": "subql build",
+    "prepack": "rm -rf dist && subql build",
     "test": "jest",
     "codegen": "./node_modules/.bin/subql codegen",
     "generate:defs": "ts-node --skip-project node_modules/.bin/polkadot-types-from-defs --package centrifuge-subql/src/api-interfaces --endpoint 'wss://fullnode.development.cntrfg.com' --input ./src/api-interfaces",
@@ -24,16 +24,15 @@
   ],
   "author": "Centrifuge",
   "license": "MIT",
-  "dependencies": {
-    "@polkadot/api": "^8"
-  },
+  "dependencies": {},
   "devDependencies": {
     "@commitlint/cli": "^17.0.2",
     "@commitlint/config-conventional": "^17.0.2",
-    "@polkadot/typegen": "^8",
-    "@polkadot/types": "^8",
-    "@polkadot/util": "^8",
-    "@subql/cli": "^1.5.0",
+    "@polkadot/api": "^9",
+    "@polkadot/typegen": "^9",
+    "@polkadot/types": "^9",
+    "@polkadot/util": "^9",
+    "@subql/cli": "latest",
     "@typescript-eslint/eslint-plugin": "^5.28.0",
     "@typescript-eslint/parser": "^5.28.0",
     "eslint": "^8.17.0",

project.yaml

@@ -15,7 +15,7 @@ schema:
 network:
   endpoint: wss://fullnode.development.cntrfg.com
   genesisHash: '0x2c33566932add70371a14fded47c139e9bc04412e53216e16978fa1e9e122c56'
-  dictionary: ''
+#  dictionary: ''
   chaintypes:
     file: ./dist/chaintypes.js
 dataSources:
@@ -103,3 +103,18 @@ dataSources:
           filter:
             module: ormlTokens
             method: Transfer
+        - handler: handleTokenEndowed
+          kind: substrate/EventHandler
+          filter:
+            module: ormlTokens
+            method: Endowed
+        - handler: handleTokenDeposited
+          kind: substrate/EventHandler
+          filter:
+            module: ormlTokens
+            method: Deposited
+        - handler: handleTokenWithdrawn
+          kind: substrate/EventHandler
+          filter:
+            module: ormlTokens
+            method: Withdrawn

schema.graphql

@@ -33,6 +33,7 @@ type Pool @entity {
 type PoolState @entity {
   id: ID! #poolId
   type: String! @index
+
   #States
   netAssetValue: BigInt!
   totalReserve: BigInt!
@@ -48,6 +49,8 @@ type PoolState @entity {
   totalRedeemed_: BigInt
   totalNumberOfLoans_: BigInt
 
+  totalNumberOfActiveLoans: BigInt!
+
   # Cumulated transaction data since pool creation
   totalEverBorrowed: BigInt
   totalEverNumberOfLoans: BigInt
@@ -74,8 +77,12 @@ type PoolSnapshot @entity {
   # Aggregated transaction data over the last period
   totalBorrowed_: BigInt
   totalRepaid_: BigInt
+  totalInvested_: BigInt
+  totalRedeemed_: BigInt
   totalNumberOfLoans_: BigInt
 
+  totalNumberOfActiveLoans: BigInt!
+
   # Cumulated transaction data since pool creation
   totalEverBorrowed: BigInt
   totalEverNumberOfLoans: BigInt
@@ -100,6 +107,7 @@ type Tranche @entity {
 type TrancheState @entity {
   id: ID! #poolId-trancheId
   type: String! @index
+  active: Boolean! @index
 
   supply: BigInt
   price: BigInt
@@ -158,6 +166,8 @@ type Epoch @entity {
   # Aggregated data during this epoch
   totalBorrowed: BigInt!
   totalRepaid: BigInt!
+  totalInvested: BigInt!
+  totalRedeemed: BigInt!
 
   epochStates: [EpochState] @derivedFrom(field: "epoch")
 
@@ -169,9 +179,6 @@ type EpochState @entity {
   epoch: Epoch! @index
   trancheId: String!
 
-  #investorTransactions: [InvestorTransaction] @derivedFrom(field: "epochState")
-  #borrowerTransctions: [BorrowerTransaction] @derivedFrom(field: "epochState")
-
   price: BigInt
 
   outstandingInvestOrders: BigInt!
@@ -198,15 +205,28 @@ enum InvestorTransactionType {
   REDEEM_COLLECT
 }
 
+type OutstandingOrder @entity {
+  id: ID! # pool id - tranche id - investor address
+  hash: String!
+  account: Account! @index #Account @index
+  pool: Pool! @index
+  tranche: Tranche! @index
+  epochNumber: Int!
+  timestamp: Date!
+
+  invest: BigInt!
+  redeem: BigInt!
+}
+
 type InvestorTransaction @entity {
   id: ID! # extrinsic hash - epoch number - transaction type
   hash: String!
-  accountId: String #Account @index
+  account: Account! @index #Account @index
   pool: Pool! @index
   tranche: Tranche! @index
-  epochNumber: Int! @index
+  epochNumber: Int!
   timestamp: Date!
-  epoch: Epoch!
+  epoch: Epoch! @index
 
   type: InvestorTransactionType!
   tokenAmount: BigInt
@@ -215,34 +235,64 @@ type InvestorTransaction @entity {
   transactionFee: BigInt
 }
 
-type OutstandingOrder @entity {
-  id: ID! # pool id - tranche id - investor address
-  hash: String!
-  accountId: String #Account @index
-  pool: Pool! @index
-  tranche: Tranche! @index
-  epochNumber: Int! @index
+enum BorrowerTransactionType {
+  CREATED
+  PRICED
+  BORROWED
+  REPAID
+  CLOSED
+}
+
+type BorrowerTransaction @entity {
+  id: ID! # extrinsic hash - epoch number - transaction type
   timestamp: Date!
+  pool: Pool! @index
+  hash: String!
 
-  invest: BigInt!
-  redeem: BigInt!
+  account: Account! @index
+  epochNumber: Int!
+  epoch: Epoch! @index
+  loan: Loan!
+  type: BorrowerTransactionType!
+
+  # only applies to BORROWED and REPAID transactions
+  amount: BigInt
 }
 
 type Account @entity {
   id: ID!
 
-  publicAddress: String!
+  anonymousProxies: [AnonymousProxy] @derivedFrom(field: "account")
+
+  investorTransactions: [InvestorTransaction] @derivedFrom(field: "account")
+  borrowerTransactions: [BorrowerTransaction] @derivedFrom(field: "account")
+
+  outstandingOrders: [OutstandingOrder] @derivedFrom(field: "account")
+  trancheBalances: [TrancheBalance] @derivedFrom(field: "account")
+  currencyBalances: [CurrencyBalance] @derivedFrom(field: "account")
 }
 
-type AccountBalance @entity {
-  id: ID! # pool id - tranche id - account id
-  account: Account @index
+type TrancheBalance @entity {
+  id: ID! # address - pool id - tranche id
+  account: Account! @index
   pool: Pool! @index
   tranche: Tranche! @index
 
-  collected: BigInt!
-  uncollected: BigInt!
-  ordered: BigInt!
+  investOrdered: BigInt!
+  investUncollected: BigInt!
+  investCollected: BigInt!
+
+  redeemOrdered: BigInt!
+  redeemUncollected: BigInt!
+  redeemCollected: BigInt!
+}
+
+type CurrencyBalance @entity {
+  id: ID! # address - currencyId
+  account: Account! @index
+
+  currency: Currency!
+  amount: BigInt!
 }
 
 enum LoanStatus {
@@ -261,54 +311,57 @@ type Loan @entity {
   id: ID! # poolId - loanId
   createdAt: Date!
 
-  # collateral:
-  status: LoanStatus!
+  collateral: BigInt!
+  maturityDate: Date
 
   type: LoanType
   spec: String
 
   interestRatePerSec: BigInt
+
+  adminWrittenOff: Boolean
+
+  pool: Pool!
+  state: LoanState!
+}
+
+type LoanState @entity {
+  id: ID! # poolId - loanId
+  active: Boolean! @index
+  status: LoanStatus!
+
   outstandingDebt: BigInt
 
-  totalBorrowed: BigInt
-  totalRepaid: BigInt
+  totalBorrowed_: BigInt
+  totalRepaid_: BigInt
 
   writeOffIndex: Int
   writtenOffPercentage: BigInt
   penaltyInterestRatePerSec: BigInt
-  adminWrittenOff: Boolean
-
-  pool: Pool!
 }
 
-enum BorrowerTransactionType {
-  CREATED
-  PRICED
-  BORROWED
-  REPAID
-  CLOSED
-}
+type LoanSnapshot @entity {
+  id: ID! # poolId - loanId - blockNumber
+  loan: Loan!
 
-type BorrowerTransaction @entity {
-  id: ID! # extrinsic hash - epoch number - transaction type
   timestamp: Date!
-  pool: Pool! @index
-  hash: String!
+  blockNumber: Int!
+  periodStart: Date! @index
 
-  accountId: String!
-  epochNumber: Int! @index
-  epoch: Epoch!
-  loan: Loan! @index
-  type: BorrowerTransactionType!
+  outstandingDebt: BigInt
 
-  # only applies to BORROWED and REPAID transactions
-  amount: BigInt
+  totalBorrowed_: BigInt
+  totalRepaid_: BigInt
+
+  writeOffIndex: Int
+  writtenOffPercentage: BigInt
+  penaltyInterestRatePerSec: BigInt
 }
 
 type AnonymousProxy @entity {
   id: ID!
 
-  account: String!
+  account: Account! @index
   createdBy: String!
   proxyType: String
 }

src/helpers/paginatedGetter.ts

@@ -0,0 +1,21 @@
+import { errorHandler } from './errorHandler'
+
+type StoreArgs = Parameters<typeof store.getByField>
+type StoreReturn = ReturnType<typeof store.getByField>
+
+interface Entity {
+  id: string
+}
+
+async function _paginatedGetter(entity: StoreArgs[0], field: StoreArgs[1], value: StoreArgs[2]): StoreReturn {
+  let results: Entity[] = []
+  const batch = 100
+  let amount = 0
+  do {
+    const entities = await store.getByField(entity, field, value, { offset: amount, limit: batch })
+    results = results.concat(entities)
+    amount += results.length
+  } while (results.length === batch)
+  return results
+}
+export const paginatedGetter = errorHandler(_paginatedGetter)