From eddd7f8b6ef7a1c01f5bc624a2460758014c6fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 13 Jan 2025 14:51:59 -0800 Subject: [PATCH] update get-contracts tool to use new find.xyz API --- .../compatibility-check-template.yml | 6 +- .github/workflows/compatibility-check.yml | 4 +- .github/workflows/get-contracts.yml | 6 +- tools/get-contracts/main.go | 108 ++++++------------ 4 files changed, 45 insertions(+), 79 deletions(-) diff --git a/.github/workflows/compatibility-check-template.yml b/.github/workflows/compatibility-check-template.yml index 2f69e88045..e566575388 100644 --- a/.github/workflows/compatibility-check-template.yml +++ b/.github/workflows/compatibility-check-template.yml @@ -17,7 +17,7 @@ on: required: true type: string secrets: - FLOWDIVER_API_KEY: + FIND_API_AUTH: required: true env: @@ -62,10 +62,10 @@ jobs: - name: Download contracts if: ${{ steps.restore-cached-contracts.outputs.cache-hit != 'true' }} env: - FLOWDIVER_API_KEY: ${{ secrets.FLOWDIVER_API_KEY }} + FIND_API_AUTH: ${{ secrets.FIND_API_AUTH }} working-directory: ./tools/get-contracts run: | - go run . --chain=${{ inputs.chain }} --apiKey="$FLOWDIVER_API_KEY" > ../../tmp/contracts.csv + go run . -chain=${{ inputs.chain }} -auth="$FIND_API_AUTH" > ../../tmp/contracts.csv - name: Cache Contracts id: cache-contracts diff --git a/.github/workflows/compatibility-check.yml b/.github/workflows/compatibility-check.yml index 2be88025c5..b13358a497 100644 --- a/.github/workflows/compatibility-check.yml +++ b/.github/workflows/compatibility-check.yml @@ -66,7 +66,7 @@ jobs: current-branch: ${{ needs.setup.outputs.branch }} chain: mainnet secrets: - FLOWDIVER_API_KEY: ${{ secrets.FLOWDIVER_API_KEY }} + FIND_API_AUTH: ${{ secrets.FIND_API_AUTH }} testnet: needs: setup @@ -77,4 +77,4 @@ jobs: current-branch: ${{ needs.setup.outputs.branch }} chain: testnet secrets: - FLOWDIVER_API_KEY: ${{ secrets.FLOWDIVER_API_KEY }} + FIND_API_AUTH: ${{ secrets.FIND_API_AUTH }} diff --git a/.github/workflows/get-contracts.yml b/.github/workflows/get-contracts.yml index 93a1f4ffbc..1b688bc3ef 100644 --- a/.github/workflows/get-contracts.yml +++ b/.github/workflows/get-contracts.yml @@ -7,7 +7,7 @@ on: required: true type: string secrets: - FLOWDIVER_API_KEY: + FIND_API_AUTH: required: true env: @@ -37,10 +37,10 @@ jobs: - name: Download contracts env: - FLOWDIVER_API_KEY: ${{ secrets.FLOWDIVER_API_KEY }} + FIND_API_AUTH: ${{ secrets.FIND_API_AUTH }} working-directory: ./tools/get-contracts run: | - go run . --chain=${{ inputs.chain }} --apiKey="$FLOWDIVER_API_KEY" > ../../tmp/contracts.csv + go run . -chain=${{ inputs.chain }} -auth="$FIND_API_AUTH" > ../../tmp/contracts.csv # Upload diff --git a/tools/get-contracts/main.go b/tools/get-contracts/main.go index 3519d24f27..7ee944ed10 100644 --- a/tools/get-contracts/main.go +++ b/tools/get-contracts/main.go @@ -21,16 +21,13 @@ package main import ( - "context" + "encoding/base64" "encoding/csv" "flag" - "fmt" "log" "net/http" "os" "sort" - - "github.com/hasura/go-graphql-client" ) type chainID string @@ -41,17 +38,15 @@ const ( ) var chainFlag = flag.String("chain", "", "mainnet or testnet") -var apiKeyFlag = flag.String("apiKey", "", "Flowdiver API key") -var batchFlag = flag.Int("batch", 500, "batch size") -var csvHeader = []string{"location", "code"} +const authFlagUsage = "find.xyz API auth (username:password)" -func main() { - flag.Parse() +var authFlag = flag.String("auth", "", authFlagUsage) - // Get batch size from flags +var resultCSVHeader = []string{"location", "code"} - batchSize := *batchFlag +func main() { + flag.Parse() // Get chain ID from flags @@ -65,11 +60,11 @@ func main() { log.Fatalf("invalid chain: %s", chain) } - // Get API key from flags + // Get auth from flags - apiKey := *apiKeyFlag - if apiKey == "" { - log.Fatal("missing Flowdiver API key") + auth := *authFlag + if auth == "" { + log.Fatal("missing " + authFlagUsage) } // Get contracts from network @@ -77,71 +72,37 @@ func main() { var apiURL string switch chain { case mainnet: - apiURL = "https://api.findlabs.io/hasura/v1/graphql" + apiURL = "https://api.find.xyz" case testnet: - apiURL = "https://api.findlabs.io/hasura_testnet/v1/graphql" + apiURL = "https://api.test-find.xyz" } - client := graphql.NewClient(apiURL, nil). - WithRequestModifier(func(r *http.Request) { - r.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - // NOTE: important, default is forbidden by API's bot prevention - // (https://github.com/Kong/kong/blob/master/kong/plugins/bot-detection/rules.lua) - r.Header.Set("User-Agent", "Flow Foundation Cadence Tool") - }) - - var total, offset int - var contracts [][]string - - for { - - log.Printf("fetching contracts %d-%d", offset, offset+batchSize) - - var req struct { - ContractsAggregate struct { - Aggregate struct { - Count int - } - } `graphql:"contracts_aggregate(where: {valid_to: {_is_null: true}})"` - Contracts []struct { - Identifier string - Body string - } `graphql:"contracts(where: {valid_to: {_is_null: true}}, limit: $limit, offset: $offset)"` - } - - if err := client.Query( - context.Background(), - &req, - map[string]any{ - "offset": offset, - "limit": batchSize, - }, - ); err != nil { - log.Fatalf("failed to query: %s", err) - } + apiURL += "/bulk/v1/contract?valid_only=true" - total = req.ContractsAggregate.Aggregate.Count + req, err := http.NewRequest("GET", apiURL, nil) + if err != nil { + log.Fatalf("failed to create HTTP request: %s", err) + } - if contracts == nil { - contracts = make([][]string, 0, total) - } + req.Header.Set("Accept", "text/csv") + req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth))) - for _, contract := range req.Contracts { - contracts = append( - contracts, []string{ - contract.Identifier, - contract.Body, - }, - ) - } + res, err := http.DefaultClient.Do(req) + if err != nil { + log.Fatalf("failed to send HTTP request: %s", err) + } - offset += batchSize + reader := csv.NewReader(res.Body) + reader.FieldsPerRecord = -1 - if offset >= total { - break - } + contracts, err := reader.ReadAll() + if err != nil { + log.Fatalf("failed to read CSV: %s", err) } + // Skip header + contracts = contracts[1:] + // Sort sort.Slice( @@ -155,12 +116,17 @@ func main() { writer := csv.NewWriter(os.Stdout) - if err := writer.Write(csvHeader); err != nil { + if err := writer.Write(resultCSVHeader); err != nil { log.Fatalf("failed to write CSV header: %s", err) return } for _, contract := range contracts { + identifier := contract[0] + if identifier == "A." || identifier == "null" { + continue + } + err := writer.Write(contract) if err != nil { log.Fatalf("failed to write contract to CSV: %s", err)