Merge pull request #3725 from onflow/bastian/fix-get-contracts

Update get-contracts tool to use new find.xyz API
onflow · Jan 14, 2025 · 86e7e4e · 86e7e4e
2 parents 5364220 + eddd7f8
commit 86e7e4e
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 79 deletions.
diff --git a/.github/workflows/compatibility-check-template.yml b/.github/workflows/compatibility-check-template.yml
@@ -17,7 +17,7 @@ on:
         required: true
         type: string
     secrets:
-      FLOWDIVER_API_KEY:
+      FIND_API_AUTH:
         required: true
 
 env:
@@ -62,10 +62,10 @@ jobs:
       - name: Download contracts
         if: ${{ steps.restore-cached-contracts.outputs.cache-hit != 'true' }}
         env:
-          FLOWDIVER_API_KEY: ${{ secrets.FLOWDIVER_API_KEY }}
+          FIND_API_AUTH: ${{ secrets.FIND_API_AUTH }}
         working-directory: ./tools/get-contracts
         run: |
-          go run . --chain=${{ inputs.chain }} --apiKey="$FLOWDIVER_API_KEY" > ../../tmp/contracts.csv
+          go run . -chain=${{ inputs.chain }} -auth="$FIND_API_AUTH" > ../../tmp/contracts.csv
 
       - name: Cache Contracts
         id: cache-contracts

diff --git a/.github/workflows/compatibility-check.yml b/.github/workflows/compatibility-check.yml
@@ -66,7 +66,7 @@ jobs:
       current-branch: ${{ needs.setup.outputs.branch }}
       chain: mainnet
     secrets:
-      FLOWDIVER_API_KEY: ${{ secrets.FLOWDIVER_API_KEY }}
+      FIND_API_AUTH: ${{ secrets.FIND_API_AUTH }}
 
   testnet:
     needs: setup
@@ -77,4 +77,4 @@ jobs:
       current-branch: ${{ needs.setup.outputs.branch }}
       chain: testnet
     secrets:
-      FLOWDIVER_API_KEY: ${{ secrets.FLOWDIVER_API_KEY }}
+      FIND_API_AUTH: ${{ secrets.FIND_API_AUTH }}
diff --git a/.github/workflows/get-contracts.yml b/.github/workflows/get-contracts.yml
@@ -7,7 +7,7 @@ on:
         required: true
         type: string
     secrets:
-      FLOWDIVER_API_KEY:
+      FIND_API_AUTH:
         required: true
 
 env:
@@ -37,10 +37,10 @@ jobs:
 
       - name: Download contracts
         env:
-          FLOWDIVER_API_KEY: ${{ secrets.FLOWDIVER_API_KEY }}
+          FIND_API_AUTH: ${{ secrets.FIND_API_AUTH }}
         working-directory: ./tools/get-contracts
         run: |
-          go run . --chain=${{ inputs.chain }} --apiKey="$FLOWDIVER_API_KEY" > ../../tmp/contracts.csv
+          go run . -chain=${{ inputs.chain }} -auth="$FIND_API_AUTH" > ../../tmp/contracts.csv
 
       # Upload
 

diff --git a/tools/get-contracts/main.go b/tools/get-contracts/main.go
@@ -21,16 +21,13 @@
 package main
 
 import (
-	"context"
+	"encoding/base64"
 	"encoding/csv"
 	"flag"
-	"fmt"
 	"log"
 	"net/http"
 	"os"
 	"sort"
-
-	"github.com/hasura/go-graphql-client"
 )
 
 type chainID string
@@ -41,17 +38,15 @@ const (
 )
 
 var chainFlag = flag.String("chain", "", "mainnet or testnet")
-var apiKeyFlag = flag.String("apiKey", "", "Flowdiver API key")
-var batchFlag = flag.Int("batch", 500, "batch size")
 
-var csvHeader = []string{"location", "code"}
+const authFlagUsage = "find.xyz API auth (username:password)"
 
-func main() {
-	flag.Parse()
+var authFlag = flag.String("auth", "", authFlagUsage)
 
-	// Get batch size from flags
+var resultCSVHeader = []string{"location", "code"}
 
-	batchSize := *batchFlag
+func main() {
+	flag.Parse()
 
 	// Get chain ID from flags
 
@@ -65,83 +60,49 @@ func main() {
 		log.Fatalf("invalid chain: %s", chain)
 	}
 
-	// Get API key from flags
+	// Get auth from flags
 
-	apiKey := *apiKeyFlag
-	if apiKey == "" {
-		log.Fatal("missing Flowdiver API key")
+	auth := *authFlag
+	if auth == "" {
+		log.Fatal("missing " + authFlagUsage)
 	}
 
 	// Get contracts from network
 
 	var apiURL string
 	switch chain {
 	case mainnet:
-		apiURL = "https://api.findlabs.io/hasura/v1/graphql"
+		apiURL = "https://api.find.xyz"
 	case testnet:
-		apiURL = "https://api.findlabs.io/hasura_testnet/v1/graphql"
+		apiURL = "https://api.test-find.xyz"
 	}
 
-	client := graphql.NewClient(apiURL, nil).
-		WithRequestModifier(func(r *http.Request) {
-			r.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
-			// NOTE: important, default is forbidden by API's bot prevention
-			// (https://github.com/Kong/kong/blob/master/kong/plugins/bot-detection/rules.lua)
-			r.Header.Set("User-Agent", "Flow Foundation Cadence Tool")
-		})
-
-	var total, offset int
-	var contracts [][]string
-
-	for {
-
-		log.Printf("fetching contracts %d-%d", offset, offset+batchSize)
-
-		var req struct {
-			ContractsAggregate struct {
-				Aggregate struct {
-					Count int
-				}
-			} `graphql:"contracts_aggregate(where: {valid_to: {_is_null: true}})"`
-			Contracts []struct {
-				Identifier string
-				Body       string
-			} `graphql:"contracts(where: {valid_to: {_is_null: true}}, limit: $limit, offset: $offset)"`
-		}
-
-		if err := client.Query(
-			context.Background(),
-			&req,
-			map[string]any{
-				"offset": offset,
-				"limit":  batchSize,
-			},
-		); err != nil {
-			log.Fatalf("failed to query: %s", err)
-		}
+	apiURL += "/bulk/v1/contract?valid_only=true"
 
-		total = req.ContractsAggregate.Aggregate.Count
+	req, err := http.NewRequest("GET", apiURL, nil)
+	if err != nil {
+		log.Fatalf("failed to create HTTP request: %s", err)
+	}
 
-		if contracts == nil {
-			contracts = make([][]string, 0, total)
-		}
+	req.Header.Set("Accept", "text/csv")
+	req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth)))
 
-		for _, contract := range req.Contracts {
-			contracts = append(
-				contracts, []string{
-					contract.Identifier,
-					contract.Body,
-				},
-			)
-		}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Fatalf("failed to send HTTP request: %s", err)
+	}
 
-		offset += batchSize
+	reader := csv.NewReader(res.Body)
+	reader.FieldsPerRecord = -1
 
-		if offset >= total {
-			break
-		}
+	contracts, err := reader.ReadAll()
+	if err != nil {
+		log.Fatalf("failed to read CSV: %s", err)
 	}
 
+	// Skip header
+	contracts = contracts[1:]
+
 	// Sort
 
 	sort.Slice(
@@ -155,12 +116,17 @@ func main() {
 
 	writer := csv.NewWriter(os.Stdout)
 
-	if err := writer.Write(csvHeader); err != nil {
+	if err := writer.Write(resultCSVHeader); err != nil {
 		log.Fatalf("failed to write CSV header: %s", err)
 		return
 	}
 
 	for _, contract := range contracts {
+		identifier := contract[0]
+		if identifier == "A." || identifier == "null" {
+			continue
+		}
+
 		err := writer.Write(contract)
 		if err != nil {
 			log.Fatalf("failed to write contract to CSV: %s", err)