Remove API interface (#27)

* Add Syncer (WIP commit)

* Further implement the blocklist sync

* Add section in README.md

* Add missing nil check

* Refactor the syncer and cover with unit tests

* Remove unused method

* Add NOTE

* Add testing for load env helpers

* Implement MR remarks

* Implement MR remarks

* Add index on invalid

* Implement MR remarks

* Append unset err

* Implement MR remarks

* Use portal blocklist and add tests

* Cleanup PR

* Cleanup PR

* Check sync start error

* Cleanup PR

* Cleanup PR

* Cleanup PR

* Update start prop on blocker

* Defer unlock

* Implement MR remarks

* Implement MR remarks

* Add testing

* Update endpoint

* Add a unit test for the client

* Remove the API interface

* Implement MR remarks

api/api.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 
 	"github.com/SkynetLabs/blocker/database"
-	"github.com/SkynetLabs/blocker/skyd"
 	"github.com/julienschmidt/httprouter"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/NebulousLabs/errors"
@@ -14,31 +13,31 @@ import (
 // API is our central entry point to all subsystems relevant to serving
 // requests.
 type API struct {
-	staticDB      *database.DB
-	staticLogger  *logrus.Logger
-	staticRouter  *httprouter.Router
-	staticSkydAPI skyd.API
+	staticDB         *database.DB
+	staticLogger     *logrus.Logger
+	staticRouter     *httprouter.Router
+	staticSkydClient *Client
 }
 
 // New creates a new API instance.
-func New(skydAPI skyd.API, db *database.DB, logger *logrus.Logger) (*API, error) {
+func New(skydClient *Client, db *database.DB, logger *logrus.Logger) (*API, error) {
 	if db == nil {
 		return nil, errors.New("no DB provided")
 	}
 	if logger == nil {
 		return nil, errors.New("no logger provided")
 	}
-	if skydAPI == nil {
-		return nil, errors.New("no skyd API provided")
+	if skydClient == nil {
+		return nil, errors.New("no skyd client provided")
 	}
 	router := httprouter.New()
 	router.RedirectTrailingSlash = true
 
 	api := &API{
-		staticDB:      db,
-		staticLogger:  logger,
-		staticRouter:  router,
-		staticSkydAPI: skydAPI,
+		staticDB:         db,
+		staticLogger:     logger,
+		staticRouter:     router,
+		staticSkydClient: skydClient,
 	}
 
 	api.buildHTTPRoutes()

api/api_test.go

@@ -10,9 +10,7 @@ import (
 	url "net/url"
 
 	"github.com/SkynetLabs/blocker/database"
-	"github.com/SkynetLabs/blocker/skyd"
 	"github.com/sirupsen/logrus"
-	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
 // apiTester is a helper struct wrapping handlers of the underlying API that
@@ -27,32 +25,16 @@ func newAPITester(api *API) *apiTester {
 }
 
 // newTestAPI returns a new API instance
-func newTestAPI(dbName string, skyd skyd.API) (*API, error) {
+func newTestAPI(dbName string, client *Client) (*API, error) {
 	// create a nil logger
 	logger := logrus.New()
 	logger.Out = ioutil.Discard
 
 	// create database
-	db, err := database.NewCustomDB(context.Background(), "mongodb://localhost:37017", dbName, options.Credential{
-		Username: "admin",
-		Password: "aO4tV5tC1oU3oQ7u",
-	}, logger)
-	if err != nil {
-		return nil, err
-	}
-
-	// create a context with timeout
-	ctx, cancel := context.WithTimeout(context.Background(), database.MongoDefaultTimeout)
-	defer cancel()
-
-	// purge the database
-	err = db.Purge(ctx)
-	if err != nil {
-		panic(err)
-	}
+	db := database.NewTestDB(context.Background(), dbName, logger)
 
 	// create the API
-	api, err := New(skyd, db, logger)
+	api, err := New(client, db, logger)
 	if err != nil {
 		return nil, err
 	}

api/client.go

@@ -1,26 +1,101 @@
 package api
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
-	url "net/url"
+	"net/url"
 
+	skyapi "gitlab.com/SkynetLabs/skyd/node/api"
+	"gitlab.com/SkynetLabs/skyd/skymodules"
+
+	"github.com/SkynetLabs/blocker/database"
 	"gitlab.com/NebulousLabs/errors"
 	"gitlab.com/SkynetLabs/skyd/node/api"
 )
 
-// Client is a helper struct that gets initialised using a portal url. It
-// exposes API methods and abstracts the response handling.
-type Client struct {
-	staticPortalURL string
-}
+const (
+	// clientDefaultTimeout is the timeout of the http calls to in seconds
+	clientDefaultTimeout = "30"
+)
+
+type (
+	// Client is a helper struct that gets initialised using a portal url. It
+	// exposes API methods and abstracts the response handling.
+	Client struct {
+		staticDefaultHeaders http.Header
+		staticPortalURL      string
+	}
+
+	// BlockResponse is the response object returned by the Skyd API's block
+	// endpoint
+	BlockResponse struct {
+		Invalids []InvalidInput `json:"invalids"`
+	}
+
+	// DaemonReadyResponse is the response object returned by the Skyd API's
+	// ready endpoint
+	DaemonReadyResponse struct {
+		Ready     bool `json:"ready"`
+		Consensus bool `json:"consensus"`
+		Gateway   bool `json:"gateway"`
+		Renter    bool `json:"renter"`
+	}
+
+	// InvalidInput is a struct that wraps the invalid input along with an error
+	// string indicating why it was deemed invalid
+	InvalidInput struct {
+		Input string `json:"input"`
+		Error string `json:"error"`
+	}
+
+	// resolveResponse is the response object returned by the Skyd API's resolve
+	// endpoint
+	resolveResponse struct {
+		Skylink string `json:"skylink"`
+	}
+)
 
 // NewClient returns a new Client instance for given portal url.
 func NewClient(portalURL string) *Client {
-	return &Client{staticPortalURL: portalURL}
+	return NewCustomClient(portalURL, http.Header{})
+}
+
+// NewSkydClient returns a client that has the default user-agent set.
+func NewSkydClient(portalURL string, headers http.Header) *Client {
+	headers.Add("User-Agent", "Sia-Agent")
+	return NewCustomClient(portalURL, headers)
+}
+
+// NewCustomClient returns a new Client instance for given portal url and lets
+// you pass a set of headers that will be set on every request.
+func NewCustomClient(portalURL string, headers http.Header) *Client {
+	return &Client{
+		staticDefaultHeaders: headers,
+		staticPortalURL:      portalURL,
+	}
+}
+
+// InvalidHashes is a helper method that converts the list of invalid inputs to
+// an array of hashes.
+func (br *BlockResponse) InvalidHashes() ([]database.Hash, error) {
+	if len(br.Invalids) == 0 {
+		return nil, nil
+	}
+
+	hashes := make([]database.Hash, len(br.Invalids))
+	for i, invalid := range br.Invalids {
+		var h database.Hash
+		err := h.LoadString(invalid.Input)
+		if err != nil {
+			return nil, err
+		}
+		hashes[i] = h
+	}
+	return hashes, nil
 }
 
 // BlocklistGET calls the `/portal/blocklist` endpoint with given parameters
@@ -40,12 +115,96 @@ func (c *Client) BlocklistGET(offset int) (*BlocklistGET, error) {
 	return &blg, nil
 }
 
+// BlockHashes will perform an API call to skyd to block the given hashes. It
+// returns which hashes were blocked, which hashes were invalid and potentially
+// an error.
+func (c *Client) BlockHashes(hashes []database.Hash) ([]database.Hash, []database.Hash, error) {
+	// convert the hashes to strings
+	adds := make([]string, len(hashes))
+	for h, hash := range hashes {
+		adds[h] = hash.String()
+	}
+
+	// build the post body
+	reqBody, err := json.Marshal(skyapi.SkynetBlocklistPOST{
+		Add:    adds,
+		Remove: nil,
+		IsHash: true,
+	})
+	if err != nil {
+		return nil, nil, errors.AddContext(err, "failed to build request body")
+	}
+	body := bytes.NewBuffer(reqBody)
+
+	// build the query parameters
+	query := url.Values{}
+	query.Add("timeout", clientDefaultTimeout)
+
+	// execute the request
+	var response BlockResponse
+	err = c.post("/skynet/blocklist", query, body, &response)
+	if err != nil {
+		return nil, nil, errors.AddContext(err, "failed to execute POST request")
+	}
+
+	// parse the invalid hashes from the response
+	invalids, err := response.InvalidHashes()
+	if err != nil {
+		return nil, nil, errors.AddContext(err, "failed to parse invalid hashes from skyd response")
+	}
+
+	return database.DiffHashes(hashes, invalids), invalids, nil
+}
+
+// ResolveSkylink will resolve the given skylink.
+func (c *Client) ResolveSkylink(skylink skymodules.Skylink) (skymodules.Skylink, error) {
+	// no need to resolve the skylink if it's a v1 skylink
+	if skylink.IsSkylinkV1() {
+		return skylink, nil
+	}
+
+	// execute the request
+	var response resolveResponse
+	endpoint := fmt.Sprintf("/skynet/resolve/%s", skylink.String())
+	err := c.get(endpoint, url.Values{}, &response)
+	if err != nil {
+		return skymodules.Skylink{}, errors.AddContext(err, "failed to execute GET request")
+	}
+
+	// check whether we resolved a valid skylink
+	err = skylink.LoadString(response.Skylink)
+	if err != nil {
+		return skymodules.Skylink{}, errors.AddContext(err, "unable to load the resolved skylink")
+	}
+	return skylink, nil
+}
+
+// DaemonReady connects to the local skyd and checks its status.
+// Returns true only if skyd is fully ready.
+func (c *Client) DaemonReady() bool {
+	var response DaemonReadyResponse
+	err := c.get("/daemon/ready", url.Values{}, &response)
+	if err != nil {
+		return false
+	}
+
+	return response.Ready &&
+		response.Consensus &&
+		response.Gateway &&
+		response.Renter
+}
+
 // get is a helper function that executes a GET request on the given endpoint
 // with the provided query values. The response will get unmarshaled into the
 // given response object.
 func (c *Client) get(endpoint string, query url.Values, obj interface{}) error {
 	// create the request
-	url := fmt.Sprintf("%s%s?%s", c.staticPortalURL, endpoint, query.Encode())
+	queryString := query.Encode()
+	url := fmt.Sprintf("%s%s", c.staticPortalURL, endpoint)
+	if queryString != "" {
+		url = fmt.Sprintf("%s%s?%s", c.staticPortalURL, endpoint, queryString)
+	}
+
 	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return errors.AddContext(err, "failed to create request")
@@ -72,6 +231,39 @@ func (c *Client) get(endpoint string, query url.Values, obj interface{}) error {
 	return nil
 }
 
+// post is a helper function that executes a POST request on the given endpoint
+// with the provided query values.
+func (c *Client) post(endpoint string, query url.Values, body io.Reader, obj interface{}) error {
+	// create the request
+	url := fmt.Sprintf("%s%s?%s", c.staticPortalURL, endpoint, query.Encode())
+	req, err := http.NewRequest(http.MethodPost, url, body)
+	if err != nil {
+		return errors.AddContext(err, "failed to create request")
+	}
+
+	// set headers and execute the request
+	for k, v := range c.staticDefaultHeaders {
+		req.Header.Set(k, v[0])
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer drainAndClose(res.Body)
+
+	// return an error if the status code is not in the 200s
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		return fmt.Errorf("GET request to '%s' with status %d error %v", url, res.StatusCode, readAPIError(res.Body))
+	}
+
+	// handle the response body
+	err = json.NewDecoder(res.Body).Decode(obj)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // drainAndClose reads rc until EOF and then closes it. drainAndClose should
 // always be called on HTTP response bodies, because if the body is not fully
 // read, the underlying connection can't be reused.