Merge pull request #11 from SkynetLabs/pj/allowlist

Allow List

README.md

@@ -6,6 +6,23 @@ The service exposes a REST API that allows callers to request the blocking of ne
 
 The blocklist is shared between the servers that make up a portal cluster via MongoDB.
 
+# AllowList
+
+The blocker service can only block skylinks which are not in the allow list.
+To add a skylink to the allow list, one has to manually query the database and
+perform the follow operation:
+
+```
+db.getCollection('allowlist').insertOne({
+  skylink: "[INSERT V1 SKYLINK HERE]",
+  description: "[INSERT SKYLINK DESCRIPTION]",
+  timestamp_added: new Date(),
+})
+```
+
+The skylink is expected to be in the following form: `_B19BtlWtjjR7AD0DDzxYanvIhZ7cxXrva5tNNxDht1kaA`.
+So that's without portal and without the `sia://` prefix.
+
 # Environment
 
 This service depends on the following environment variables:

api/api.go

@@ -5,33 +5,39 @@ import (
 	"net/http"
 
 	"github.com/SkynetLabs/blocker/database"
+	"github.com/SkynetLabs/blocker/skyd"
 	"github.com/julienschmidt/httprouter"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/NebulousLabs/errors"
 )
 
 // API is our central entry point to all subsystems relevant to serving requests.
 type API struct {
-	staticDB     *database.DB
-	staticRouter *httprouter.Router
-	staticLogger *logrus.Logger
+	staticDB      *database.DB
+	staticLogger  *logrus.Logger
+	staticRouter  *httprouter.Router
+	staticSkydAPI *skyd.SkydAPI
 }
 
 // New creates a new API instance.
-func New(db *database.DB, logger *logrus.Logger) (*API, error) {
+func New(skydAPI *skyd.SkydAPI, db *database.DB, logger *logrus.Logger) (*API, error) {
 	if db == nil {
 		return nil, errors.New("no DB provided")
 	}
 	if logger == nil {
 		return nil, errors.New("no logger provided")
 	}
+	if skydAPI == nil {
+		return nil, errors.New("no skyd API provided")
+	}
 	router := httprouter.New()
 	router.RedirectTrailingSlash = true
 
 	api := &API{
-		staticDB:     db,
-		staticRouter: router,
-		staticLogger: logger,
+		staticDB:      db,
+		staticLogger:  logger,
+		staticRouter:  router,
+		staticSkydAPI: skydAPI,
 	}
 
 	api.buildHTTPRoutes()

api/handlers.go

@@ -93,11 +93,11 @@ func (api *API) healthGET(w http.ResponseWriter, r *http.Request, _ httprouter.P
 	skyapi.WriteJSON(w, status)
 }
 
-// blockWithPoWPOST blocks a skylink. It is meant to be used by untrusted sources such as
-// the abuse report skapp. The PoW prevents users from easily and anonymously
-// blocking large numbers of skylinks. Instead it encourages reuse of proofs
-// which improves the linkability between reports, thus allowing us to more
-// easily unblock a batch of links.
+// blockWithPoWPOST blocks a skylink. It is meant to be used by untrusted
+// sources such as the abuse report skapp. The PoW prevents users from easily
+// and anonymously blocking large numbers of skylinks. Instead it encourages
+// reuse of proofs which improves the linkability between reports, thus allowing
+// us to more easily unblock a batch of links.
 func (api *API) blockWithPoWPOST(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 	// Protect against large bodies.
 	b := http.MaxBytesReader(w, r.Body, 1<<16) // 64 kib
@@ -122,12 +122,18 @@ func (api *API) blockWithPoWPOST(w http.ResponseWriter, r *http.Request, _ httpr
 	// reporter authenticated.
 	sub := hex.EncodeToString(body.PoW.MySkyID[:])
 
+	// Check whether the skylink is on the allow list
+	if api.staticSkydAPI.IsAllowListed(r.Context(), string(body.Skylink)) {
+		skyapi.WriteJSON(w, statusResponse{"reported"})
+		return
+	}
+
 	// Block the link.
 	err = api.block(r.Context(), body.BlockPOST, sub, true)
 	if err != nil {
 		skyapi.WriteError(w, skyapi.Error{err.Error()}, http.StatusInternalServerError)
 	}
-	skyapi.WriteSuccess(w)
+	skyapi.WriteJSON(w, statusResponse{"reported"})
 }
 
 // blockWithPoWGET is the handler for the /blockpow [GET] endpoint.
@@ -155,6 +161,12 @@ func (api *API) blockPOST(w http.ResponseWriter, r *http.Request, _ httprouter.P
 		}
 	}
 
+	// Check whether the skylink is on the allow list
+	if api.staticSkydAPI.IsAllowListed(r.Context(), string(body.Skylink)) {
+		skyapi.WriteJSON(w, statusResponse{"reported"})
+		return
+	}
+
 	// Block the link.
 	err = api.block(r.Context(), body, sub, sub == "")
 	if errors.Contains(err, database.ErrSkylinkExists) {
@@ -165,7 +177,7 @@ func (api *API) blockPOST(w http.ResponseWriter, r *http.Request, _ httprouter.P
 		skyapi.WriteError(w, skyapi.Error{err.Error()}, http.StatusInternalServerError)
 		return
 	}
-	skyapi.WriteJSON(w, statusResponse{"blocked"})
+	skyapi.WriteJSON(w, statusResponse{"reported"})
 }
 
 // block blocks a skylink

blocker/blocker.go

@@ -1,23 +1,17 @@
 package blocker
 
 import (
-	"bytes"
 	"context"
-	"encoding/base64"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
 	"os"
 	"sort"
 	"strings"
 	"time"
 
 	"github.com/SkynetLabs/blocker/database"
+	"github.com/SkynetLabs/blocker/skyd"
 	"github.com/SkynetLabs/skynet-accounts/build"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/NebulousLabs/errors"
-	skyapi "gitlab.com/SkynetLabs/skyd/node/api"
 )
 
 const (
@@ -27,11 +21,6 @@ const (
 )
 
 var (
-	// ErrSkydOffline is returned if skyd is unreachable on startup.
-	ErrSkydOffline = errors.New("skyd is offline")
-
-	// skydTimeout is the timeout of the http calls to skyd in seconds
-	skydTimeout = "30"
 	// sleepBetweenScans defines how long the scanner should sleep after
 	// scanning the DB and not finding any skylinks to scan.
 	sleepBetweenScans = build.Select(
@@ -57,20 +46,17 @@ var (
 // Blocker scans the database for skylinks that should be blocked and calls
 // skyd to block them.
 type Blocker struct {
-	staticSkydHost        string
-	staticSkydPort        int
-	staticSkydAPIPassword string
-
 	staticNginxCachePurgerListPath string
 	staticNginxCachePurgeLockPath  string
 
-	staticCtx    context.Context
-	staticDB     *database.DB
-	staticLogger *logrus.Logger
+	staticCtx     context.Context
+	staticDB      *database.DB
+	staticLogger  *logrus.Logger
+	staticSkydAPI *skyd.SkydAPI
 }
 
 // New returns a new Blocker with the given parameters.
-func New(ctx context.Context, db *database.DB, logger *logrus.Logger, skydHost, skydPassword string, skydPort int, nginxCachePurgerListPath, nginxCachePurgeLockPath string) (*Blocker, error) {
+func New(ctx context.Context, skydAPI *skyd.SkydAPI, db *database.DB, logger *logrus.Logger, nginxCachePurgerListPath, nginxCachePurgeLockPath string) (*Blocker, error) {
 	if ctx == nil {
 		return nil, errors.New("invalid context provided")
 	}
@@ -80,54 +66,21 @@ func New(ctx context.Context, db *database.DB, logger *logrus.Logger, skydHost,
 	if logger == nil {
 		return nil, errors.New("invalid logger provided")
 	}
+	if skydAPI == nil {
+		return nil, errors.New("invalid Skyd API provided")
+	}
 	bl := &Blocker{
-		staticSkydHost:        skydHost,
-		staticSkydPort:        skydPort,
-		staticSkydAPIPassword: skydPassword,
-
 		staticNginxCachePurgerListPath: nginxCachePurgerListPath,
 		staticNginxCachePurgeLockPath:  nginxCachePurgeLockPath,
 
-		staticCtx:    ctx,
-		staticDB:     db,
-		staticLogger: logger,
-	}
-	if !bl.staticIsSkydUp() {
-		return nil, ErrSkydOffline
+		staticCtx:     ctx,
+		staticDB:      db,
+		staticLogger:  logger,
+		staticSkydAPI: skydAPI,
 	}
 	return bl, nil
 }
 
-// staticIsSkydUp connects to the local skyd and checks its status.
-// Returns true only if skyd is fully ready.
-func (bl *Blocker) staticIsSkydUp() bool {
-	status := struct {
-		Ready     bool
-		Consensus bool
-		Gateway   bool
-		Renter    bool
-	}{}
-	url := fmt.Sprintf("http://%s:%d/daemon/ready", bl.staticSkydHost, bl.staticSkydPort)
-	r, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		bl.staticLogger.Error(err)
-		return false
-	}
-	r.Header.Set("User-Agent", "Sia-Agent")
-	resp, err := http.DefaultClient.Do(r)
-	if err != nil {
-		bl.staticLogger.Warnf("Failed to query skyd: %s", err.Error())
-		return false
-	}
-	defer resp.Body.Close()
-	err = json.NewDecoder(resp.Body).Decode(&status)
-	if err != nil {
-		bl.staticLogger.Warnf("Bad body from skyd's /daemon/ready: %s", err.Error())
-		return false
-	}
-	return status.Ready && status.Consensus && status.Gateway && status.Renter
-}
-
 // SweepAndBlock sweeps the DB for new skylinks, blocks them in skyd and writes
 // down the timestamp of the latest one, so it will scan from that moment
 // further on its next sweep.
@@ -254,40 +207,10 @@ func (bl *Blocker) blockSkylinks(sls []string) error {
 	if err != nil {
 		bl.staticLogger.Warnf("Failed to write to nginx cache purger's list: %s", err)
 	}
-	// Build the call to skyd.
-	reqBody := skyapi.SkynetBlocklistPOST{
-		Add:    sls,
-		Remove: nil,
-		IsHash: false,
-	}
-	reqBodyBytes, err := json.Marshal(reqBody)
-	if err != nil {
-		return errors.AddContext(err, "failed to build request body")
-	}
 
-	url := fmt.Sprintf("http://%s:%d/skynet/blocklist?timeout=%s", bl.staticSkydHost, bl.staticSkydPort, skydTimeout)
-	bl.staticLogger.Debugf("blockSkylinks: POST on %+s", url)
-	req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(reqBodyBytes))
-	if err != nil {
-		return errors.AddContext(err, "failed to build request to skyd")
-	}
-	req.Header.Set("User-Agent", "Sia-Agent")
-	req.Header.Set("Authorization", bl.staticAuthHeader())
-	bl.staticLogger.Debugf("blockSkylinks: headers: %+v", req.Header)
-	resp, err := http.DefaultClient.Do(req)
+	err = bl.staticSkydAPI.BlockSkylinks(sls)
 	if err != nil {
-		return errors.AddContext(err, "failed to make request to skyd")
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
-		respBody, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			bl.staticLogger.Warn(errors.AddContext(err, "failed to parse response body after a failed call to skyd").Error())
-			respBody = []byte{}
-		}
-		err = errors.New(fmt.Sprintf("call to skyd failed with status '%s' and response '%s'", resp.Status, string(respBody)))
-		bl.staticLogger.Warn(err.Error())
-		return err
+		return errors.AddContext(err, "block skylinks failed")
 	}
 	return nil
 }
@@ -346,9 +269,3 @@ func (bl *Blocker) writeToNginxCachePurger(sls []string) error {
 	}
 	return nil
 }
-
-// staticAuthHeader returns the value we need to set to the `Authorization`
-// header in order to call `skyd`.
-func (bl *Blocker) staticAuthHeader() string {
-	return fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(":"+bl.staticSkydAPIPassword)))
-}