Allow List

README.md

@@ -6,6 +6,20 @@ The service exposes a REST API that allows callers to request the blocking of ne
 
 The blocklist is shared between the servers that make up a portal cluster via MongoDB.
 
+# AllowList
+
+The blocker service can only block skylinks which are not in the allow list.
+To add a skylink to the allow list, one has to manually query the database and
+perform the follow operation:
+
+```
+db.getCollection('allowlist').insertOne({
+  skylink: "[INSERT V1 SKYLINK HERE]",
+  description: "[INSERT SKYLINK DESCRIPTION]",
+  timestamp_added: new Date(),
+})
+```
+
 # Environment
 
 This service depends on the following environment variables:

api/api.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"encoding/base64"
 	"fmt"
 	"net/http"
 
@@ -57,3 +58,9 @@ func (api *API) ListenAndServe(port int) error {
 func (api *API) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	api.staticRouter.ServeHTTP(w, req)
 }
+
+// AuthHeader returns the value we need to set to the `Authorization` header in
+// order to call `skyd`.
+func AuthHeader() string {
+	return fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(":"+SkydAPIPassword)))
+}

api/handlers.go

@@ -1,7 +1,9 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -14,6 +16,12 @@ import (
 	"gitlab.com/SkynetLabs/skyd/skymodules"
 )
 
+var (
+	// ErrSkylinkAllowListed is returned when we try to add a skylink to the
+	// database that is part of the allow list.
+	ErrSkylinkAllowListed = errors.New("skylink can not be blocked, it is allow listed")
+)
+
 type (
 	// BlockPOST ...
 	BlockPOST struct {
@@ -60,6 +68,13 @@ func (api *API) blockPOST(w http.ResponseWriter, r *http.Request, _ httprouter.P
 		skyapi.WriteError(w, skyapi.Error{errors.AddContext(err, "invalid skylink provided").Error()}, http.StatusBadRequest)
 		return
 	}
+
+	// Check whether the skylink is on the allow list
+	if api.staticIsAllowListed(r.Context(), body.Skylink) {
+		skyapi.WriteError(w, skyapi.Error{ErrSkylinkAllowListed.Error()}, http.StatusBadRequest)
+		return
+	}
+
 	body.Skylink = sl.String()
 	skylink := &database.BlockedSkylink{
 		Skylink:        body.Skylink,
@@ -108,3 +123,59 @@ func extractSkylinkHash(skylink string) (string, error) {
 	}
 	return m[2], nil
 }
+
+// staticIsAllowListed will resolve the given skylink and verify it against the
+// allow list, it returns true if the skylink is present on the allow list
+func (api *API) staticIsAllowListed(ctx context.Context, skylink string) bool {
+	// build the request to resolve the skylink with skyd
+	url := fmt.Sprintf("http://%s:%d/skynet/resolve/%s", SkydHost, SkydPort, skylink)
+	api.staticLogger.Debugf("isAllowListed: GET on %+s", url)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		api.staticLogger.Error("failed to build request to skyd", err)
+		return false
+	}
+
+	// set headers and execute the request
+	req.Header.Set("User-Agent", "Sia-Agent")
+	req.Header.Set("Authorization", AuthHeader())
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		api.staticLogger.Error("failed to make request to skyd", err)
+		return false
+	}
+	defer resp.Body.Close()
+
+	// if the skylink was blocked it was not allow listed
+	if resp.StatusCode == http.StatusUnavailableForLegalReasons {
+		return false
+	}
+
+	// if the status code indicates it was a bad request, check the allow list
+	// for the given skylink
+	if resp.StatusCode == http.StatusBadRequest {
+		allowlisted, err := api.staticDB.IsAllowListed(ctx, skylink)
+		if err != nil {
+			api.staticLogger.Error("failed to verify skylink against the allow list", err)
+			return false
+		}
+		return allowlisted
+	}
+
+	// in all other cases use the resolved skylink when checking the allow list
+	resolved := struct {
+		Skylink string
+	}{}
+	err = json.NewDecoder(resp.Body).Decode(&resolved)
+	if err != nil {
+		api.staticLogger.Error("bad response body from skyd", err)
+		return false
+	}
+
+	allowlisted, err := api.staticDB.IsAllowListed(ctx, resolved.Skylink)
+	if err != nil {
+		api.staticLogger.Error("failed to verify skylink against the allow list", err)
+		return false
+	}
+	return allowlisted
+}

blocker/blocker.go

@@ -3,7 +3,6 @@ package blocker
 import (
 	"bytes"
 	"context"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -240,7 +239,7 @@ func (bl *Blocker) blockSkylinks(sls []string) error {
 		return errors.AddContext(err, "failed to build request to skyd")
 	}
 	req.Header.Set("User-Agent", "Sia-Agent")
-	req.Header.Set("Authorization", authHeader())
+	req.Header.Set("Authorization", api.AuthHeader())
 	bl.staticLogger.Debugf("blockSkylinks: headers: %+v", req.Header)
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
@@ -314,9 +313,3 @@ func (bl *Blocker) writeToNginxCachePurger(sls []string) error {
 	}
 	return nil
 }
-
-// authHeader returns the value we need to set to the `Authorization` header in
-// order to call `skyd`.
-func authHeader() string {
-	return fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(":"+api.SkydAPIPassword)))
-}

database/database.go

@@ -25,6 +25,10 @@ var (
 	// and it already exists there.
 	ErrSkylinkExists = errors.New("skylink already exists")
 
+	// mongoErrNoDocuments is returned when a database operation completes
+	// successfully but it doesn't find or affect any documents.
+	mongoErrNoDocuments = errors.New("no documents in result")
+
 	// Portal is the preferred portal to use, e.g. https://siasky.net
 	Portal string
 	// ServerDomain is the unique server name, e.g. eu-pol-4.siasky.net
@@ -37,17 +41,20 @@ var (
 	dbName = "blocker"
 	// dbSkylinks defines the name of the skylinks collection
 	dbSkylinks = "skylinks"
+	// dbAllowList defines the name of the allowlist collection
+	dbAllowList = "allowlist"
 	// dbLatestBlockTimestamps dbLatestBlockTimestamps
 	dbLatestBlockTimestamps = "latest_block_timestamps"
 )
 
 // DB holds a connection to the database, as well as helpful shortcuts to
 // collections and utilities.
 type DB struct {
-	Ctx      context.Context
-	DB       *mongo.Database
-	Skylinks *mongo.Collection
-	Logger   *logrus.Logger
+	Ctx       context.Context
+	DB        *mongo.Database
+	Skylinks  *mongo.Collection
+	AllowList *mongo.Collection
+	Logger    *logrus.Logger
 }
 
 // New creates a new database connection.
@@ -77,10 +84,11 @@ func NewCustomDB(ctx context.Context, dbName string, creds database.DBCredential
 		return nil, err
 	}
 	return &DB{
-		Ctx:      ctx,
-		DB:       db,
-		Skylinks: db.Collection(dbSkylinks),
-		Logger:   logger,
+		Ctx:       ctx,
+		DB:        db,
+		Skylinks:  db.Collection(dbSkylinks),
+		AllowList: db.Collection(dbAllowList),
+		Logger:    logger,
 	}, nil
 }
 
@@ -122,9 +130,10 @@ func (db *DB) BlockedSkylinkByID(ctx context.Context, id primitive.ObjectID) (*B
 	return &sl, nil
 }
 
-// BlockedSkylinkCreate creates a new skylink. If the skylink already exists it does
-// nothing.
+// BlockedSkylinkCreate creates a new skylink. If the skylink already exists it
+// does nothing.
 func (db *DB) BlockedSkylinkCreate(ctx context.Context, skylink *BlockedSkylink) error {
+	// Try and insert the skylink
 	_, err := db.Skylinks.InsertOne(ctx, skylink)
 	if err != nil && strings.Contains(err.Error(), "E11000 duplicate key error collection") {
 		db.Logger.Debugf("BlockedSkylinkCreate: duplicate key, returning '%s'", ErrSkylinkExists.Error())
@@ -216,6 +225,18 @@ func (db *DB) SetLatestBlockTimestamp(t time.Time) error {
 	return nil
 }
 
+// IsAllowListed returns whether the given skylink is on the allow list.
+func (db *DB) IsAllowListed(ctx context.Context, skylink string) (bool, error) {
+	res := db.AllowList.FindOne(ctx, bson.M{"skylink": skylink})
+	if isDocumentNotFound(res.Err()) {
+		return false, nil
+	}
+	if res.Err() != nil {
+		return false, res.Err()
+	}
+	return true, nil
+}
+
 // connectionString is a helper that returns a valid MongoDB connection string
 // based on the passed credentials and a set of constants. The connection string
 // is using the standalone approach because the service is supposed to talk to
@@ -245,6 +266,16 @@ func ensureDBSchema(ctx context.Context, db *mongo.Database, log *logrus.Logger)
 	// schema defines a mapping between a collection name and the indexes that
 	// must exist for that collection.
 	schema := map[string][]mongo.IndexModel{
+		dbAllowList: {
+			{
+				Keys:    bson.D{{"skylink", 1}},
+				Options: options.Index().SetName("skylink").SetUnique(true),
+			},
+			{
+				Keys:    bson.D{{"timestamp_added", 1}},
+				Options: options.Index().SetName("timestamp_added"),
+			},
+		},
 		dbSkylinks: {
 			{
 				Keys:    bson.D{{"skylink", 1}},
@@ -295,3 +326,12 @@ func ensureCollection(ctx context.Context, db *mongo.Database, collName string)
 	}
 	return coll, nil
 }
+
+// isDocumentNotFound is a helper function that returns whether the given error
+// contains the mongo documents not found error message.
+func isDocumentNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), mongoErrNoDocuments.Error())
+}

database/skylink.go

@@ -6,6 +6,15 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
+// AllowListedSkylink is a skylink that is allow listed and thus prohibited from
+// ever being blocked.
+type AllowListedSkylink struct {
+	ID             primitive.ObjectID `bson:"_id,omitempty"`
+	Skylink        string             `bson:"skylink"`
+	Description    string             `bson:"description"`
+	TimestampAdded time.Time          `bson:"timestamp_added"`
+}
+
 // BlockedSkylink is a skylink blocked by an external request.
 type BlockedSkylink struct {
 	ID                primitive.ObjectID `bson:"_id,omitempty" json:"-"`