Merge pull request #10 from SkynetLabs/chris/pow-block

Add endpoint to allow for blocking without authentication by using a pow

.gitignore

@@ -1,2 +1,3 @@
 /.idea/
 /.env
+cover/

Makefile

@@ -11,6 +11,8 @@ racevars= history_size=3 halt_on_error=1 atexit_sleep_ms=2000
 # all will build and install release binaries
 all: release
 
+run = .
+
 # count says how many times to run the tests.
 count = 1
 # pkgs changes which packages the makefile calls operate on. run changes which
@@ -31,7 +33,6 @@ vet:
 # markdown-spellcheck runs codespell on all markdown files that are not
 # vendored.
 markdown-spellcheck:
-	pip install codespell 1>/dev/null 2>&1
 	git ls-files "*.md" :\!:"vendor/**" | xargs codespell --check-filenames
 
 # lint runs golangci-lint (which includes golint, a spellcheck of the codebase,
@@ -72,16 +73,14 @@ endef
 start-mongo:
 	-docker stop blocker-mongo-test-db 1>/dev/null 2>&1
 	-docker rm blocker-mongo-test-db 1>/dev/null 2>&1
-	chmod 400 $(shell pwd)/test/fixtures/mongo_keyfile
 	docker run \
      --rm \
      --detach \
      --name blocker-mongo-test-db \
      -p $(MONGO_PORT):$(MONGO_PORT) \
      -e MONGO_INITDB_ROOT_USERNAME=$(MONGO_USER) \
      -e MONGO_INITDB_ROOT_PASSWORD=$(MONGO_PASSWORD) \
-     -v $(shell pwd)/test/fixtures/mongo_keyfile:/data/mgkey \
-	mongo:4.4.1 mongod --port=$(MONGO_PORT) --replSet=skynet --keyFile=/data/mgkey 1>/dev/null 2>&1
+	mongo:4.4.1 mongod --port=$(MONGO_PORT) --replSet=skynet 1>/dev/null 2>&1
 	# wait for mongo to start before we try to configure it
 	status=1 ; while [[ $$status -gt 0 ]]; do \
 		sleep 1 ; \
@@ -123,15 +122,15 @@ bench: fmt
 	go test -tags='debug testing netgo' -timeout=500s -run=XXX -bench=. $(pkgs) -count=$(count)
 
 test:
-	go test -short -tags='debug testing netgo' -timeout=5s $(pkgs) -run=. -count=$(count)
+	go test -short -tags='debug testing netgo' -timeout=5s $(pkgs) -run=$(run) -count=$(count)
 
 test-long: lint lint-ci
 	@mkdir -p cover
-	GORACE='$(racevars)' go test -race --coverprofile='./cover/cover.out' -v -failfast -tags='testing debug netgo' -timeout=30s $(pkgs) -run=. -count=$(count)
+	GORACE='$(racevars)' go test -race --coverprofile='./cover/cover.out' -v -failfast -tags='testing debug netgo' -timeout=30s $(pkgs) -run=$(run) -count=$(count)
 
 # These env var values are for testing only. They can be freely changed.
 test-int: test-long start-mongo
-	GORACE='$(racevars)' go test -race -v -tags='testing debug netgo' -timeout=300s $(integration-pkgs) -run=. -count=$(count)
+	GORACE='$(racevars)' go test -race -v -tags='testing debug netgo' -timeout=300s $(integration-pkgs) -run=$(run) -count=$(count)
 	-make stop-mongo
 
 # test-single allows us to run a single integration test.

api/api.go

@@ -10,15 +10,6 @@ import (
 	"gitlab.com/NebulousLabs/errors"
 )
 
-var (
-	// SkydHost is where we connect to skyd
-	SkydHost = "sia"
-	// SkydPort is where we connect to skyd
-	SkydPort = 9980
-	// SkydAPIPassword is the API password for skyd
-	SkydAPIPassword string
-)
-
 // API is our central entry point to all subsystems relevant to serving requests.
 type API struct {
 	staticDB     *database.DB

api/handlers.go

@@ -1,12 +1,14 @@
 package api
 
 import (
+	"context"
+	"encoding/hex"
 	"encoding/json"
 	"net/http"
-	"net/url"
 	"regexp"
 	"time"
 
+	"github.com/SkynetLabs/blocker/blocker"
 	"github.com/SkynetLabs/blocker/database"
 	"github.com/julienschmidt/httprouter"
 	"gitlab.com/NebulousLabs/errors"
@@ -15,61 +17,134 @@ import (
 )
 
 type (
-	// BlockPOST ...
+	// BlockPOST describes a request to the /block endpoint.
 	BlockPOST struct {
-		Skylink  string            `json:"skylink"`
-		Reporter database.Reporter `json:"reporter"`
-		Tags     []string          `json:"tags"`
+		Skylink  skylink  `json:"skylink"`
+		Reporter Reporter `json:"reporter"`
+		Tags     []string `json:"tags"`
+	}
+
+	// BlockWithPoWPOST describes a request to the /blockpow endpoint
+	// containing a pow.
+	BlockWithPoWPOST struct {
+		BlockPOST
+		PoW blocker.BlockPoW `json:"pow"`
+	}
+
+	// BlockWithPoWGET is the response a user gets from the /blockpow
+	// endpoint.
+	BlockWithPoWGET struct {
+		Target string `json:"target"`
+	}
+
+	// Reporter is a person who reported that a given skylink should be
+	// blocked.
+	Reporter struct {
+		Name         string `json:"name"`
+		Email        string `json:"email"`
+		OtherContact string `json:"othercontact"`
 	}
 
 	// statusResponse is what we return on block requests
 	statusResponse struct {
 		Status string `json:"status"`
 	}
+
+	// skylink is a helper type which adds custom decoding for skylinks.
+	skylink string
 )
 
+// UnmarshalJSON implements json.Unmarshaler for a skylink.
+func (sl *skylink) UnmarshalJSON(b []byte) error {
+	var link string
+	err := json.Unmarshal(b, &link)
+	if err != nil {
+		return err
+	}
+	// Trim all the redundant information.
+	link, err = extractSkylinkHash(link)
+	if err != nil {
+		return err
+	}
+	// Normalise the skylink hash. We want to use the same hash encoding in the
+	// database, regardless of the encoding of the skylink when we receive it -
+	// base32 or base64.
+	var slNormalized skymodules.Skylink
+	err = slNormalized.LoadString(link)
+	if err != nil {
+		return errors.AddContext(err, "invalid skylink provided")
+	}
+	*sl = skylink(slNormalized.String())
+	return nil
+}
+
 // healthGET returns the status of the service
 func (api *API) healthGET(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 	status := struct {
 		DBAlive bool `json:"dbAlive"`
 	}{}
-	err := api.staticDB.Ping(r.Context())
+
+	// Apply a timeout.
+	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
+	defer cancel()
+
+	err := api.staticDB.Ping(ctx)
 	status.DBAlive = err == nil
 	skyapi.WriteJSON(w, status)
 }
 
-// blockPOST blocks a skylink
-func (api *API) blockPOST(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-	var body BlockPOST
-	err := json.NewDecoder(r.Body).Decode(&body)
+// blockWithPoWPOST blocks a skylink. It is meant to be used by untrusted sources such as
+// the abuse report skapp. The PoW prevents users from easily and anonymously
+// blocking large numbers of skylinks. Instead it encourages reuse of proofs
+// which improves the linkability between reports, thus allowing us to more
+// easily unblock a batch of links.
+func (api *API) blockWithPoWPOST(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	// Protect against large bodies.
+	b := http.MaxBytesReader(w, r.Body, 1<<16) // 64 kib
+	defer b.Close()
+
+	// Parse the request.
+	var body BlockWithPoWPOST
+	err := json.NewDecoder(b).Decode(&body)
 	if err != nil {
 		skyapi.WriteError(w, skyapi.Error{err.Error()}, http.StatusBadRequest)
 		return
 	}
-	body.Skylink, err = extractSkylinkHash(body.Skylink)
+
+	// Verify the pow.
+	err = body.PoW.Verify()
 	if err != nil {
-		skyapi.WriteError(w, skyapi.Error{errors.AddContext(err, "invalid skylink provided").Error()}, http.StatusBadRequest)
+		skyapi.WriteError(w, skyapi.Error{err.Error()}, http.StatusBadRequest)
 		return
 	}
-	// Normalise the skylink hash. We want to use the same hash encoding in the
-	// database, regardless of the encoding of the skylink when we receive it -
-	// base32 or base64.
-	var sl skymodules.Skylink
-	err = sl.LoadString(body.Skylink)
+
+	// Use the MySkyID as the suband make sure we don't consider the
+	// reporter authenticated.
+	sub := hex.EncodeToString(body.PoW.MySkyID[:])
+
+	// Block the link.
+	err = api.block(r.Context(), body.BlockPOST, sub, true)
 	if err != nil {
-		skyapi.WriteError(w, skyapi.Error{errors.AddContext(err, "invalid skylink provided").Error()}, http.StatusBadRequest)
-		return
-	}
-	body.Skylink = sl.String()
-	skylink := &database.BlockedSkylink{
-		Skylink:        body.Skylink,
-		Reporter:       body.Reporter,
-		Tags:           body.Tags,
-		TimestampAdded: time.Now().UTC(),
+		skyapi.WriteError(w, skyapi.Error{err.Error()}, http.StatusInternalServerError)
 	}
-	// Avoid nullpointer.
-	if r.Form == nil {
-		r.Form = url.Values{}
+	skyapi.WriteSuccess(w)
+}
+
+// blockWithPoWGET is the handler for the /blockpow [GET] endpoint.
+func (api *API) blockWithPoWGET(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	skyapi.WriteJSON(w, BlockWithPoWGET{
+		Target: hex.EncodeToString(blocker.MySkyTarget[:]),
+	})
+}
+
+// blockPOST blocks a skylink. It is meant to be used by trusted sources such as
+// the malware scanner or abuse email scanner.
+func (api *API) blockPOST(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	var body BlockPOST
+	err := json.NewDecoder(r.Body).Decode(&body)
+	if err != nil {
+		skyapi.WriteError(w, skyapi.Error{err.Error()}, http.StatusBadRequest)
+		return
 	}
 	sub := r.Form.Get("sub")
 	if sub == "" {
@@ -79,10 +154,9 @@ func (api *API) blockPOST(w http.ResponseWriter, r *http.Request, _ httprouter.P
 			sub = u.Sub
 		}
 	}
-	skylink.Reporter.Sub = sub
-	skylink.Reporter.Unauthenticated = sub == ""
-	api.staticLogger.Tracef("blockPOST will block skylink %s", skylink.Skylink)
-	err = api.staticDB.BlockedSkylinkCreate(r.Context(), skylink)
+
+	// Block the link.
+	err = api.block(r.Context(), body, sub, sub == "")
 	if errors.Contains(err, database.ErrSkylinkExists) {
 		skyapi.WriteJSON(w, statusResponse{"duplicate"})
 		return
@@ -91,10 +165,32 @@ func (api *API) blockPOST(w http.ResponseWriter, r *http.Request, _ httprouter.P
 		skyapi.WriteError(w, skyapi.Error{err.Error()}, http.StatusInternalServerError)
 		return
 	}
-	api.staticLogger.Debugf("Added skylink %s", skylink.Skylink)
 	skyapi.WriteJSON(w, statusResponse{"blocked"})
 }
 
+// block blocks a skylink
+func (api *API) block(ctx context.Context, bp BlockPOST, sub string, unauthenticated bool) error {
+	skylink := &database.BlockedSkylink{
+		Skylink: string(bp.Skylink),
+		Reporter: database.Reporter{
+			Name:            bp.Reporter.Name,
+			Email:           bp.Reporter.Email,
+			OtherContact:    bp.Reporter.OtherContact,
+			Sub:             sub,
+			Unauthenticated: unauthenticated,
+		},
+		Tags:           bp.Tags,
+		TimestampAdded: time.Now().UTC(),
+	}
+	api.staticLogger.Tracef("blockPOST will block skylink %s", skylink.Skylink)
+	err := api.staticDB.CreateBlockedSkylink(ctx, skylink)
+	if err != nil {
+		return err
+	}
+	api.staticLogger.Debugf("Added skylink %s", skylink.Skylink)
+	return nil
+}
+
 // extractSkylinkHash extracts the skylink hash from the given skylink that
 // might have protocol, path, etc. within it.
 func extractSkylinkHash(skylink string) (string, error) {

api/handlers_test.go

@@ -0,0 +1,23 @@
+package api
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// TestVerifySkappReport verifies a report directly generated from the abuse
+// skapp.
+func TestVerifySkappReport(t *testing.T) {
+	report := `{"reporter":{"name":"PJ","email":"pj@siasky.net"},"skylink":"https://siasky.dev/_AL4LxntE4LN3WVTtvSMad3t1QGZ8c0n1bct2zfju2H_HQ","tags":["childabuse"],"pow":{"version":"MySkyID-PoW-v1","nonce":"6128653","myskyid":"a913af653d148f905f481c28fc813b6940d24e9534abceabbc0c456b0fff6cf5","signature":"d48dd2ed9227044f22aab2034973c1967722b9f50e22bf07340829a89487a764d748dc9a3640a08d7ed420a442986c24ab3fdc4cb7b959901556cf9ee87b650b"}}`
+
+	var bp BlockWithPoWPOST
+	err := json.Unmarshal([]byte(report), &bp)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = bp.PoW.Verify()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

api/routes.go

@@ -24,18 +24,9 @@ var (
 // buildHTTPRoutes registers all HTTP routes and their handlers.
 func (api *API) buildHTTPRoutes() {
 	api.staticRouter.GET("/health", api.healthGET)
-	api.staticRouter.POST("/block", api.addCORSHeader(api.blockPOST))
-}
-
-// addCORSHeader sets the CORS headers.
-func (api *API) addCORSHeader(h httprouter.Handle) httprouter.Handle {
-	return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
-		w.Header().Set("Content-Type", "text/html; charset=utf-8")
-		w.Header().Set("Access-Control-Allow-Origin", "https://0404guluqu38oaqapku91ed11kbhkge55smh9lhjukmlrj37lfpm8no.siasky.net")
-		w.Header().Set("Access-Control-Allow-Headers", "Content-Type,access-control-allow-origin, access-control-allow-headers")
-		w.Header().Set("Access-Control-Allow-Credentials", "true")
-		h(w, req, ps)
-	}
+	api.staticRouter.POST("/block", api.blockPOST)
+	api.staticRouter.GET("/powblock", api.blockWithPoWGET)
+	api.staticRouter.POST("/powblock", api.blockWithPoWPOST)
 }
 
 // validateCookie extracts the cookie from the incoming blocking request and