Merge pull request ethereum#402 from aquiladev/swarm-network-rewrite

swarm: adds handling GET robots.txt ethereum#154 (our default is to disallow crawling, but this can be overwritten using nginx config)

swarm/api/http/server.go

@@ -850,6 +850,12 @@ func (s *Server) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if r.URL.Path == "/robots.txt" {
+		w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
+		fmt.Fprintf(w, "User-agent: *\nDisallow: /")
+		return
+	}
+
 	uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
 	if err != nil {
 		Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)