flesh out the web package and transition to worker abstraction

.gitignore

@@ -3,4 +3,7 @@ shotis-node
 shotis-*.json
 .DS_Store
 *.png
-*.mov
+*.mov
+*.crt
+*.csr
+*.key

cmd/http.go

@@ -1,16 +1,11 @@
 package cmd
 
 import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"io"
+	"context"
 	"log"
-	"os"
 
-	"github.com/shotis/shotis-node/network"
+	"github.com/shotis/shotis-node/web"
 	"github.com/spf13/cobra"
-	"google.golang.org/grpc"
 )
 
 // httpCmd represents the http command
@@ -19,52 +14,13 @@ var httpCmd = &cobra.Command{
 	Short: "Runs an HTTP API server for Shotis",
 	Long:  ``,
 	Run: func(cmd *cobra.Command, args []string) {
-		f, _ := os.Open(args[0])
-
-		conn, err := grpc.Dial(":1337", grpc.WithInsecure())
+		ws, err := web.Init(Config)
 
 		if err != nil {
 			log.Fatalln(err)
 		}
 
-		shotisClient := network.NewShotisServiceClient(conn)
-
-		stream, err := shotisClient.UploadImage(cmd.Context())
-
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		br := bufio.NewReader(f)
-
-		buffer := make([]byte, 512)
-
-		header := &network.FileHeader{
-			FileName: f.Name(),
-			FileType: "whatever",
-		}
-
-		totalRead := 0
-
-		for read, err := br.Read(buffer); read != -1 && err != io.EOF; {
-			stream.Send(&network.UploadImageMessage{
-				Header: header,
-				Data:   buffer[:read],
-			})
-
-			totalRead += read
-			read, err = br.Read(buffer)
-		}
-
-		response, err := stream.CloseAndRecv()
-
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		j, _ := json.MarshalIndent(response, "", " ")
-
-		fmt.Println(string(j))
+		ws.Start(context.Background())
 	},
 }
 

cmd/root.go

@@ -1,17 +1,18 @@
 package cmd
 
 import (
-	"fmt"
-	"os"
+	"log"
 
+	"github.com/shotis/shotis-node/config"
 	"github.com/spf13/cobra"
 
-	homedir "github.com/mitchellh/go-homedir"
 	"github.com/spf13/viper"
 )
 
 var cfgFile string
 
+var Config *config.NodeConfig
+
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
 	Use:   "shotis-node",
@@ -44,28 +45,21 @@ func init() {
 	// Cobra supports persistent flags, which, if defined here,
 	// will be global for your application.
 
-	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.cmd.yaml)")
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "config.json", "Path to the configuration file")
 }
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
 	if cfgFile != "" {
 		// Use config file from the flag.
 		viper.SetConfigFile(cfgFile)
-	} else {
-		// Find home directory.
-		home, err := homedir.Dir()
-		cobra.CheckErr(err)
-
-		// Search config in home directory with name ".cmd" (without extension).
-		viper.AddConfigPath(home)
-		viper.SetConfigName(".cmd")
 	}
 
-	viper.AutomaticEnv() // read in environment variables that match
+	conf, err := config.ReadConfig(cfgFile)
 
-	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err == nil {
-		fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
+	if err != nil {
+		log.Fatalln(err)
 	}
+
+	Config = conf
 }

cmd/rpc.go

@@ -2,35 +2,57 @@ package cmd
 
 import (
 	"context"
-	"fmt"
+	"crypto/tls"
 	"log"
 	"net"
 
 	"github.com/shotis/shotis-node/network"
 	"github.com/shotis/shotis-node/storage"
+	"github.com/shotis/shotis-node/tasks"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
 )
 
+type GRPCWorker struct {
+	Backing       *network.ServerImpl
+	AwaitingTasks *tasks.Queue
+}
+
 // rpcCmd represents the rpc command
 var rpcCmd = &cobra.Command{
 	Use:   "rpc",
 	Short: "Runs an gRPC server on the Shotis network",
 	Long:  ``,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("rpc called")
 
-		//TODO: Implement a more robust server with go routines & worker pools and actually read from a config file
-		//       etc. etc.
-		server := grpc.NewServer()
+		log.Println("starting gRPC server on", Config.Server.RPC.Host)
+
+		cert, err := tls.LoadX509KeyPair(Config.Server.TLS.CertPath, Config.Server.TLS.KeyPath)
+
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		server := grpc.NewServer(grpc.Creds(credentials.NewServerTLSFromCert(&cert)))
+		service, err := storage.NewGoogleCloudStorage(context.Background(), Config.Storage.AuthKey, storage.Bucket(Config.Storage.Bucket))
+
+		if err != nil {
+			log.Println(err)
+		} else {
+			log.Println("created Google Cloud Storage server")
+		}
 
-		service, _ := storage.NewGoogleCloudStorage(context.Background(), "key", storage.Bucket("bucket"))
+		worker := &GRPCWorker{
+			Backing: &network.ServerImpl{
+				GCPService: service,
+			},
+			AwaitingTasks: tasks.NewQueue(100),
+		}
 
-		network.RegisterShotisServiceServer(server, &network.ServerImpl{
-			GCPService: service,
-		})
+		network.RegisterShotisServiceServer(server, worker.Backing)
 
-		l, err := net.Listen("tcp", ":1337")
+		l, err := net.Listen("tcp", Config.Server.RPC.Host)
 
 		if err != nil {
 			log.Fatalln(err)

config.json

@@ -1,16 +1,21 @@
 {
-    "peers": [
-        "grpc://192.168.0.1:1337"
-    ],
-    "grpc": {
-        "tlsEnabled": false,
+    "rpc":"",
+    "server": {
         "tls": {
-            "key": "some_key.key",
-            "cert": "some_cert.cert"
-        }
+            "enabled": false,
+            "key": "shotis.key",
+            "cert": "shotis.crt"
+        },
+        "web": {
+            "maxStreams": 250
+        },
+        "rpc":{
+            "host": ":1337"
+        },
+        "host": ":443"
     },
     "storage": {
-        "bucket": "",
-        "key": ""
+        "bucket": "shot_is_image_1",
+        "authKey": "shotis-2ab802400c4e.json"
     }
 }

config/config.go

@@ -0,0 +1,56 @@
+package config
+
+import (
+	"encoding/json"
+	"io/ioutil"
+)
+
+type NodeConfig struct {
+	Server *ServerConfig `json:"server"`
+	// The storage
+	Storage *StorageConfig `json:"storage"`
+}
+
+type RPCConfig struct {
+	Host string
+}
+
+type TLSConfig struct {
+	Enabled  bool   `json:"enabled"`
+	KeyPath  string `json:"key"`
+	CertPath string `json:"cert"`
+}
+
+type ServerConfig struct {
+	// The RPC server that should be connected to
+	RPC  *RPCConfig `json:"rpc"`
+	TLS  *TLSConfig `json:"tls"`
+	Web  *WebConfig
+	Host string `json:"host"`
+}
+
+type StorageConfig struct {
+	Bucket  string `json:"bucket"`
+	AuthKey string `json:"authKey"`
+}
+
+type WebConfig struct {
+	MaxStreams int `json:"maxStreams"`
+}
+
+func ReadConfig(configPath string) (*NodeConfig, error) {
+	var conf NodeConfig
+
+	confFile, err := ioutil.ReadFile(configPath)
+
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(confFile, &conf)
+	if err != nil {
+		return nil, err
+	}
+
+	return &conf, nil
+}

crypto/pem.go

@@ -0,0 +1,26 @@
+package crypto
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"io/ioutil"
+)
+
+func SingleCertificatePool(certFile string) (*x509.CertPool, error) {
+	pemCert, err := ioutil.ReadFile(certFile)
+
+	if err != nil {
+		return nil, err
+	}
+
+	pemBlock, _ := pem.Decode(pemCert)
+	cert, err := x509.ParseCertificate(pemBlock.Bytes)
+
+	if err != nil {
+		return nil, err
+	}
+
+	certPool := x509.NewCertPool()
+	certPool.AddCert(cert)
+	return certPool, nil
+}

go.mod

@@ -7,7 +7,7 @@ require (
 	cloud.google.com/go/storage v1.16.0
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/uuid v1.2.0
-	github.com/mitchellh/go-homedir v1.1.0
+	github.com/labstack/echo/v4 v4.3.0
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/viper v1.8.1
 	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect