Small refactor, mostly logs and cli flags

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/alecthomas/kong v0.8.1
-	github.com/cockroachdb/pebble v0.0.0-20231009150004-a678d0968383
+	github.com/cockroachdb/pebble v0.0.0-20231011191824-ede31f1a8e4b
 	github.com/fxamacker/cbor/v2 v2.5.0
 	github.com/gin-contrib/cors v1.4.0
 	github.com/gin-contrib/graceful v0.0.0-20230904152018-e56ed94cd808
@@ -28,7 +28,7 @@ require (
 require (
 	github.com/DataDog/zstd v1.5.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.9.0 // indirect
+	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/bytedance/sonic v1.10.2 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
@@ -41,7 +41,7 @@ require (
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/getsentry/sentry-go v0.25.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect

go.sum

@@ -28,8 +28,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bits-and-blooms/bitset v1.2.2-0.20220111210104-dfa3e347c392/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/bits-and-blooms/bitset v1.9.0 h1:g1YivPG8jOtrN013Fe8OBXubkiTwvm7/vG2vXz03ANU=
-github.com/bits-and-blooms/bitset v1.9.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88=
+github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
@@ -52,8 +52,8 @@ github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZ
 github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20231009150004-a678d0968383 h1:98LmGECk4ksnbvBVr0DgssnH0Mfwkd6+L6D/JHVd144=
-github.com/cockroachdb/pebble v0.0.0-20231009150004-a678d0968383/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
+github.com/cockroachdb/pebble v0.0.0-20231011191824-ede31f1a8e4b h1:/fElHApeWG4Btj37APrPuilBce1sinPUEjEXCFs8Tds=
+github.com/cockroachdb/pebble v0.0.0-20231011191824-ede31f1a8e4b/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
@@ -73,8 +73,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE=
 github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
-github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
-github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
 github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI=
 github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
 github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g=

src/bucket/bucket.go

@@ -76,11 +76,10 @@ func SetResultResponse(link string, response *colly.Response, relay *Relay, seNa
 	if !exists {
 		relay.Mutex.Unlock()
 		log.Error().Msgf("URL not in map when adding response! Should not be possible. URL: %v", link)
-		return
+	} else {
+		mapRes.Response = response
+		relay.Mutex.Unlock()
 	}
-
-	mapRes.Response = response
-	relay.Mutex.Unlock()
 }
 
 func MakeSEResult(urll string, title string, description string, searchEngineName engines.Name, sePage int, seOnPageRank int) *engines.RetrievedResult {

src/cache/pebble/pebble.go

@@ -18,7 +18,7 @@ func New(path string) *DB {
 	pdb, err := pebble.Open(pebblePath, &pebble.Options{})
 
 	if err != nil {
-		log.Fatal().Msgf("Error opening pebble: %v (path: %v)", err, pebblePath)
+		log.Fatal().Err(err).Msgf("Error opening pebble at path: %v", pebblePath)
 	} else {
 		log.Info().Msgf("Successfully opened pebble (path: %v)", pebblePath)
 	}
@@ -28,7 +28,7 @@ func New(path string) *DB {
 
 func (db *DB) Close() {
 	if err := db.pdb.Close(); err != nil {
-		log.Fatal().Msgf("Error closing pebble: %v", err)
+		log.Fatal().Err(err).Msg("Error closing pebble")
 	} else {
 		log.Debug().Msg("Successfully closed pebble")
 	}
@@ -39,11 +39,12 @@ func (db *DB) Set(k string, v cache.Value) {
 	cacheTimer := time.Now()
 
 	if val, err := cbor.Marshal(v); err != nil {
-		log.Error().Msgf("Error marshaling value: %v", err)
+		log.Error().Err(err).Msg("Error marshaling value")
 	} else if err := db.pdb.Set([]byte(k), val, pebble.NoSync); err != nil {
-		log.Fatal().Msgf("Error setting KV to pebble: %v", err)
+		log.Fatal().Err(err).Msg("Error setting KV to pebble")
 	} else {
-		log.Debug().Msgf("Cached results in %vns", time.Since(cacheTimer).Nanoseconds())
+		cacheTimeSince := time.Since(cacheTimer)
+		log.Debug().Msgf("Cached results in %vms (%vns)", cacheTimeSince.Milliseconds(), cacheTimeSince.Nanoseconds())
 	}
 }
 
@@ -52,12 +53,12 @@ func (db *DB) Get(k string, o cache.Value) {
 	val := []byte(v) // copy data before closing, casting needed for unmarshal
 
 	if err == pebble.ErrNotFound {
-		log.Trace().Msgf("Found no value in pebble for key (%v): %v", k, err)
+		log.Trace().Msgf("Found no value in pebble for key %v", k)
 	} else if err != nil {
-		log.Fatal().Msgf("Error getting value from pebble for key (%v): %v", k, err)
+		log.Fatal().Err(err).Msgf("Error getting value from pebble for key %v", k)
 	} else if err := c.Close(); err != nil {
-		log.Fatal().Msgf("Error closing io to pebble for key (%v): %v", k, err)
+		log.Fatal().Err(err).Msgf("Error closing io to pebble for key %v", k)
 	} else if err := cbor.Unmarshal(val, o); err != nil {
-		log.Error().Msgf("Failed unmarshaling value from pebble for key (%v): %v", k, err)
+		log.Error().Err(err).Msgf("Failed unmarshaling value from pebble for key %v", k)
 	}
 }

src/cache/redis/redis.go

@@ -25,7 +25,7 @@ func New(ctx context.Context, config config.Redis) *DB {
 	})
 
 	if err := rdb.Ping(ctx).Err(); err != nil {
-		log.Fatal().Msgf("Error connecting to redis (addr: %v:%v/%v): %v", config.Host, config.Port, config.Database, err)
+		log.Fatal().Err(err).Msgf("Error connecting to redis with addr: %v:%v/%v", config.Host, config.Port, config.Database)
 	} else {
 		log.Info().Msgf("Successful connection to redis (addr: %v:%v/%v)", config.Host, config.Port, config.Database)
 	}
@@ -35,7 +35,7 @@ func New(ctx context.Context, config config.Redis) *DB {
 
 func (db *DB) Close() {
 	if err := db.rdb.Close(); err != nil {
-		log.Fatal().Msgf("Error disconnecting from redis: %v", err)
+		log.Fatal().Err(err).Msg("Error disconnecting from redis")
 	} else {
 		log.Debug().Msg("Successfully disconnected from redis")
 	}
@@ -46,11 +46,12 @@ func (db *DB) Set(k string, v cache.Value) {
 	cacheTimer := time.Now()
 
 	if val, err := cbor.Marshal(v); err != nil {
-		log.Error().Msgf("Error marshaling value: %v", err)
+		log.Error().Err(err).Msg("Error marshaling value")
 	} else if err := db.rdb.Set(db.ctx, k, val, 0).Err(); err != nil {
-		log.Fatal().Msgf("Error setting KV to redis: %v", err)
+		log.Fatal().Err(err).Msg("Error setting KV to redis")
 	} else {
-		log.Debug().Msgf("Cached results in %vns", time.Since(cacheTimer).Nanoseconds())
+		cacheTimeSince := time.Since(cacheTimer)
+		log.Debug().Msgf("Cached results in %vms (%vns)", cacheTimeSince.Milliseconds(), cacheTimeSince.Nanoseconds())
 	}
 }
 
@@ -59,10 +60,10 @@ func (db *DB) Get(k string, o cache.Value) {
 	val := []byte(v) // copy data before closing, casting needed for unmarshal
 
 	if err == redis.Nil {
-		log.Trace().Msgf("Found no value in redis for key (%v): %v", k, err)
+		log.Trace().Msgf("Found no value in redis for key %v", k)
 	} else if err != nil {
-		log.Fatal().Msgf("Error getting value from redis for key (%v): %v", k, err)
+		log.Fatal().Err(err).Msgf("Error getting value from redis for key %v", k)
 	} else if err := cbor.Unmarshal(val, o); err != nil {
-		log.Error().Msgf("Failed unmarshaling value from redis for key (%v): %v", k, err)
+		log.Error().Err(err).Msgf("Failed unmarshaling value from redis for key %v", k)
 	}
 }

src/category/category.go

@@ -4,7 +4,7 @@ import (
 	"strings"
 )
 
-var FromString map[string]Name = map[string]Name{
+var FromString = map[string]Name{
 	//main
 	"general": GENERAL,
 	"info":    INFO,

src/cli/setup.go

@@ -19,14 +19,14 @@ func Setup() Flags {
 		}),
 		kong.Vars{
 			"version":      fmt.Sprintf("%v (%v@%v)", Version, GitCommit, Timestamp),
-			"config_path":  ".",
-			"log_path":     "./log",
+			"data_folder":  ".",
+			"log_folder":   "./log",
 			"query_string": "banana death",
 		},
 	)
 
 	if err := ctx.Validate(); err != nil {
-		log.Fatal().Err(err).Msg("Failed parsing cli")
+		log.Panic().Err(err).Msg("Failed parsing cli")
 	}
 
 	return cli

src/cli/structs.go

@@ -15,8 +15,8 @@ type Flags struct {
 	Cli       bool   `type:"bool" default:"false" env:"BRZAGUZA_CLI" help:"Use CLI mode"`
 	Visit     bool   `type:"bool" default:"false" env:"BRZAGUZA_VISIT" help:"Should results be visited"`
 	Silent    bool   `type:"bool" default:"false" short:"s" env:"BRZAGUZA_SILENT" help:"Should results be printed"`
-	Config    string `type:"path" default:"${config_path}" env:"BRZAGUZA_CONFIG" help:"Config folder path"`
-	Log       string `type:"path" default:"${log_path}" env:"BRZAGUZA_LOG" help:"Log file path"`
+	Data      string `type:"path" default:"${data_folder}" env:"BRZAGUZA_DATA" help:"Data folder path"`
+	Log       string `type:"path" default:"${log_folder}" env:"BRZAGUZA_LOG" help:"Log folder path"`
 	Verbosity int8   `type:"counter" default:"0" short:"v" env:"BRZAGUZA_VERBOSITY" help:"Log level verbosity"`
 	Category  string `type:"string" default:"" short:"c" env:"BRZAGUZA_CATEGORY" help:"Search result category. Can also be supplied through the query (e.g. \"!info smartphone\"). Supported values: info[/wiki], science[/sci], news, blog, surf, newnews[/nnews]"`
 	// profiler

src/config/load.go

@@ -92,7 +92,7 @@ func (c *Config) Load(path string, logPath string) {
 	// We provide a struct along with the struct tag `koanf` to the
 	// provider.
 	if err := k.Load(structs.Provider(&rc, "koanf"), nil); err != nil {
-		log.Fatal().Err(err).Msg("failed loading default values")
+		log.Panic().Err(err).Msg("failed loading default values")
 	}
 
 	// Load YAML config
@@ -103,22 +103,22 @@ func (c *Config) Load(path string, logPath string) {
 		if _, errr := os.Stat(yamlPath); errr != nil {
 			log.Trace().Msgf("no yaml config present at path: %v", yamlPath)
 		} else if errr := k.Load(file.Provider(yamlPath), yaml.Parser()); errr != nil {
-			log.Panic().Msgf("error loading yaml config: %v", err)
+			log.Panic().Err(err).Msg("error loading yaml config")
 		}
 	} else if err := k.Load(file.Provider(yamlPath), yaml.Parser()); err != nil {
-		log.Panic().Msgf("error loading yaml config: %v", err)
+		log.Panic().Err(err).Msg("error loading yaml config")
 	}
 
 	// Load ENV config
 	if err := k.Load(env.Provider("BRZAGUZA_", ".", func(s string) string {
 		return strings.Replace(strings.ToLower(strings.TrimPrefix(s, "BRZAGUZA_")), "_", ".", -1)
 	}), nil); err != nil {
-		log.Panic().Msgf("error loading env config: %v", err)
+		log.Panic().Err(err).Msg("error loading env config")
 	}
 
 	// Unmarshal config into struct
 	if err := k.Unmarshal("", &rc); err != nil {
-		log.Fatal().Err(err).Msg("failed unmarshaling koanf config")
+		log.Panic().Err(err).Msg("failed unmarshaling koanf config")
 	}
 
 	c.fromReader(&rc)

src/engines/swisscows/dontaskjustenjoy.go

@@ -21,10 +21,10 @@ func performMagic(input string) string {
 		if value_str, err := value.ToString(); err == nil {
 			result = value_str
 		} else {
-			log.Error().Msgf("Swisscows: MagicPerformer - couldn't convert returned value to string. Error: %v", err)
+			log.Error().Err(err).Msg("Swisscows: MagicPerformer - couldn't convert returned value to string.")
 		}
 	} else {
-		log.Error().Msgf("Swisscows: MagicPerformer - couldn't get result from javascript VM. Error: %v", err)
+		log.Error().Err(err).Msg("Swisscows: MagicPerformer - couldn't get result from javascript VM.")
 	}
 
 	return result

src/engines/swisscows/swisscows.go

@@ -53,7 +53,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
 	})
 
 	col.OnError(func(r *colly.Response, err error) {
-		log.Error().Msgf("%v: SE Collector - OnError.\nMethod: %v\nURL: %v\nError: %v", Info.Name, r.Request.Method, r.Request.URL.String(), err)
+		log.Error().Err(err).Msgf("%v: SE Collector - OnError.\nMethod: %v\nURL: %v", Info.Name, r.Request.Method, r.Request.URL.String())
 		log.Error().Msgf("%v: HTML Response written to %v%v_col.log.html", Info.Name, config.LogDumpLocation, Info.Name)
 		writeErr := os.WriteFile(config.LogDumpLocation+string(Info.Name)+"_col.log.html", r.Body, 0644)
 		if writeErr != nil {

src/main.go

@@ -35,13 +35,13 @@ func main() {
 
 	// load config file
 	conf := config.New()
-	conf.Load(cliFlags.Config, cliFlags.Log)
+	conf.Load(cliFlags.Data, cliFlags.Log)
 
 	// cache database
 	var db cache.DB
 	switch conf.Server.Cache.Type {
 	case "pebble":
-		db = pebble.New(cliFlags.Config)
+		db = pebble.New(cliFlags.Data)
 	case "redis":
 		db = redis.New(ctx, conf.Server.Cache.Redis)
 	default:
@@ -54,7 +54,7 @@ func main() {
 		cli.Run(cliFlags, db, conf)
 	} else {
 		if rw, err := router.New(conf, cliFlags.Verbosity); err != nil {
-			log.Error().Msgf("Failed creating a router: %v", err)
+			log.Error().Err(err).Msg("Failed creating a router")
 		} else {
 			rw.Start(ctx, db, cliFlags.ServeProfiler)
 		}

src/router/router.go

@@ -40,11 +40,11 @@ func (rw *RouterWrapper) addCors() {
 
 func (rw *RouterWrapper) runWithContext(ctx context.Context) {
 	if err := rw.router.RunWithContext(ctx); err != context.Canceled {
-		log.Error().Msgf("Failed starting router: %v", err)
+		log.Error().Err(err).Msg("Failed starting router")
 	} else if err != nil {
-		log.Info().Msgf("Stopping router...")
+		log.Info().Msg("Stopping router...")
 		rw.router.Close()
-		log.Debug().Msgf("Successfully stopped router")
+		log.Debug().Msg("Successfully stopped router")
 	}
 }
 

src/search/search.go

@@ -38,7 +38,8 @@ func PerformSearch(query string, options engines.Options, conf *config.Config) [
 	rankTimer := time.Now()
 	log.Debug().Msg("Ranking...")
 	results := rank.Rank(relay.ResultMap, conf.Categories[options.Category].Ranking) // have to make copy, since its a map value
-	log.Debug().Msgf("Finished ranking in %vns", time.Since(rankTimer).Nanoseconds())
+	rankTimeSince := time.Since(rankTimer)
+	log.Debug().Msgf("Finished ranking in %vms (%vns)", rankTimeSince.Milliseconds(), rankTimeSince.Nanoseconds())
 
 	log.Debug().Msgf("Found results in %vms", time.Since(searchTimer).Milliseconds())
 

src/sedefaults/sedefaults.go

@@ -50,7 +50,7 @@ func ColRequest(seName engines.Name, col *colly.Collector, ctx *context.Context,
 
 func ColError(seName engines.Name, col *colly.Collector, retError *error) {
 	col.OnError(func(r *colly.Response, err error) {
-		log.Error().Msgf("%v: SE Collector - OnError.\nURL: %v\nError: %v", seName, r.Request.URL.String(), err)
+		log.Error().Err(err).Msgf("%v: SE Collector - OnError.\nURL: %v", seName, r.Request.URL.String())
 		log.Debug().Msgf("%v: HTML Response written to %v%v_col.log.html", seName, config.LogDumpLocation, seName)
 		writeErr := os.WriteFile(config.LogDumpLocation+string(seName)+"_col.log.html", r.Body, 0644)
 		if writeErr != nil {