Max elastic exported request/response size

.github/workflows/build-test.yml

@@ -1,7 +1,6 @@
 name: 🔨 Build Test
 
 on:
-  push:
   pull_request:
   workflow_dispatch:
 
@@ -16,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
 
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/codeql-analysis.yml

@@ -2,7 +2,6 @@ name: 🚨 CodeQL Analysis
 
 on:
   workflow_dispatch:
-  push:
   pull_request:
     branches:
       - dev

.github/workflows/lint-test.yml

@@ -1,6 +1,5 @@
 name: 🙏🏻 Lint Test
 on:
-  push:
   pull_request:
   workflow_dispatch:
 
@@ -16,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
 
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v3.4.0

.github/workflows/release-binary.yml

@@ -18,7 +18,7 @@ jobs:
       - name: "Set up Go"
         uses: actions/setup-go@v3
         with: 
-          go-version: 1.18
+          go-version: 1.19
 
       - name: "Create release on GitHub"
         uses: goreleaser/goreleaser-action@v4

.github/workflows/sonarcloud.yml

@@ -20,7 +20,7 @@ jobs:
       - name: "Set up Go"
         uses: actions/setup-go@v3
         with: 
-          go-version: 1.18
+          go-version: 1.19
 
       - name: Run unit Tests
         run: |

README.md

@@ -90,6 +90,7 @@ PROXY:
    -c int                       Number of requests before switching to the next upstream proxy (default 1)
 
 EXPORT:
+   -max-size int              Max export data size (request/responses will be truncated) (default 9223372036854775807)
    -elastic-address string    elasticsearch address (ip:port)
    -elastic-ssl               enable elasticsearch ssl
    -elastic-ssl-verification  enable elasticsearch ssl verification

cmd/proxify/proxify.go

@@ -21,19 +21,17 @@ func main() {
 
 	// Setup close handler
 	go func() {
-		c := make(chan os.Signal,1) //added size 1 to channel buffer
+		c := make(chan os.Signal, 1)
 		signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-		go func() {
-			<-c
+		for range c {
 			fmt.Println("\r- Ctrl+C pressed in Terminal")
 			proxifyRunner.Close()
 			os.Exit(0)
-		}()
+		}
 	}()
 
 	err = proxifyRunner.Run()
 	if err != nil {
 		gologger.Fatal().Msgf("Could not run proxify: %s\n", err)
 	}
-
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/projectdiscovery/proxify
 
-go 1.18
+go 1.19
 
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible
@@ -18,7 +18,7 @@ require (
 	github.com/projectdiscovery/goflags v0.1.7
 	github.com/projectdiscovery/gologger v1.1.8
 	github.com/projectdiscovery/tinydns v0.0.1
-	github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d
+	github.com/projectdiscovery/utils v0.0.11
 	github.com/rs/xid v1.4.0
 	golang.org/x/net v0.7.0
 	gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -439,8 +439,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA
 github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
 github.com/projectdiscovery/tinydns v0.0.1 h1:Ls5TAlMJoCfEObpsC7S+Hg4WLyF5wiN+O1xNgEIjnEc=
 github.com/projectdiscovery/tinydns v0.0.1/go.mod h1:xPZcaMje/MWozKc5Sdln3XWptINYoWqCBXgLeSrNB1Q=
-github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d h1:iB/n2/NL4oh1IaEcqX6pBxj0WHfYN7finzNOKVNVISM=
-github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ=
+github.com/projectdiscovery/utils v0.0.11 h1:JwyW3Fp7iF6d3XjzsaC+Gz38EROYuXlMxm7re7LyGvk=
+github.com/projectdiscovery/utils v0.0.11/go.mod h1:0AqTr9qCXU7dD+c1hItoVY2xhr4gLr3U4/ABiQvhW1U=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=

internal/runner/options.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"math"
 	"os"
 	"path/filepath"
 
@@ -40,6 +41,7 @@ type Options struct {
 	Elastic                     elastic.Options
 	Kafka                       kafka.Options
 	PassThrough                 goflags.StringSlice // Passthrough items list
+	MaxSize                     int
 }
 
 func ParseOptions() *Options {
@@ -54,35 +56,36 @@ func ParseOptions() *Options {
 	flagSet := goflags.NewFlagSet()
 	flagSet.SetDescription(`Swiss Army Knife Proxy for rapid deployments. Supports multiple operations such as request/response dump,filtering and manipulation via DSL language, upstream HTTP/Socks5 proxy`)
 
-	createGroup(flagSet, "output", "Output",
+	flagSet.CreateGroup("output", "Output",
 		// Todo:	flagSet.BoolVar(&options.Dump, "dump", true, "Dump HTTP requests/response to output file"),
 		flagSet.StringVarP(&options.OutputDirectory, "output", "o", "logs", "Output Directory to store HTTP proxy logs"),
 		flagSet.BoolVar(&options.DumpRequest, "dump-req", false, "Dump only HTTP requests to output file"),
 		flagSet.BoolVar(&options.DumpResponse, "dump-resp", false, "Dump only HTTP responses to output file"),
 	)
 
-	createGroup(flagSet, "filter", "Filter",
+	flagSet.CreateGroup("filter", "Filter",
 		flagSet.StringSliceVarP(&options.RequestDSL, "request-dsl", "req-fd", nil, "Request Filter DSL", goflags.StringSliceOptions),
 		flagSet.StringSliceVarP(&options.ResponseDSL, "response-dsl", "resp-fd", nil, "Response Filter DSL", goflags.StringSliceOptions),
 		flagSet.StringSliceVarP(&options.RequestMatchReplaceDSL, "request-match-replace-dsl", "req-mrd", nil, "Request Match-Replace DSL", goflags.StringSliceOptions),
 		flagSet.StringSliceVarP(&options.ResponseMatchReplaceDSL, "response-match-replace-dsl", "resp-mrd", nil, "Response Match-Replace DSL", goflags.StringSliceOptions),
 	)
 
-	createGroup(flagSet, "network", "Network",
+	flagSet.CreateGroup("network", "Network",
 		flagSet.StringVarP(&options.ListenAddrHTTP, "http-addr", "ha", "127.0.0.1:8888", "Listening HTTP IP and Port address (ip:port)"),
 		flagSet.StringVarP(&options.ListenAddrSocks5, "socks-addr", "sa", "127.0.0.1:10080", "Listening SOCKS IP and Port address (ip:port)"),
 		flagSet.StringVarP(&options.ListenDNSAddr, "dns-addr", "da", "", "Listening DNS IP and Port address (ip:port)"),
 		flagSet.StringVarP(&options.DNSMapping, "dns-mapping", "dm", "", "Domain to IP DNS mapping (eg domain:ip,domain:ip,..)"),
 		flagSet.StringVarP(&options.DNSFallbackResolver, "resolver", "r", "", "Custom DNS resolvers to use (ip:port)"),
 	)
 
-	createGroup(flagSet, "proxy", "Proxy",
+	flagSet.CreateGroup("proxy", "Proxy",
 		flagSet.StringSliceVarP(&options.UpstreamHTTPProxies, "http-proxy", "hp", nil, "Upstream HTTP Proxies (eg http://proxy-ip:proxy-port)", goflags.NormalizedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.UpstreamSocks5Proxies, "socks5-proxy", "sp", nil, "Upstream SOCKS5 Proxies (eg socks5://proxy-ip:proxy-port)", goflags.NormalizedStringSliceOptions),
 		flagSet.IntVar(&options.UpstreamProxyRequestsNumber, "c", 1, "Number of requests before switching to the next upstream proxy"),
 	)
 
-	createGroup(flagSet, "export", "Export",
+	flagSet.CreateGroup("export", "Export",
+		flagSet.IntVar(&options.MaxSize, "max-size", math.MaxInt, "Max export data size (request/responses will be truncated)"),
 		flagSet.StringVar(&options.Elastic.Addr, "elastic-address", "", "elasticsearch address (ip:port)"),
 		flagSet.BoolVar(&options.Elastic.SSL, "elastic-ssl", false, "enable elasticsearch ssl"),
 		flagSet.BoolVar(&options.Elastic.SSLVerification, "elastic-ssl-verification", false, "enable elasticsearch ssl verification"),
@@ -93,7 +96,7 @@ func ParseOptions() *Options {
 		flagSet.StringVar(&options.Kafka.Topic, "kafka-topic", "proxify", "kafka topic to publish messages on"),
 	)
 
-	createGroup(flagSet, "configuration", "Configuration",
+	flagSet.CreateGroup("configuration", "Configuration",
 		// Todo: default config file support (homeDir/.config/proxify/config.yaml)
 		flagSet.StringVar(&options.Directory, "config", filepath.Join(homeDir, ".config", "proxify"), "Directory for storing program information"),
 		flagSet.IntVar(&options.CertCacheSize, "cert-cache-size", 256, "Number of certificates to cache"),
@@ -103,7 +106,7 @@ func ParseOptions() *Options {
 	)
 
 	silent, verbose, veryVerbose := false, false, false
-	createGroup(flagSet, "debug", "debug",
+	flagSet.CreateGroup("debug", "debug",
 		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", true, "No Color"),
 		flagSet.BoolVar(&options.Version, "version", false, "Version"),
 		flagSet.BoolVar(&silent, "silent", false, "Silent"),
@@ -156,10 +159,3 @@ func (options *Options) configureOutput() {
 		gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
 	}
 }
-
-func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
-	flagSet.SetGroup(groupName, description)
-	for _, currentFlag := range flags {
-		currentFlag.Group(groupName)
-	}
-}

internal/runner/runner.go

@@ -35,6 +35,7 @@ func NewRunner(options *Options) (*Runner, error) {
 		ResponseMatchReplaceDSL:     options.ResponseMatchReplaceDSL,
 		DumpRequest:                 options.DumpRequest,
 		DumpResponse:                options.DumpResponse,
+		MaxSize:                     options.MaxSize,
 		UpstreamProxyRequestsNumber: options.UpstreamProxyRequestsNumber,
 		Elastic:                     &options.Elastic,
 		Kafka:                       &options.Kafka,

pkg/logger/elastic/elasticsearch.go

@@ -34,6 +34,7 @@ type Options struct {
 // Client type for elasticsearch
 type Client struct {
 	index    string
+	options  *Options
 	esClient *elasticsearch.Client
 }
 
@@ -57,10 +58,12 @@ func New(option *Options) (*Client, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "error creating elasticsearch client")
 	}
-	return &Client{
+	client := &Client{
 		esClient: elasticsearchClient,
 		index:    option.IndexName,
-	}, nil
+		options:  option,
+	}
+	return client, nil
 
 }
 

pkg/logger/logger.go

@@ -28,6 +28,7 @@ type OptionsLogger struct {
 	OutputFolder string
 	DumpRequest  bool
 	DumpResponse bool
+	MaxSize      int
 	Elastic      *elastic.Options
 	Kafka        *kafka.Options
 }
@@ -112,6 +113,10 @@ func (l *Logger) AsyncWrite() {
 
 			outputdata.DataString = fmt.Sprintf(outputdata.Format, outputdata.Data)
 
+			if l.options.MaxSize > 0 {
+				outputdata.DataString = stringsutil.Truncate(outputdata.DataString, l.options.MaxSize)
+			}
+
 			for _, store := range l.Store {
 				err := store.Save(outputdata)
 				if err != nil {

proxy.go

@@ -42,6 +42,7 @@ type OnConnectFunc func(string, *goproxy.ProxyCtx) (*goproxy.ConnectAction, stri
 type Options struct {
 	DumpRequest                 bool
 	DumpResponse                bool
+	MaxSize                     int
 	Verbosity                   types.Verbosity
 	CertCacheSize               int
 	Directory                   string
@@ -377,6 +378,7 @@ func NewProxy(options *Options) (*Proxy, error) {
 		OutputFolder: options.OutputDirectory,
 		DumpRequest:  options.DumpRequest,
 		DumpResponse: options.DumpResponse,
+		MaxSize:      options.MaxSize,
 		Elastic:      options.Elastic,
 		Kafka:        options.Kafka,
 	})