From 4f29b7a9303ec2de99ad92214adc85673fc9d71a Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Fri, 10 Feb 2023 00:25:35 +0530 Subject: [PATCH 1/3] use standard url enc format --- url/rawparam.go | 19 +++++++++++++++---- url/rawparam_test.go | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/url/rawparam.go b/url/rawparam.go index e864d07..fba7458 100644 --- a/url/rawparam.go +++ b/url/rawparam.go @@ -172,7 +172,7 @@ func URLEncodeWithEscapes(data string, charset ...rune) string { case r < rune(20): // control character buff.WriteRune('%') - buff.WriteString(strconv.FormatInt(int64(r), 16)) // 2 digit hex + buff.WriteString(getasciihex(r)) // 2 digit hex case r == ' ': // prefer using + when space buff.WriteRune('+') @@ -181,7 +181,7 @@ func URLEncodeWithEscapes(data string, charset ...rune) string { if _, ok := mustescape[r]; ok { // reserved char must escape buff.WriteRune('%') - buff.WriteString(strconv.FormatInt(int64(r), 16)) + buff.WriteString(getasciihex(r)) } else { // do not percent encode buff.WriteRune(r) @@ -189,7 +189,7 @@ func URLEncodeWithEscapes(data string, charset ...rune) string { case r == rune(127): // [DEL] char should be encoded buff.WriteRune('%') - buff.WriteString(strconv.FormatInt(int64(r), 16)) + buff.WriteString(getasciihex(r)) case r > rune(128): // non-ascii characters i.e chinese chars or any other utf-8 buff.WriteRune('%') @@ -209,7 +209,7 @@ func PercentEncoding(data string) string { buff.WriteRune('%') if r <= rune(127) { // these are all ascii characters - buff.WriteString(strconv.FormatInt(int64(r), 16)) + buff.WriteString(getasciihex(r)) } else { // unicode characters buff.WriteString(getutf8hex(r)) @@ -238,6 +238,7 @@ func getrunemap(runes []rune) map[rune]struct{} { return x } +// returns hex value of utf-8 non-ascii char func getutf8hex(r rune) string { // Percent Encoding is only done in hexadecimal values and in ASCII Range only // other UTF-8 chars (chinese etc) can be used by utf-8 encoding and byte conversion @@ -253,3 +254,13 @@ func getutf8hex(r rune) string { } return buff.String() } + +// returns hex value of ascii char +func getasciihex(r rune) string { + val := strconv.FormatInt(int64(r), 16) + if len(val) == 1 { + // append 0 formatInt skips it by default + val = "0" + val + } + return strings.ToUpper(val) +} diff --git a/url/rawparam_test.go b/url/rawparam_test.go index 7013d86..98cdded 100644 --- a/url/rawparam_test.go +++ b/url/rawparam_test.go @@ -69,10 +69,10 @@ func TestParamIntegration(t *testing.T) { func TestPercentEncoding(t *testing.T) { // From Burpsuite - expected := "%74%65%73%74%20%26%23%20%28%29%20%70%65%72%63%65%6e%74%20%5b%5d%7c%2a%20%65%6e%63%6f%64%69%6e%67" + expected := "%74%65%73%74%20%26%23%20%28%29%20%70%65%72%63%65%6E%74%20%5B%5D%7C%2A%20%65%6E%63%6F%64%69%6E%67" payload := "test &# () percent []|* encoding" value := PercentEncoding(payload) - require.Equalf(t, value, expected, "expected percentencoding to be %v but got %v", expected, payload) + require.Equalf(t, value, expected, "expected percentencoding to be %v but got %v", expected, value) } func TestGetParams(t *testing.T) { @@ -84,3 +84,16 @@ func TestGetParams(t *testing.T) { require.Equalf(t, p.Get("sqli"), values.Get("sqli"), "malformed or missing value for param sqli expected %v but got %v", values.Get("sqli"), p.Get("sqli")) require.Equalf(t, p.Get("xss"), values.Get("xss"), "malformed or missing value for param xss expected %v but got %v", values.Get("xss"), p.Get("xss")) } + +func TestURLEncode(t *testing.T) { + example := "\r\n" + got := URLEncodeWithEscapes(example) + require.Equalf(t, "%0D%0A", got, "failed to url encode characters") + + // verify with stdlib + for r := 0; r < 20; r++ { + expected := url.QueryEscape(string(rune(r))) + got := URLEncodeWithEscapes(string(rune(r))) + require.Equalf(t, expected, got, "url encoding mismatch for non-printable char with ascii val:%v", r) + } +} From 234ad5ea272b9437393430b2e84c9c8668bfb1d3 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Fri, 10 Feb 2023 00:29:15 +0530 Subject: [PATCH 2/3] add url decode test --- url/rawparam_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/url/rawparam_test.go b/url/rawparam_test.go index 98cdded..1d38008 100644 --- a/url/rawparam_test.go +++ b/url/rawparam_test.go @@ -73,6 +73,9 @@ func TestPercentEncoding(t *testing.T) { payload := "test &# () percent []|* encoding" value := PercentEncoding(payload) require.Equalf(t, value, expected, "expected percentencoding to be %v but got %v", expected, value) + decoded, err := url.QueryUnescape(value) + require.Nil(t, err) + require.Equal(t, payload, decoded) } func TestGetParams(t *testing.T) { From bd8b9dc814d59cb8712762cd3186bce8c4cc758f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 9 Feb 2023 23:42:19 +0100 Subject: [PATCH 3/3] minor changes --- url/rawparam.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/url/rawparam.go b/url/rawparam.go index fba7458..720f415 100644 --- a/url/rawparam.go +++ b/url/rawparam.go @@ -163,12 +163,11 @@ func ParamEncode(data string) string { func URLEncodeWithEscapes(data string, charset ...rune) string { mustescape := getrunemap(charset) var buff bytes.Buffer - totallen := len(data) // In any case - buff.Grow(totallen) + buff.Grow(len(data)) for _, r := range data { - switch true { + switch { case r < rune(20): // control character buff.WriteRune('%')