Merge pull request #82 from projectdiscovery/issue-81-url-char-format

use standard url encoding format

url/rawparam.go

@@ -163,16 +163,15 @@ func ParamEncode(data string) string {
 func URLEncodeWithEscapes(data string, charset ...rune) string {
 	mustescape := getrunemap(charset)
 	var buff bytes.Buffer
-	totallen := len(data)
 	// In any case
-	buff.Grow(totallen)
+	buff.Grow(len(data))
 
 	for _, r := range data {
-		switch true {
+		switch {
 		case r < rune(20):
 			// control character
 			buff.WriteRune('%')
-			buff.WriteString(strconv.FormatInt(int64(r), 16)) // 2 digit hex
+			buff.WriteString(getasciihex(r)) // 2 digit hex
 		case r == ' ':
 			// prefer using + when space
 			buff.WriteRune('+')
@@ -181,15 +180,15 @@ func URLEncodeWithEscapes(data string, charset ...rune) string {
 			if _, ok := mustescape[r]; ok {
 				// reserved char must escape
 				buff.WriteRune('%')
-				buff.WriteString(strconv.FormatInt(int64(r), 16))
+				buff.WriteString(getasciihex(r))
 			} else {
 				// do not percent encode
 				buff.WriteRune(r)
 			}
 		case r == rune(127):
 			// [DEL] char should be encoded
 			buff.WriteRune('%')
-			buff.WriteString(strconv.FormatInt(int64(r), 16))
+			buff.WriteString(getasciihex(r))
 		case r > rune(128):
 			// non-ascii characters i.e chinese chars or any other utf-8
 			buff.WriteRune('%')
@@ -209,7 +208,7 @@ func PercentEncoding(data string) string {
 		buff.WriteRune('%')
 		if r <= rune(127) {
 			// these are all ascii characters
-			buff.WriteString(strconv.FormatInt(int64(r), 16))
+			buff.WriteString(getasciihex(r))
 		} else {
 			// unicode characters
 			buff.WriteString(getutf8hex(r))
@@ -238,6 +237,7 @@ func getrunemap(runes []rune) map[rune]struct{} {
 	return x
 }
 
+// returns hex value of utf-8 non-ascii char
 func getutf8hex(r rune) string {
 	// Percent Encoding is only done in hexadecimal values and in ASCII Range only
 	// other UTF-8 chars (chinese etc) can be used by utf-8 encoding and byte conversion
@@ -253,3 +253,13 @@ func getutf8hex(r rune) string {
 	}
 	return buff.String()
 }
+
+// returns hex value of ascii char
+func getasciihex(r rune) string {
+	val := strconv.FormatInt(int64(r), 16)
+	if len(val) == 1 {
+		// append 0 formatInt skips it by default
+		val = "0" + val
+	}
+	return strings.ToUpper(val)
+}

url/rawparam_test.go

@@ -69,10 +69,13 @@ func TestParamIntegration(t *testing.T) {
 
 func TestPercentEncoding(t *testing.T) {
 	// From Burpsuite
-	expected := "%74%65%73%74%20%26%23%20%28%29%20%70%65%72%63%65%6e%74%20%5b%5d%7c%2a%20%65%6e%63%6f%64%69%6e%67"
+	expected := "%74%65%73%74%20%26%23%20%28%29%20%70%65%72%63%65%6E%74%20%5B%5D%7C%2A%20%65%6E%63%6F%64%69%6E%67"
 	payload := "test &# () percent []|* encoding"
 	value := PercentEncoding(payload)
-	require.Equalf(t, value, expected, "expected percentencoding to be %v but got %v", expected, payload)
+	require.Equalf(t, value, expected, "expected percentencoding to be %v but got %v", expected, value)
+	decoded, err := url.QueryUnescape(value)
+	require.Nil(t, err)
+	require.Equal(t, payload, decoded)
 }
 
 func TestGetParams(t *testing.T) {
@@ -84,3 +87,16 @@ func TestGetParams(t *testing.T) {
 	require.Equalf(t, p.Get("sqli"), values.Get("sqli"), "malformed or missing value for param sqli expected %v but got %v", values.Get("sqli"), p.Get("sqli"))
 	require.Equalf(t, p.Get("xss"), values.Get("xss"), "malformed or missing value for param xss expected %v but got %v", values.Get("xss"), p.Get("xss"))
 }
+
+func TestURLEncode(t *testing.T) {
+	example := "\r\n"
+	got := URLEncodeWithEscapes(example)
+	require.Equalf(t, "%0D%0A", got, "failed to url encode characters")
+
+	// verify with stdlib
+	for r := 0; r < 20; r++ {
+		expected := url.QueryEscape(string(rune(r)))
+		got := URLEncodeWithEscapes(string(rune(r)))
+		require.Equalf(t, expected, got, "url encoding mismatch for non-printable char with ascii val:%v", r)
+	}
+}