Properly escape all control chars when formatting (#46)

endocrimes · Sep 7, 2016 · ebda409 · ebda409
1 parent 535666f
commit ebda409
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 16 deletions.
diff --git a/Sources/Jay/Consts.swift b/Sources/Jay/Consts.swift
@@ -51,7 +51,9 @@ struct Const {
         Const.FormFeed: Const.FormFeedChar,
         Const.CarriageReturn: Const.CarriageReturnChar
     ]
-
+
+    static let ControlCharacters: Set<JChar> = Set(0x00...0x1f)
+
     // Strings
     static let QuotationMark: JChar         = 0x22 // """
     static let ReverseSolidus: JChar        = 0x5c // "\"
@@ -66,7 +68,7 @@ struct Const {
 
     static let UnicodeStart: JChar          = 0x75 // "u"
 
-    static let Escape: JChar            = Const.ReverseSolidus
+    static let Escape: JChar                = Const.ReverseSolidus
 
     static let SimpleEscaped: Set<JChar> = [
         Const.QuotationMark,

diff --git a/Sources/Jay/Extensions.swift b/Sources/Jay/Extensions.swift
@@ -21,6 +21,22 @@ extension JChar {
     }
 }
 
+extension JChar {
+
+    func controlCharacterHexString() -> [JChar] {
+        var hex = String(self, radix: 16, uppercase: false).chars()
+
+        //control chars are always only two hex characters, so we either got 1 or 2 chars,
+        //pad to two
+        if hex.count == 1 {
+            hex = [Const.Zero] + hex
+        }
+
+        //and prepend with \u00, which is followed by our two hex bytes
+        return [Const.Escape, Const.UnicodeStart, Const.Zero, Const.Zero] + hex
+    }
+}
+
 extension String {
 
     func chars() -> [JChar] {

diff --git a/Sources/Jay/Formatter.swift b/Sources/Jay/Formatter.swift
@@ -50,7 +50,7 @@ extension JSON: JsonFormattable {
 
     func format(to stream: JsonOutputStream, string: String) throws {
 
-        var contents = [JChar]()
+        var contents: [JChar] = []
         for c in string.utf8 {
 
             //if this is an escapable character, escape it
@@ -66,6 +66,13 @@ extension JSON: JsonFormattable {
                 continue
             }
 
+            //control character that wasn't escaped above, just convert to 
+            //an escaped unicode sequence, i.e. "\u0006" for "ACK"
+            if Const.ControlCharacters.contains(c) {
+                contents.append(contentsOf: c.controlCharacterHexString())
+                continue
+            }
+
             //nothing to escape, just append byte
             contents.append(c)
         }

diff --git a/Tests/JayTests/FormattingTests.swift b/Tests/JayTests/FormattingTests.swift
@@ -155,9 +155,10 @@ class FormattingTests: XCTestCase {
     }
 
     func testString_Escaping() {
-        let json = ["he \r\n l \t l \n o w\"o\rrld "]
+        let json = ["he \r\n l \u{0006} \t l \n o w\"o\rrld \u{0015} "]
         let data = try! Jay().dataFromJson(any: json)
-        XCTAssertEqual(data, "[\"he \\r\\n l \\t l \\n o w\\\"o\\rrld \"]".chars())
+        let res = "[\"he \\r\\n l \\u0006 \\t l \\n o w\\\"o\\rrld \\u0015 \"]".chars()
+        XCTAssertEqual(data, res)
     }
 
     func testVaporExample_Dict() {

diff --git a/Tests/JayTests/ParsingTests.swift b/Tests/JayTests/ParsingTests.swift
@@ -55,6 +55,7 @@ extension ParsingTests {
         ("testEscape_Unicode_InvalidUnicode_MissingDigit", testEscape_Unicode_InvalidUnicode_MissingDigit),
         ("testEscape_Unicode_InvalidUnicode_MissingAllDigits", testEscape_Unicode_InvalidUnicode_MissingAllDigits),
         ("testEscape_SpecialChars", testEscape_SpecialChars),
+        ("testString_Escaping", testString_Escaping),
         ("testString_Empty", testString_Empty),
         ("testString_Normal", testString_Normal),
         ("testString_Normal_WhitespaceInside", testString_Normal_WhitespaceInside),
@@ -378,7 +379,7 @@ class ParsingTests:XCTestCase {
         XCTAssertNil(try? StringParser().unescapedCharacter(reader))
     }
 
-    func testEscape_SpecialChars() {
+    func testEscape_SpecialChars() throws {
 
         let chars: [JChar] = [
             //regular translation
@@ -392,40 +393,58 @@ class ParsingTests:XCTestCase {
             Const.Escape, Const.NewLineChar,
             Const.Escape, Const.CarriageReturnChar,
             Const.Escape, Const.HorizontalTabChar,
-            Const.Space
+
+            //basic control chars
+            Const.Escape, Const.UnicodeStart, Const.Zero, Const.Zero, Const.Zero, 0x36 /* ACK */,
+            Const.Escape, Const.UnicodeStart, Const.Zero, Const.Zero, 0x32, 0x31 /* NACK */,
+
+            Const.Space,
         ]
 
         let reader = ByteReader(content: chars)
         var char: UnicodeScalar
 
         //regular
 
-        char = try! StringParser().unescapedCharacter(reader)
+        char = try StringParser().unescapedCharacter(reader)
         XCTAssert(try! Const.QuotationMark.string() == String(char))
 
-        char = try! StringParser().unescapedCharacter(reader)
+        char = try StringParser().unescapedCharacter(reader)
         XCTAssert(try! Const.ReverseSolidus.string() == String(char))
 
-        char = try! StringParser().unescapedCharacter(reader)
+        char = try StringParser().unescapedCharacter(reader)
         XCTAssert(try! Const.Solidus.string() == String(char))
 
-
         //rule-based
 
-        char = try! StringParser().unescapedCharacter(reader)
+        char = try StringParser().unescapedCharacter(reader)
         XCTAssert(try! Const.Backspace.string() == String(char))
 
-        char = try! StringParser().unescapedCharacter(reader)
+        char = try StringParser().unescapedCharacter(reader)
         XCTAssert(try! Const.FormFeed.string() == String(char))
 
-        char = try! StringParser().unescapedCharacter(reader)
+        char = try StringParser().unescapedCharacter(reader)
         XCTAssertEqual(try! Const.NewLine.string(), String(char))
 
-        char = try! StringParser().unescapedCharacter(reader)
+        char = try StringParser().unescapedCharacter(reader)
         XCTAssertEqual(try! Const.CarriageReturn.string(), String(char))
 
-        char = try! StringParser().unescapedCharacter(reader)
+        char = try StringParser().unescapedCharacter(reader)
         XCTAssertEqual(try! Const.HorizontalTab.string(), String(char))
+
+        //basic control
+        char = try StringParser().unescapedCharacter(reader)
+        XCTAssert("\u{0006}" == String(char))
+
+        char = try StringParser().unescapedCharacter(reader)
+        XCTAssert("\u{0021}" == String(char))
+    }
+
+    func testString_Escaping() {
+        let reader = ByteReader(content: "[\"he \\r\\n l \\u0006 \\t l \\n o w\\\"o\\rrld \\u0015 \"]")
+        let json: [JSON] = [.string("he \r\n l \u{0006} \t l \n o w\"o\rrld \u{0015} ")]
+        let ret = try! Jay().jsonFromReader(reader)
+        ensureArray(ret, exp: json)
     }
 
     func testString_Empty() {