From f3df4465c94bbc97af047ba2dc43fd9685412c8a Mon Sep 17 00:00:00 2001 From: molsonkiko <46202915+molsonkiko@users.noreply.github.com> Date: Fri, 29 Nov 2024 13:29:05 -0800 Subject: [PATCH] Allow parsing of \u0000 and \x00 Previously the JSON parser was deliberately designed to choke if it encountered \x00 or \u0000, since those represent the NUL character, which is treated as the terminating character in any string passed from NPP to a plugin or vice versa. However, I have decided that this limitation does not make much sense, because when JsonTools reformats JSON, it represents all control characters using the \u escape (so it is internally consistent and correct to represent NUL as \u0000), and thus most plugin commands will not cause problems when working with JSON that contains \u0000 or \x00. WARNINGS: 1. JsonTools is still incapable of parsing text that contains a literal NUL character (as opposed to the \x00 or \u0000 escape sequences), because when Notepad++ sends the text of a file to a plugin, it truncates that text at the first NUL character. 2. The "Dump JSON string(s) as raw text" plugin command (and any other plugin commands that return the raw text of a JSON string), when used to dump a JSON string that contains \x00 or \u0000 escapes, will instead dump all the raw text in that string *up until the first \x00 or \u0000*, because JsonTools sends Notepad++ a string containing a literal NUL character, which Notepad++ treats as the end of the string. --- CHANGELOG.md | 1 + JsonToolsNppPlugin/JSONTools/JNode.cs | 1 + JsonToolsNppPlugin/JSONTools/JsonParser.cs | 5 +- JsonToolsNppPlugin/JSONTools/YamlDumper.cs | 6 +- JsonToolsNppPlugin/Properties/AssemblyInfo.cs | 4 +- JsonToolsNppPlugin/Tests/JsonParserTests.cs | 12 +- JsonToolsNppPlugin/Tests/YamlDumperTests.cs | 2 + most recent errors.txt | 112 +++++++++--------- 8 files changed, 75 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cfbdde..684bbc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed 1. When a file is [pretty-printed or compressed](/docs/README.md#the-basics) or [edited with RemesPath](/docs/RemesPath.md#editing-with-assignment-expressions), the caret will be scrolled into view. Previously, if the file contained very long lines and word wrapping was turned off, [the user might have to manually scroll to the left](https://github.com/molsonkiko/JsonToolsNppPlugin/issues/84) after pretty-printing or compressing. +2. NUL characters (when rendered as `\u0000` or `\x00`) can now be parsed. Previously [they raised a fatal error](https://github.com/molsonkiko/JsonToolsNppPlugin/issues/85). ## [8.2.0] - 2024-11-09 diff --git a/JsonToolsNppPlugin/JSONTools/JNode.cs b/JsonToolsNppPlugin/JSONTools/JNode.cs index db38aca..094d78b 100644 --- a/JsonToolsNppPlugin/JSONTools/JNode.cs +++ b/JsonToolsNppPlugin/JSONTools/JNode.cs @@ -295,6 +295,7 @@ public static void CharToSb(StringBuilder sb, char c) { case '\\': sb.Append("\\\\" ); break; case '"': sb.Append("\\\"" ); break; + case '\x00': sb.Append("\\u0000"); break; case '\x01': sb.Append("\\u0001"); break; case '\x02': sb.Append("\\u0002"); break; case '\x03': sb.Append("\\u0003"); break; diff --git a/JsonToolsNppPlugin/JSONTools/JsonParser.cs b/JsonToolsNppPlugin/JSONTools/JsonParser.cs index 6ce5d5f..cbd5e8f 100644 --- a/JsonToolsNppPlugin/JSONTools/JsonParser.cs +++ b/JsonToolsNppPlugin/JSONTools/JsonParser.cs @@ -191,7 +191,6 @@ public string TranslateMessageIfDesired(bool translated) // FATAL messages case JsonLintType.FATAL_EXPECTED_JAVASCRIPT_COMMENT: return Translator.TranslateLintMessage(translated, lintType, "Expected JavaScript comment after '/'"); case JsonLintType.FATAL_HEXADECIMAL_TOO_SHORT: return TryTranslateWithOneParam(translated, lintType, "Could not find valid hexadecimal of length {0}", param1); - case JsonLintType.FATAL_NUL_CHAR: return Translator.TranslateLintMessage(translated, lintType, "'\\x00' is the null character, which is illegal in JsonTools"); case JsonLintType.FATAL_UNTERMINATED_KEY: return Translator.TranslateLintMessage(translated, lintType, "Unterminated object key"); case JsonLintType.FATAL_INVALID_STARTSWITH_n: return Translator.TranslateLintMessage(translated, lintType, "Expected literal starting with 'n' to be null or nan"); case JsonLintType.FATAL_PLUS_OR_MINUS_AT_EOF: return TryTranslateWithOneParam(translated, lintType, "'{0}' sign at end of document", param1); @@ -366,7 +365,7 @@ public enum JsonLintType : short /// param1 = expected_hex_length (int) /// FATAL_HEXADECIMAL_TOO_SHORT = FATAL_EXPECTED_JAVASCRIPT_COMMENT + 1, - FATAL_NUL_CHAR = FATAL_EXPECTED_JAVASCRIPT_COMMENT + 2, + FATAL_UNTERMINATED_KEY = FATAL_EXPECTED_JAVASCRIPT_COMMENT + 3, FATAL_INVALID_STARTSWITH_n = FATAL_EXPECTED_JAVASCRIPT_COMMENT + 4, /// @@ -888,8 +887,6 @@ private bool HandleCharErrors(int c, string inp, int ii) { if (c == '\n') return HandleError(JsonLintType.BAD_STRING_CONTAINS_NEWLINE, inp, ii, ParserState.BAD); - if (c == 0) - return HandleError(JsonLintType.FATAL_NUL_CHAR, inp, ii); if (c < 0) return true; return HandleError(JsonLintType.OK_CONTROL_CHAR, inp, ii); diff --git a/JsonToolsNppPlugin/JSONTools/YamlDumper.cs b/JsonToolsNppPlugin/JSONTools/YamlDumper.cs index 9e1aa4a..6a8f954 100644 --- a/JsonToolsNppPlugin/JSONTools/YamlDumper.cs +++ b/JsonToolsNppPlugin/JSONTools/YamlDumper.cs @@ -61,6 +61,7 @@ private string EscapeBackslash(string s) { switch (c) { + case '\x00': sb.Append("\\0"); break; case '\n': sb.Append("\\n"); break; case '\t': sb.Append(@"\\t"); break; case '\\': sb.Append(@"\\"); break; @@ -83,6 +84,7 @@ private string EscapeBackslashKey(string s) { switch (c) { + case '\x00': sb.Append("\\0"); break; case '\n': sb.Append("\\n"); break; case '\t': sb.Append(@"\\t"); break; case '\\': sb.Append(@"\\"); break; @@ -112,7 +114,7 @@ private string YamlKeyRepr(string k) // a YAML parser will recognize that it is not actually a number. return "'" + k + "'"; } - Regex forbiddenKeyChars = new Regex(@"[\t :]"); + Regex forbiddenKeyChars = new Regex(@"[\t :\x00]"); if (forbiddenKeyChars.IsMatch(k)) { // '\t', ' ', and ':' are all illegal inside a YAML key. We will escape those out @@ -141,7 +143,7 @@ private string YamlValRepr(JNode v) { return $"\"{strv}\""; } - Regex backslash = new Regex("([\\\\:\"'\r\t\n\f\b])"); + Regex backslash = new Regex("([\\\\:\"'\r\t\n\f\b\x00])"); if (backslash.IsMatch(strv)) { return EscapeBackslash(strv); diff --git a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs index 67fdbb7..afcc0c1 100644 --- a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs +++ b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs @@ -28,5 +28,5 @@ // Build Number // Revision // -[assembly: AssemblyVersion("8.2.0.1")] -[assembly: AssemblyFileVersion("8.2.0.1")] +[assembly: AssemblyVersion("8.2.0.2")] +[assembly: AssemblyFileVersion("8.2.0.2")] diff --git a/JsonToolsNppPlugin/Tests/JsonParserTests.cs b/JsonToolsNppPlugin/Tests/JsonParserTests.cs index ba33b08..4253417 100644 --- a/JsonToolsNppPlugin/Tests/JsonParserTests.cs +++ b/JsonToolsNppPlugin/Tests/JsonParserTests.cs @@ -947,7 +947,7 @@ public static bool TestLinter() "Whitespace characters other than ' ', '\\t', '\\r', and '\\n' are only allowed in JSON5", }), ("{foo: 1, $baz: 2, 草: 2, _quЯ: 3, \\ud83d\\ude00_$\\u1ed3: 4, a\\uff6acf: 5, \\u0008\\u000a: 6, f\\u0000o: 1}", - "{\"foo\": 1, \"$baz\": 2, \"草\": 2, \"_quЯ\": 3, \"😀_$ồ\": 4, \"aｪcf\": 5, \"\\b\\n\": 6}", + "{\"foo\": 1, \"$baz\": 2, \"草\": 2, \"_quЯ\": 3, \"😀_$ồ\": 4, \"aｪcf\": 5, \"\\b\\n\": 6, \"f\\u0000o\": 1}", new string[]{ "Unquoted keys are only allowed in JSON5", "Unquoted keys are only allowed in JSON5", @@ -959,7 +959,7 @@ public static bool TestLinter() "Control characters (ASCII code less than 0x20) are disallowed inside strings under the strict JSON specification", "String literal contains newline", // the \u000a in \\b\\u000a is secretly a newline "Unquoted keys are only allowed in JSON5", - "'\\x00' is the null character, which is illegal in JsonTools" + "Control characters (ASCII code less than 0x20) are disallowed inside strings under the strict JSON specification" }), ("[1,\"b\\\nb\\\rb\\\r\nb\"]", "[1, \"bbbb\"]", new string[] @@ -975,8 +975,12 @@ public static bool TestLinter() "Escaped newline characters are only allowed in JSON5", "Escaped newline characters are only allowed in JSON5", }), - ("[\"a\\x00b\", 1]", "[\"a\"]", new string[]{"'\\x00' is the null character, which is illegal in JsonTools"}), - ("[\"a\\u0000b\", 1]", "[\"a\"]", new string[]{"'\\x00' is the null character, which is illegal in JsonTools"}), + ("[\"a\\x00b\", 1]", "[\"a\\u0000b\", 1]", new string[] + { + "Control characters (ASCII code less than 0x20) are disallowed inside strings under the strict JSON specification", + "\\x escapes are only allowed in JSON5", + }), + ("[\"a\\u0000b\", 1]", "[\"a\\u0000b\", 1]", new string[]{"Control characters (ASCII code less than 0x20) are disallowed inside strings under the strict JSON specification"}), ("{\"\\1\\A\": \"\\7\\B\"}", "{\"1A\": \"7B\"}", new string[]{ "Escaped char '1' is only allowed in JSON5", diff --git a/JsonToolsNppPlugin/Tests/YamlDumperTests.cs b/JsonToolsNppPlugin/Tests/YamlDumperTests.cs index 0d41ed1..87e7c74 100644 --- a/JsonToolsNppPlugin/Tests/YamlDumperTests.cs +++ b/JsonToolsNppPlugin/Tests/YamlDumperTests.cs @@ -66,6 +66,8 @@ public static bool Test() "a:\n -\n - 1\n - 2.0\n -\n '3':\n - '5'\n'2': 6\n", "nested iterables" }, new string[] { "{\"a\": \"a: b\"}", "a: \"a: b\"\n", "value contains colon" }, + new string[] { "{\"a\\u0000\": \"a: b\"}", "\"a\\0\": \"a: b\"\n", "value contains colon and key contains NUL" }, + new string[] { "{\"zwg\": \"a\\u0000b\"}", "zwg: \"a\\0b\"\n", "value contains NUL" }, new string[] { "{\"a: b\": \"a\"}", "\"a: b\": a\n", "key contains colon" }, new string[] { "{\"a\": \"RT @blah: MondayMo\\\"r\'ing\"}", "a: \'RT @blah: MondayMo\"r\'\'ing\'\n", "Value contains quotes and colon" }, new string[] { "{\"a\": \"a\\n\'big\'\\ndog\"}", "a: \"a\\n\'big\'\\ndog\"\n", "Value contains quotes and newline" }, diff --git a/most recent errors.txt b/most recent errors.txt index 8170760..bb15585 100644 --- a/most recent errors.txt +++ b/most recent errors.txt @@ -1,4 +1,4 @@ -Test results for JsonTools v8.2.0.1 on Notepad++ 8.7.1 64bit +Test results for JsonTools v8.2.0.2 on Notepad++ 8.7.1 64bit NOTE: Ctrl-F (regular expressions *on*) for "Failed [1-9]\d*" to find all failed tests Tests failed: YAML dumper ========================= @@ -63,21 +63,21 @@ Passed 39 tests. Testing YAML dumper ========================= -Test 19 (Value contains quotes and colon) failed: +Test 21 (Value contains quotes and colon) failed: Expected a: 'RT @blah: MondayMo"r''ing' Got a: "RT @blah: MondayMo\"r\'ing" -Test 20 (Value contains quotes and newline) failed: +Test 22 (Value contains quotes and newline) failed: Expected a: "a\n'big'\ndog" Got a: "a\n\'big\'\ndog" -Test 22 (key contains quotes and colon) failed: +Test 24 (key contains quotes and colon) failed: Expected '"a: ''b''': a @@ -85,7 +85,7 @@ Got "\"a: ''b''": a Failed 3 tests. -Passed 19 tests. +Passed 21 tests. ========================= Testing slice extension ========================= @@ -206,33 +206,33 @@ Testing JsonParser performance Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... -To convert JSON string of size 89556 into JNode took 3.538 +/- 2.763 ms over 32 trials -Load times (ms): 8, 12, 2, 3, 10, 3, 3, 2, 5, 1, 1, 6, 1, 1, 2, 5, 1, 1, 5, 1, 1, 1, 4, 1, 1, 5, 1, 1, 1, 4, 1, 1 +To convert JSON string of size 89556 into JNode took 3.197 +/- 3.655 ms over 32 trials +Load times (ms): 1, 1, 21, 1, 1, 5, 1, 1, 1, 4, 1, 1, 6, 1, 1, 1, 4, 1, 1, 6, 1, 1, 1, 4, 1, 1, 6, 1, 1, 1, 4, 2 ========================= Performance tests for RemesPath (float arithmetic) ========================= -Compiling query "@[@[:].a * @[:].t < @[:].e]" took 0.094 ms the first time, including approximately 0.124 ms to tokenize the query. Subsequent executions are effectively free due to caching. -To run pre-compiled query "@[@[:].a * @[:].t < @[:].e]" on JNode from JSON of size 89556 into took 0.114 +/- 0.467 ms over 40 trials -Query times (ms): 0.095, 0.062, 0.047, 0.042, 0.044, 0.057, 0.046, 0.05, 0.044, 0.045, 0.044, 3.028, 0.05, 0.035, 0.033, 0.032, 0.032, 0.032, 0.033, 0.034, 0.032, 0.033, 0.032, 0.032, 0.032, 0.034, 0.032, 0.032, 0.032, 0.033, 0.032, 0.033, 0.055, 0.032, 0.032, 0.032, 0.032, 0.033, 0.032, 0.032 +Compiling query "@[@[:].a * @[:].t < @[:].e]" took 0.088 ms the first time, including approximately 0.118 ms to tokenize the query. Subsequent executions are effectively free due to caching. +To run pre-compiled query "@[@[:].a * @[:].t < @[:].e]" on JNode from JSON of size 89556 into took 0.038 +/- 0.018 ms over 40 trials +Query times (ms): 0.109, 0.06, 0.039, 0.045, 0.039, 0.039, 0.039, 0.051, 0.041, 0.04, 0.04, 0.044, 0.04, 0.079, 0.039, 0.039, 0.051, 0.068, 0.038, 0.054, 0.023, 0.052, 0.024, 0.023, 0.023, 0.032, 0.025, 0.023, 0.023, 0.023, 0.024, 0.033, 0.023, 0.023, 0.024, 0.024, 0.023, 0.051, 0.023, 0.022 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= Performance tests for RemesPath (string operations) ========================= -Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" took 0.056 ms the first time, including approximately 0.071 ms to tokenize the query. Subsequent executions are effectively free due to caching. -To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 89556 into took 0.092 +/- 0.011 ms over 40 trials -Query times (ms): 0.152, 0.1, 0.091, 0.09, 0.089, 0.088, 0.088, 0.089, 0.088, 0.088, 0.088, 0.089, 0.087, 0.088, 0.089, 0.087, 0.111, 0.09, 0.09, 0.089, 0.089, 0.09, 0.088, 0.089, 0.1, 0.088, 0.09, 0.116, 0.088, 0.089, 0.088, 0.089, 0.088, 0.088, 0.088, 0.088, 0.088, 0.095, 0.088, 0.087 +Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" took 0.04 ms the first time, including approximately 0.044 ms to tokenize the query. Subsequent executions are effectively free due to caching. +To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 89556 into took 0.154 +/- 0.521 ms over 40 trials +Query times (ms): 0.113, 0.056, 0.098, 0.062, 0.065, 0.053, 0.053, 0.053, 0.054, 0.054, 0.053, 0.053, 0.053, 0.054, 0.057, 0.054, 0.053, 3.401, 0.062, 0.054, 0.052, 0.052, 0.051, 0.051, 0.051, 0.131, 0.058, 0.087, 0.097, 0.059, 0.074, 0.06, 0.122, 0.129, 0.061, 0.098, 0.058, 0.196, 0.059, 0.053 Preview of result: [{"A": "\n]o1VQ5t6g", "a": 4710024278, "b": 3268860721, "B": "g4Y7+ew^.v", "C": "NK nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" took 0.19 ms the first time, including approximately 0.145 ms to tokenize the query. Subsequent executions are effectively free due to caching. +ifelse(nmax_q > nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" took 0.144 ms the first time, including approximately 0.123 ms to tokenize the query. Subsequent executions are effectively free due to caching. To run pre-compiled query "var qmask = @[:].q; var nmax_q = max(@[qmask].n); var nmax_notq = max(@[not qmask].n); -ifelse(nmax_q > nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" on JNode from JSON of size 89556 into took 0.02 +/- 0.011 ms over 40 trials -Query times (ms): 0.069, 0.017, 0.028, 0.016, 0.016, 0.016, 0.016, 0.017, 0.018, 0.017, 0.054, 0.035, 0.016, 0.016, 0.041, 0.016, 0.015, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016, 0.016, 0.015, 0.016, 0.016, 0.016, 0.015, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.015 +ifelse(nmax_q > nmax_notq, `when q=true, nmax = ` + str(nmax_q), `when q=false, nmax= ` + str(nmax_notq))" on JNode from JSON of size 89556 into took 0.018 +/- 0.01 ms over 40 trials +Query times (ms): 0.08, 0.019, 0.017, 0.016, 0.016, 0.017, 0.017, 0.017, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.017, 0.016, 0.016, 0.016, 0.016, 0.016, 0.015, 0.016, 0.017, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.024, 0.016, 0.016 Preview of result: "when q=false, nmax= 9830935647.0" ... ========================= @@ -271,11 +271,11 @@ Performance tests for RemesPath (references to compile-time constant variables) Compiling query "var X = X; var onetwo = j`[1, 2]`; -@[:]->at(@, X)->at(@, onetwo)" took 0.078 ms the first time, including approximately 0.102 ms to tokenize the query. Subsequent executions are effectively free due to caching. +@[:]->at(@, X)->at(@, onetwo)" took 0.723 ms the first time, including approximately 0.115 ms to tokenize the query. Subsequent executions are effectively free due to caching. To run pre-compiled query "var X = X; var onetwo = j`[1, 2]`; -@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.023 +/- 0.053 ms over 40 trials -Query times (ms): 0.057, 0.013, 0.35, 0.019, 0.012, 0.013, 0.013, 0.012, 0.013, 0.012, 0.013, 0.037, 0.012, 0.013, 0.013, 0.013, 0.013, 0.012, 0.013, 0.013, 0.013, 0.013, 0.012, 0.013, 0.013, 0.013, 0.013, 0.013, 0.012, 0.013, 0.013, 0.013, 0.014, 0.013, 0.013, 0.014, 0.012, 0.013, 0.013, 0.013 +@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.019 +/- 0.015 ms over 40 trials +Query times (ms): 0.064, 0.014, 0.012, 0.012, 0.012, 0.013, 0.013, 0.015, 0.018, 0.02, 0.024, 0.022, 0.02, 0.015, 0.013, 0.018, 0.02, 0.013, 0.012, 0.012, 0.024, 0.013, 0.012, 0.04, 0.012, 0.012, 0.013, 0.02, 0.09, 0.024, 0.019, 0.02, 0.012, 0.014, 0.017, 0.016, 0.014, 0.013, 0.013, 0.013 Preview of result: [[1695727848, 0.287562638736685], [2126430375, 0.00767794129708177], [5310550656, 0.380769772645687], [2519183283, 0.153176220930558], [6610062385, 0.662996225870666], [987168256, 0.924410189999928], [6615003609, 0.917112691225947], [4465232046, 0.684311931851536], [8654414565, 0.631485392105992], [ ... ========================= @@ -284,29 +284,29 @@ Performance tests for RemesPath (references to variables that are not compile-ti Compiling query "var X = @->`X`; var onetwo = @{1, 2}; -@[:]->at(@, X)->at(@, onetwo)" took 0.176 ms the first time, including approximately 0.164 ms to tokenize the query. Subsequent executions are effectively free due to caching. +@[:]->at(@, X)->at(@, onetwo)" took 0.102 ms the first time, including approximately 0.116 ms to tokenize the query. Subsequent executions are effectively free due to caching. To run pre-compiled query "var X = @->`X`; var onetwo = @{1, 2}; -@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.025 +/- 0.014 ms over 40 trials -Query times (ms): 0.079, 0.028, 0.019, 0.019, 0.018, 0.019, 0.026, 0.048, 0.03, 0.022, 0.018, 0.018, 0.018, 0.018, 0.021, 0.018, 0.022, 0.05, 0.019, 0.018, 0.018, 0.018, 0.018, 0.019, 0.061, 0.027, 0.017, 0.017, 0.016, 0.017, 0.017, 0.021, 0.05, 0.018, 0.016, 0.017, 0.017, 0.017, 0.017, 0.036 +@[:]->at(@, X)->at(@, onetwo)" on JNode from JSON of size 89556 into took 0.017 +/- 0.007 ms over 40 trials +Query times (ms): 0.058, 0.018, 0.016, 0.023, 0.017, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.017, 0.016, 0.016, 0.017, 0.016, 0.016, 0.016, 0.016, 0.015, 0.016, 0.017, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.016, 0.017, 0.015, 0.016, 0.016, 0.015, 0.016, 0.016 Preview of result: [[1695727848, 0.287562638736685], [2126430375, 0.00767794129708177], [5310550656, 0.380769772645687], [2519183283, 0.153176220930558], [6610062385, 0.662996225870666], [987168256, 0.924410189999928], [6615003609, 0.917112691225947], [4465232046, 0.684311931851536], [8654414565, 0.631485392105992], [ ... ========================= Performance tests for RemesPath (simple string mutations) ========================= -Compiling query "@[:].z = s_sub(@, g, B)" took 0.065 ms the first time, including approximately 0.064 ms to tokenize the query. Subsequent executions are effectively free due to caching. -To run pre-compiled query "@[:].z = s_sub(@, g, B)" on JNode from JSON of size 89556 into took 0.017 +/- 0.01 ms over 40 trials -Query times (ms): 0.041, 0.03, 0.019, 0.012, 0.011, 0.013, 0.012, 0.014, 0.023, 0.023, 0.023, 0.021, 0.016, 0.034, 0.057, 0.036, 0.024, 0.015, 0.012, 0.011, 0.01, 0.01, 0.011, 0.01, 0.036, 0.011, 0.012, 0.011, 0.02, 0.013, 0.011, 0.01, 0.012, 0.011, 0.01, 0.011, 0.012, 0.01, 0.011, 0.01 +Compiling query "@[:].z = s_sub(@, g, B)" took 0.062 ms the first time, including approximately 0.05 ms to tokenize the query. Subsequent executions are effectively free due to caching. +To run pre-compiled query "@[:].z = s_sub(@, g, B)" on JNode from JSON of size 89556 into took 0.027 +/- 0.024 ms over 40 trials +Query times (ms): 0.068, 0.048, 0.033, 0.024, 0.016, 0.018, 0.023, 0.028, 0.018, 0.034, 0.046, 0.029, 0.019, 0.02, 0.026, 0.026, 0.016, 0.013, 0.025, 0.031, 0.02, 0.02, 0.023, 0.024, 0.025, 0.023, 0.015, 0.027, 0.022, 0.012, 0.011, 0.01, 0.01, 0.01, 0.055, 0.023, 0.154, 0.011, 0.01, 0.014 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= Performance tests for RemesPath (simple number mutations) ========================= -Compiling query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" took 0.074 ms the first time, including approximately 0.096 ms to tokenize the query. Subsequent executions are effectively free due to caching. -To run pre-compiled query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" on JNode from JSON of size 89556 into took 0.039 +/- 0.111 ms over 40 trials -Query times (ms): 0.732, 0.021, 0.02, 0.043, 0.038, 0.018, 0.018, 0.018, 0.02, 0.02, 0.019, 0.019, 0.019, 0.026, 0.024, 0.02, 0.019, 0.018, 0.018, 0.018, 0.018, 0.02, 0.018, 0.018, 0.018, 0.016, 0.022, 0.041, 0.024, 0.022, 0.019, 0.018, 0.019, 0.019, 0.019, 0.018, 0.02, 0.016, 0.022, 0.025 +Compiling query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" took 0.081 ms the first time, including approximately 0.069 ms to tokenize the query. Subsequent executions are effectively free due to caching. +To run pre-compiled query "@[:].x = ifelse(@ < 0.5, @ + 3, @ - 3)" on JNode from JSON of size 89556 into took 0.022 +/- 0.008 ms over 40 trials +Query times (ms): 0.045, 0.021, 0.018, 0.019, 0.018, 0.02, 0.02, 0.018, 0.018, 0.018, 0.018, 0.022, 0.017, 0.026, 0.052, 0.026, 0.022, 0.018, 0.04, 0.018, 0.019, 0.017, 0.018, 0.018, 0.018, 0.018, 0.026, 0.024, 0.018, 0.017, 0.019, 0.043, 0.018, 0.018, 0.023, 0.019, 0.018, 0.017, 0.022, 0.023 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= @@ -316,55 +316,55 @@ Performance tests for RemesPath (mutations with a for loop) Compiling query "var xhalf = @[:].x < 0.5; for lx = zip(@[:].l, xhalf); lx[0] = ifelse(lx[1], foo, bar); -end for;" took 0.18 ms the first time, including approximately 0.135 ms to tokenize the query. Subsequent executions are effectively free due to caching. +end for;" took 13.383 ms the first time, including approximately 0.136 ms to tokenize the query. Subsequent executions are effectively free due to caching. To run pre-compiled query "var xhalf = @[:].x < 0.5; for lx = zip(@[:].l, xhalf); lx[0] = ifelse(lx[1], foo, bar); -end for;" on JNode from JSON of size 89556 into took 0.074 +/- 0.059 ms over 40 trials -Query times (ms): 0.209, 0.074, 0.053, 0.053, 0.053, 0.051, 0.374, 0.12, 0.188, 0.108, 0.078, 0.068, 0.06, 0.051, 0.051, 0.051, 0.054, 0.048, 0.05, 0.048, 0.05, 0.051, 0.052, 0.054, 0.052, 0.061, 0.052, 0.051, 0.051, 0.051, 0.052, 0.052, 0.052, 0.048, 0.053, 0.053, 0.075, 0.077, 0.078, 0.055 +end for;" on JNode from JSON of size 89556 into took 0.044 +/- 0.02 ms over 40 trials +Query times (ms): 0.134, 0.121, 0.047, 0.041, 0.038, 0.036, 0.037, 0.036, 0.037, 0.038, 0.036, 0.05, 0.038, 0.037, 0.038, 0.038, 0.037, 0.036, 0.037, 0.037, 0.037, 0.04, 0.036, 0.05, 0.052, 0.04, 0.04, 0.039, 0.037, 0.04, 0.039, 0.037, 0.038, 0.038, 0.044, 0.046, 0.038, 0.037, 0.038, 0.039 Preview of result: [["bar", false], ["bar", false], ["foo", true], ["foo", true], ["foo", true], ["foo", true], ["foo", true], ["bar", false], ["bar", false], ["bar", false], ["foo", true], ["foo", true], ["bar", false], ["bar", false], ["foo", true], ["bar", false], ["bar", false], ["bar", false], ["foo", true], ["ba ... ========================= Testing performance and correctness of parsing and dumping arrays of non-integer numbers ========================= -To parse arrays of 5000 non-integer numbers (representative length = 96372, representative example preview: "[-19926.082, -3.0, -44321.2e-15, -57945761.59228993116e15, -52298.32422403554090, 3.0873, -259347281.79649055186502, -7869333.53116e-80, -5177181.81354715059583e-41, 64440.6370296041, 706956213.044561...") took 2.191 +/- 0.861 ms over 32 trials -Times to parse (ms): 1, 1, 2, 2, 1, 3, 1, 1, 1, 1, 1, 1, 1, 2, 4, 2, 1, 1, 1, 1, 1, 1, 2, 6, 2, 1, 1, 1, 2, 1, 1, 2 -To re-compress (convert back to minimal JSON strings) the arrays made from parsing those strings took 7.522 +/- 1.028 ms over 32 trials -Times to re-compress (ms): 7, 7, 7, 7, 7, 9, 7, 7, 6, 7, 7, 7, 7, 8, 7, 7, 7, 6, 7, 7, 6, 6, 7, 12, 7, 7, 7, 6, 8, 7, 7, 7 -Representative example of result of re-compression = "[-19926.082, -3.0, -4.43212E-11, -5.7945761592289935E+22, -52298.324224035539, 3.0873, -259347281.79649055, -7.86933353116E-74, -5.1771818135471509E-35, 64440.6370296041, 706956213.04456174, -1.784344..." +To parse arrays of 5000 non-integer numbers (representative length = 96420, representative example preview: "[-2483068.16971539836465, -5458347.20614399087, 6.196500619652387, 8.6730756667e-28, -7472513.287661140, 246575405.024e-63, 85621.421855174, -381507.90752, -91557991.67853, -6.9533499247786672e88, -73...") took 1.754 +/- 0.195 ms over 32 trials +Times to parse (ms): 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +To re-compress (convert back to minimal JSON strings) the arrays made from parsing those strings took 7.337 +/- 0.43 ms over 32 trials +Times to re-compress (ms): 6, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 7, 7, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 7, 7, 7, 7, 7 +Representative example of result of re-compression = "[-2483068.1697153985, -5458347.2061439911, 6.1965006196523866, 8.6730756667E-28, -7472513.28766114, 2.46575405024E-55, 85621.421855174, -381507.90752, -91557991.67853, -6.953349924778667E+88, -7.38838..." ========================= Testing performance of JSON compression and pretty-printing ========================= Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... -To compress JNode from JSON string of 89556 took 4.836 +/- 1.504 ms over 64 trials (minimal whitespace, sortKeys=TRUE) -To compress JNode from JSON string of 89556 took 3.158 +/- 0.996 ms over 64 trials (minimal whitespace, sortKeys=FALSE) -To Google-style pretty-print JNode from JSON string of 89556 took 5.275 +/- 1.403 ms over 64 trials (sortKeys=true, indent=4) -To Whitesmith-style pretty-print JNode from JSON string of 89556 took 5.92 +/- 2.029 ms over 64 trials (sortKeys=true, indent=4) -To PPrint-style pretty-print JNode from JSON string of 89556 took 7.821 +/- 2.261 ms over 64 trials (sortKeys=true, indent=4) +To compress JNode from JSON string of 89556 took 4.364 +/- 0.655 ms over 64 trials (minimal whitespace, sortKeys=TRUE) +To compress JNode from JSON string of 89556 took 2.353 +/- 0.193 ms over 64 trials (minimal whitespace, sortKeys=FALSE) +To Google-style pretty-print JNode from JSON string of 89556 took 4.538 +/- 0.198 ms over 64 trials (sortKeys=true, indent=4) +To Whitesmith-style pretty-print JNode from JSON string of 89556 took 5.826 +/- 0.961 ms over 64 trials (sortKeys=true, indent=4) +To PPrint-style pretty-print JNode from JSON string of 89556 took 7.548 +/- 1.082 ms over 64 trials (sortKeys=true, indent=4) ========================= Testing performance of JsonSchemaValidator and random JSON creation ========================= -To create a random set of JSON from file at path C:\Program Files\Notepad++\plugins\JsonTools\testfiles\tweet_schema.json of size 210213 (array of 15 items) based on the matching schema took 5.916 +/- 3.047 ms over 25 trials -To compile the schema to a validation function took 0.833 +/- 1.411 ms over 25 trials -To validate JSON of size 210213 (array of 15 items) based on the compiled schema took 1.024 +/- 0.172 ms over 25 trials +To create a random set of JSON from file at path C:\Program Files\Notepad++\plugins\JsonTools\testfiles\tweet_schema.json of size 193766 (array of 15 items) based on the matching schema took 7.018 +/- 2.985 ms over 25 trials +To compile the schema to a validation function took 0.213 +/- 0.03 ms over 25 trials +To validate JSON of size 193766 (array of 15 items) based on the compiled schema took 1.035 +/- 0.136 ms over 25 trials ========================= Testing performance of random JSON from schema with patterns and patternProperties ========================= -To create a random set of JSON from string (see TestRunner.cs) of size 28588 (array of 120 items) based on the matching schema took 1.496 +/- 0.513 ms over 25 trials -To compile the schema to a validation function took 0.369 +/- 0.454 ms over 25 trials -To validate JSON of size 28588 (array of 120 items) based on the compiled schema took 10.08 +/- 1.542 ms over 25 trials +To create a random set of JSON from string (see TestRunner.cs) of size 30266 (array of 120 items) based on the matching schema took 1.451 +/- 0.659 ms over 25 trials +To compile the schema to a validation function took 0.256 +/- 0.012 ms over 25 trials +To validate JSON of size 30266 (array of 120 items) based on the compiled schema took 9.37 +/- 0.75 ms over 25 trials ========================= Testing performance of random JSON from schema *ignoring* patterns and patternProperties ========================= -To create a random set of JSON from string (see TestRunner.cs) of size 9956 (array of 120 items) based on the matching schema took 0.834 +/- 0.469 ms over 25 trials -To compile the schema to a validation function took 0.242 +/- 0.038 ms over 25 trials -To validate JSON of size 9956 (array of 120 items) based on the compiled schema took 5.981 +/- 0.534 ms over 25 trials +To create a random set of JSON from string (see TestRunner.cs) of size 10507 (array of 120 items) based on the matching schema took 0.903 +/- 0.636 ms over 25 trials +To compile the schema to a validation function took 0.23 +/- 0.032 ms over 25 trials +To validate JSON of size 10507 (array of 120 items) based on the compiled schema took 5.443 +/- 0.421 ms over 25 trials ========================= Testing JSON grepper's API request tool =========================