From 36be00068874296d72bdb821bb6d7bcddbd5f987 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Tue, 3 Sep 2019 12:51:20 +0200 Subject: [PATCH 01/10] SQL template with triple quote in completion --- .../console/public/src/autocomplete.js | 7 +++++- .../public/src/autocomplete/body_completer.js | 12 ++++++++++ .../spec/overrides/sql.query.json | 23 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 x-pack/legacy/plugins/console_extensions/spec/overrides/sql.query.json diff --git a/src/legacy/core_plugins/console/public/src/autocomplete.js b/src/legacy/core_plugins/console/public/src/autocomplete.js index 8275813ea2b98..3965a3e8ca379 100644 --- a/src/legacy/core_plugins/console/public/src/autocomplete.js +++ b/src/legacy/core_plugins/console/public/src/autocomplete.js @@ -345,7 +345,12 @@ export default function (editor) { let valueToInsert = termAsString; let templateInserted = false; if (context.addTemplate && !_.isUndefined(term.template) && !_.isNull(term.template)) { - const indentedTemplateLines = utils.jsonToString(term.template, true).split('\n'); + let indentedTemplateLines; + if (term.template.__raw && term.template.value) { + indentedTemplateLines = term.template.value.split('\n'); + } else { + indentedTemplateLines = utils.jsonToString(term.template, true).split('\n'); + } let currentIndentation = session.getLine(context.rangeToReplace.start.row); currentIndentation = currentIndentation.match(/^\s*/)[0]; for (let i = 1; i < indentedTemplateLines.length; i++) // skip first line diff --git a/src/legacy/core_plugins/console/public/src/autocomplete/body_completer.js b/src/legacy/core_plugins/console/public/src/autocomplete/body_completer.js index 68e0889f9c618..e377a749aaf78 100644 --- a/src/legacy/core_plugins/console/public/src/autocomplete/body_completer.js +++ b/src/legacy/core_plugins/console/public/src/autocomplete/body_completer.js @@ -127,6 +127,18 @@ class ScopeResolver extends SharedComponent { } function getTemplate(description) { if (description.__template) { + if (description.__raw && _.isString(description.__template)) { + return { + // This is a special secret attribute that gets passed through to indicate that + // the raw value should be passed through to the console without JSON.stringifying it + // first. + // + // Primary use case is to allow __templates to contain extended JSON special values like + // triple quotes. + __raw: true, + value: description.__template, + }; + } return description.__template; } else if (description.__one_of) { return getTemplate(description.__one_of[0]); diff --git a/x-pack/legacy/plugins/console_extensions/spec/overrides/sql.query.json b/x-pack/legacy/plugins/console_extensions/spec/overrides/sql.query.json new file mode 100644 index 0000000000000..9c30ce4470e71 --- /dev/null +++ b/x-pack/legacy/plugins/console_extensions/spec/overrides/sql.query.json @@ -0,0 +1,23 @@ +{ + "sql.query": { + "data_autocomplete_rules": { + "query": { + "__template": { + "__raw": true, + "value": "\n\"\"\"\nSELECT * FROM \"TABLE\"\n\"\"\"" + } + } + }, + "url_params": { + "format": [ + "csv", + "json", + "tsv", + "txt", + "yaml", + "cbor", + "smile" + ] + } + } +} From a5977368b0d8d0cda9a91171ddc72bb9275fd55d Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Thu, 5 Sep 2019 19:10:09 +0200 Subject: [PATCH 02/10] Slight update to SQL query template and bugfix for collapse/expand of request bodies --- src/legacy/core_plugins/console/public/src/utils.js | 13 +++++++------ .../spec/overrides/sql.query.json | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/legacy/core_plugins/console/public/src/utils.js b/src/legacy/core_plugins/console/public/src/utils.js index 1f2c3b9267b74..5f7ae4468a4ce 100644 --- a/src/legacy/core_plugins/console/public/src/utils.js +++ b/src/legacy/core_plugins/console/public/src/utils.js @@ -59,18 +59,19 @@ utils.reformatData = function (data, indent) { }; utils.collapseLiteralStrings = function (data) { - return data.replace(/"""(?:\s*\r?\n)?((?:.|\r?\n)*?)(?:\r?\n\s*)?"""/g, function (match, literal) { - return JSON.stringify(literal); - }); + const splitData = data.split(`"""`); + for (let idx = 1; idx < splitData.length - 1; idx += 2) { + splitData[idx] = JSON.stringify(splitData[idx]); + } + return splitData.join(''); }; utils.expandLiteralStrings = function (data) { return data.replace(/("(?:\\"|[^"])*?")/g, function (match, string) { // expand things with two slashes or more - if (string.split(/\\./).length > 2) { + if (string.match(/\\./)) { string = JSON.parse(string).replace('^\s*\n', '').replace('\n\s*^', ''); - const append = string.includes('\n') ? '\n' : ''; // only go multi line if the string has multiline - return '"""' + append + string + append + '"""'; + return '"""' + string + '"""'; } else { return string; } diff --git a/x-pack/legacy/plugins/console_extensions/spec/overrides/sql.query.json b/x-pack/legacy/plugins/console_extensions/spec/overrides/sql.query.json index 9c30ce4470e71..f2fe456a0c6e2 100644 --- a/x-pack/legacy/plugins/console_extensions/spec/overrides/sql.query.json +++ b/x-pack/legacy/plugins/console_extensions/spec/overrides/sql.query.json @@ -4,7 +4,7 @@ "query": { "__template": { "__raw": true, - "value": "\n\"\"\"\nSELECT * FROM \"TABLE\"\n\"\"\"" + "value": "\"\"\"\nSELECT * FROM \"TABLE\"\n\"\"\"" } } }, From af4dd0af65b6b9304f73d1b27071bdf31aee03a2 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Tue, 10 Sep 2019 11:28:17 +0200 Subject: [PATCH 03/10] Add comment and slight update to triple quote expansion --- .../core_plugins/console/public/src/autocomplete.js | 2 ++ src/legacy/core_plugins/console/public/src/utils.js | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/legacy/core_plugins/console/public/src/autocomplete.js b/src/legacy/core_plugins/console/public/src/autocomplete.js index 3965a3e8ca379..a29854ee2a02a 100644 --- a/src/legacy/core_plugins/console/public/src/autocomplete.js +++ b/src/legacy/core_plugins/console/public/src/autocomplete.js @@ -346,6 +346,8 @@ export default function (editor) { let templateInserted = false; if (context.addTemplate && !_.isUndefined(term.template) && !_.isNull(term.template)) { let indentedTemplateLines; + // In order to allow triple quoted strings in template completion we check the `__raw_` + // attribute to determine whether this template should go through JSON formatting. if (term.template.__raw && term.template.value) { indentedTemplateLines = term.template.value.split('\n'); } else { diff --git a/src/legacy/core_plugins/console/public/src/utils.js b/src/legacy/core_plugins/console/public/src/utils.js index 5f7ae4468a4ce..de4b7d429b2bf 100644 --- a/src/legacy/core_plugins/console/public/src/utils.js +++ b/src/legacy/core_plugins/console/public/src/utils.js @@ -69,9 +69,11 @@ utils.collapseLiteralStrings = function (data) { utils.expandLiteralStrings = function (data) { return data.replace(/("(?:\\"|[^"])*?")/g, function (match, string) { // expand things with two slashes or more - if (string.match(/\\./)) { - string = JSON.parse(string).replace('^\s*\n', '').replace('\n\s*^', ''); - return '"""' + string + '"""'; + if (string.split(/\\./).length > 2) { + string = JSON.parse(string) + .replace('^s*\n', '') + .replace('\ns*^', ''); + return '"""' + string + '"""'; } else { return string; } From 7fb689b91cb9bf9d49c17f8809f9f3dcb4c8a57e Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Tue, 10 Sep 2019 12:04:58 +0200 Subject: [PATCH 04/10] Updated tests after changes to newline behaviour --- .../public/tests/src/utils_string_collapsing.txt | 10 +++++----- .../public/tests/src/utils_string_expanding.txt | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/legacy/core_plugins/console/public/tests/src/utils_string_collapsing.txt b/src/legacy/core_plugins/console/public/tests/src/utils_string_collapsing.txt index 235151317377f..5ddd1b5a376fe 100644 --- a/src/legacy/core_plugins/console/public/tests/src/utils_string_collapsing.txt +++ b/src/legacy/core_plugins/console/public/tests/src/utils_string_collapsing.txt @@ -8,11 +8,11 @@ to you """ ========== String only 2 ------------------------------------- -""" +""" starting with new lines and ending as well - """ +""" ------------------------------------- -"starting with new lines and ending as well" +"\nstarting with new lines and ending as well\n" ========== Strings in requests ------------------------------------- @@ -27,8 +27,8 @@ test2 } ------------------------------------- { - "f": { "somefield" : "test\ntest2" }, + "f": { "somefield" : "\ntest\ntest2\n" }, "g": { "script" : "second + \"\\\";" }, "h": 1, "script": "a + 2" -} \ No newline at end of file +} diff --git a/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt b/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt index 762e0ad164634..4f3abf177490a 100644 --- a/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt +++ b/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt @@ -2,7 +2,7 @@ Scripts in requests ------------------------------------- { - "f": { "script" : { "source": "test\ntest\\2" } }, + "f": { "script" : { "source": "\ntest\ntest\\2\n" } }, "g": { "script" : "second + \"\\\";" }, "f": "short with \\", "h": 1, From b10e2e23dbe05bbe00f48db2aa2b7669a9deac38 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Tue, 10 Sep 2019 11:59:01 +0200 Subject: [PATCH 05/10] Restore old backslash triple quote expansion behaviour --- src/legacy/core_plugins/console/public/src/utils.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/legacy/core_plugins/console/public/src/utils.js b/src/legacy/core_plugins/console/public/src/utils.js index de4b7d429b2bf..15a32f90942a2 100644 --- a/src/legacy/core_plugins/console/public/src/utils.js +++ b/src/legacy/core_plugins/console/public/src/utils.js @@ -70,10 +70,8 @@ utils.expandLiteralStrings = function (data) { return data.replace(/("(?:\\"|[^"])*?")/g, function (match, string) { // expand things with two slashes or more if (string.split(/\\./).length > 2) { - string = JSON.parse(string) - .replace('^s*\n', '') - .replace('\ns*^', ''); - return '"""' + string + '"""'; + string = JSON.parse(string).replace('^\s*\n', '').replace('\n\s*^', ''); + return '"""' + string + '"""'; } else { return string; } From ff8b67eb940763842e7e581976f6db858e445aa2 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Fri, 6 Sep 2019 11:06:51 +0200 Subject: [PATCH 06/10] Handle some more tricky cases, but make sure to preserve newlines. Update tests for new cases too --- .../core_plugins/console/public/src/utils.js | 2 +- .../tests/src/utils_string_expanding.txt | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/legacy/core_plugins/console/public/src/utils.js b/src/legacy/core_plugins/console/public/src/utils.js index 15a32f90942a2..c1b014f89f9a4 100644 --- a/src/legacy/core_plugins/console/public/src/utils.js +++ b/src/legacy/core_plugins/console/public/src/utils.js @@ -71,7 +71,7 @@ utils.expandLiteralStrings = function (data) { // expand things with two slashes or more if (string.split(/\\./).length > 2) { string = JSON.parse(string).replace('^\s*\n', '').replace('\n\s*^', ''); - return '"""' + string + '"""'; + return '"""' + string + '"""'; } else { return string; } diff --git a/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt b/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt index 4f3abf177490a..e0ced2f1ae8c4 100644 --- a/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt +++ b/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt @@ -2,20 +2,20 @@ Scripts in requests ------------------------------------- { - "f": { "script" : { "source": "\ntest\ntest\\2\n" } }, - "g": { "script" : "second + \"\\\";" }, - "f": "short with \\", - "h": 1, + "f": { "script": { "source": "\ntest\ntest\\\\\\\\\\\\\\\\2\n" } }, + "g": { "script": "second + \"\\\";" }, + "a": "short with \\", + "\\\\h": 1, "script": "a + 2" } ------------------------------------- { - "f": { "script" : { "source": """ + "f": { "script": { "source": """ test -test\2 +test\\\\\\\\2 """ } }, - "g": { "script" : """second + "\";""" }, - "f": "short with \\", - "h": 1, + "g": { "script": """second + "\";""" }, + "a": "short with \\", + "\\\\h": 1, "script": "a + 2" } From 0c6f7175035de656b9d133c2bdc55a3ecaa4ebbb Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Fri, 6 Sep 2019 12:33:52 +0200 Subject: [PATCH 07/10] Make regex more specific Expanded tests --- .../core_plugins/console/public/src/utils.js | 20 +++++++++++++--- .../tests/src/utils_string_expanding.txt | 23 ++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/legacy/core_plugins/console/public/src/utils.js b/src/legacy/core_plugins/console/public/src/utils.js index c1b014f89f9a4..73516b32b85c9 100644 --- a/src/legacy/core_plugins/console/public/src/utils.js +++ b/src/legacy/core_plugins/console/public/src/utils.js @@ -66,10 +66,24 @@ utils.collapseLiteralStrings = function (data) { return splitData.join(''); }; +/* + The following regex describes global match on: + 1. one colon followed by any number of space characters (positive lookahead, non-capturing) + 2. one double quote (indicating the start of a JSON value, i.e., we are not matching JSON keys). + 3. greedily match any non double quote and non newline char OR any escaped double quote char (non-capturing). + 4. handle a special case where an escaped slash may be the last character + 5. one double quote + + For instance: `: "some characters \" here"` + Will expand to: `"""some characters " here"""` + + */ + +const LITERAL_STRING_CANDIDATES = /((?<=:[\s\r\n]*)(? 2) { + return data.replace(LITERAL_STRING_CANDIDATES, function (match, string) { + if (string.match(/\\./)) { string = JSON.parse(string).replace('^\s*\n', '').replace('\n\s*^', ''); return '"""' + string + '"""'; } else { diff --git a/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt b/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt index e0ced2f1ae8c4..34bf0f3bc20e7 100644 --- a/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt +++ b/src/legacy/core_plugins/console/public/tests/src/utils_string_expanding.txt @@ -15,7 +15,28 @@ test test\\\\\\\\2 """ } }, "g": { "script": """second + "\";""" }, - "a": "short with \\", + "a": """short with \""", "\\\\h": 1, "script": "a + 2" } +========== +Preserve triple quotes +------------------------------------- +{ + "content\\\": "tri\"ple", +} +------------------------------------- +{ + "content\\\": """tri"ple""", +} +========== +Correctly parse with JSON embedded inside values +------------------------------------- +{ + "content\\\\": " { \"json\": \"inside\\\\\" ok! 1\n \" } " +} +------------------------------------- +{ + "content\\\\": """ { "json": "inside\\" ok! 1 + " } """ +} From 9cf3b03a2c4ada7dd8f364de923b6d2144ddf7d4 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Fri, 6 Sep 2019 12:57:27 +0200 Subject: [PATCH 08/10] Now make it x-browser --- src/legacy/core_plugins/console/public/src/utils.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/legacy/core_plugins/console/public/src/utils.js b/src/legacy/core_plugins/console/public/src/utils.js index 73516b32b85c9..380db58e13446 100644 --- a/src/legacy/core_plugins/console/public/src/utils.js +++ b/src/legacy/core_plugins/console/public/src/utils.js @@ -79,13 +79,16 @@ utils.collapseLiteralStrings = function (data) { */ -const LITERAL_STRING_CANDIDATES = /((?<=:[\s\r\n]*)(? Date: Fri, 6 Sep 2019 13:02:23 +0200 Subject: [PATCH 09/10] Slight refactor and renaming --- src/legacy/core_plugins/console/public/src/utils.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/legacy/core_plugins/console/public/src/utils.js b/src/legacy/core_plugins/console/public/src/utils.js index 380db58e13446..ddc0712881103 100644 --- a/src/legacy/core_plugins/console/public/src/utils.js +++ b/src/legacy/core_plugins/console/public/src/utils.js @@ -83,12 +83,15 @@ const LITERAL_STRING_CANDIDATES = /((:[\s\r\n]*)([^\\])"(\\"|[^"\n])*\\?")/g; utils.expandLiteralStrings = function (data) { return data.replace(LITERAL_STRING_CANDIDATES, function (match, string) { + // Expand to triple quotes if there are _any_ slashes if (string.match(/\\./)) { const firstDoubleQuoteIdx = string.indexOf('"'); - const firstPart = string.slice(0, firstDoubleQuoteIdx); - let secondPart = string.slice(firstDoubleQuoteIdx, string.length); - secondPart = JSON.parse(secondPart).replace('^\s*\n', '').replace('\n\s*^', ''); - return `${firstPart}"""${secondPart}"""`; + const colonAndAnySpacing = string.slice(0, firstDoubleQuoteIdx); + const rawStringifiedValue = string.slice(firstDoubleQuoteIdx, string.length); + const jsonValue = JSON.parse(rawStringifiedValue) + .replace('^\s*\n', '') + .replace('\n\s*^', ''); + return `${colonAndAnySpacing}"""${jsonValue}"""`; } else { return string; } From e543d6fb95e3c7342e1dc32f631dcf8794780abd Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Fri, 6 Sep 2019 13:07:47 +0200 Subject: [PATCH 10/10] Update comment --- src/legacy/core_plugins/console/public/src/utils.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/legacy/core_plugins/console/public/src/utils.js b/src/legacy/core_plugins/console/public/src/utils.js index ddc0712881103..5b6bd1646c300 100644 --- a/src/legacy/core_plugins/console/public/src/utils.js +++ b/src/legacy/core_plugins/console/public/src/utils.js @@ -68,14 +68,14 @@ utils.collapseLiteralStrings = function (data) { /* The following regex describes global match on: - 1. one colon followed by any number of space characters (positive lookahead, non-capturing) - 2. one double quote (indicating the start of a JSON value, i.e., we are not matching JSON keys). + 1. one colon followed by any number of space characters + 2. one double quote (not escaped, special case for JSON in JSON). 3. greedily match any non double quote and non newline char OR any escaped double quote char (non-capturing). 4. handle a special case where an escaped slash may be the last character 5. one double quote For instance: `: "some characters \" here"` - Will expand to: `"""some characters " here"""` + Will match and be expanded to: `"""some characters " here"""` */