Fixed: [JSON backslash string decoding/encoding not correct](#453)

Added unicode BMP support for JSON strings Test: encoding/decoding tests for UTF-8
clicon · Sep 19, 2023 · 45f41e3 · 45f41e3
1 parent 1a43a32
commit 45f41e3
Show file tree

Hide file tree

Showing 12 changed files with 254 additions and 35 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -73,6 +73,7 @@ Developers may need to change their code
 
 ### Minor features
 
+* JSON: Added unicode BMP support for unicode strings as part of fixing (https://github.com/clicon/clixon/issues/453)
 * Example cli pipe grep command quotes vertical bar for OR function
 * Added: [Feature request: node's alias for CLI](https://github.com/clicon/clixon/issues/434)
    * Note: "Skip" is for all nodes, but "Alias" is only for leafs
@@ -85,7 +86,9 @@ Developers may need to change their code
 
 ### Corrected Bugs
 
+* Fixed: [JSON backslash string decoding/encoding not correct](https://github.com/clicon/clixon/issues/453)
 * Fixed: [CLI show config | display <format> exits over mountpoints with large YANGs](https://github.com/clicon/clixon-controller/issues/39)
+  * JSON string fixed according to RFC 8259: encoding/decoding of escape as defined in Section 8
   * No need to bind for xml and json, only cli and text
 * Fixed several issues with extra-config files, including overwriting of structured sub-configs
   * including `<restconf>`and m̀ <autoconf>`

diff --git a/apps/cli/cli_pipe.c b/apps/cli/cli_pipe.c
@@ -297,6 +297,7 @@ pipe_showas_fn(clicon_handle h,
     switch (format){
     case FORMAT_CLI:
     case FORMAT_TEXT:
+    case FORMAT_JSON:
         /* Requires binding. Note binding over mountpoints can cause rpc: extra latency */
         if ((ret = xml_bind_yang(h, xt, YB_MODULE, yspec, &xerr)) < 0)
             goto done;

diff --git a/lib/clixon/clixon_json.h b/lib/clixon/clixon_json.h
@@ -33,9 +33,9 @@
   ***** END LICENSE BLOCK *****
 
  * JSON support functions.
- * JSON syntax is according to:
- * http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
- * RFC 7951 JSON Encoding of Data Modeled with YANG
+ * @see http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
+ *  and RFC 7951 JSON Encoding of Data Modeled with YANG
+ *  and RFC 8259 The JavaScript Object Notation (JSON) Data Interchange Format
  */
 #ifndef _CLIXON_JSON_H
 #define _CLIXON_JSON_H

diff --git a/lib/clixon/clixon_string.h b/lib/clixon/clixon_string.h
@@ -106,6 +106,7 @@ int    nodeid_split(char *nodeid, char **prefix, char **id);
 char  *clixon_trim(char *str);
 char  *clixon_trim2(char *str, char *trims);
 int    clicon_strcmp(char *s1, char *s2);
+int    clixon_unicode2utf8(char *ucstr, char *utfstr, size_t utflen);
 
 #ifndef HAVE_STRNDUP
 char *clicon_strndup (const char *, size_t);

diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c
@@ -33,9 +33,9 @@
   ***** END LICENSE BLOCK *****
 
  * JSON support functions.
- * JSON syntax is according to:
- * http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
- * RFC 7951 JSON Encoding of Data Modeled with YANG
+ * @see http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
+ *  and RFC 7951 JSON Encoding of Data Modeled with YANG
+ *  and RFC 8259 The JavaScript Object Notation (JSON) Data Interchange Format
  * XXX: The complexity of xml2json1_cbuf() mapping from internal cxobj structure to JSON output
  * needs a rewrite due to complexity of lists/leaf-lists/null-values, etc.
  */
@@ -107,6 +107,7 @@ enum childtype{
 };
 
 /*! x is element and has exactly one child which in turn has none 
+ *
  * remove attributes from x
  * @see tleaf in clixon_xml_map.c
  */
@@ -241,6 +242,7 @@ array_eval(cxobj *xprev,
 }
 
 /*! Escape a json string as well as decode xml cdata
+ *
  * @param[out] cb   cbuf   (encoded)
  * @param[in]  str  string (unencoded)
  */
@@ -255,15 +257,27 @@ json_str_escape_cdata(cbuf *cb,
     len = strlen(str);
     for (i=0; i<len; i++)
         switch (str[i]){
-        case '\n':
-            cprintf(cb, "\\n");
-            break;
         case '\"':
             cprintf(cb, "\\\"");
             break;
         case '\\':
             cprintf(cb, "\\\\");
             break;
+        case '\b':
+            cprintf(cb, "\\b");
+            break;
+        case '\f':
+            cprintf(cb, "\\f");
+            break;
+        case '\n':
+            cprintf(cb, "\\n");
+            break;
+        case '\r':
+            cprintf(cb, "\\r");
+            break;
+        case '\t':
+            cprintf(cb, "\\t");
+            break;
         default: /* fall thru */
             cprintf(cb, "%c", str[i]);
             break;
@@ -274,6 +288,7 @@ json_str_escape_cdata(cbuf *cb,
 }
 
 /*! Decode types from JSON to XML identityrefs
+ *
  * Assume an xml tree where prefix:name have been split into "module":"name"
  * In other words, from JSON RFC7951 to XML namespace trees
  * @param[in]     x     XML tree. Must be yang populated.
@@ -433,6 +448,7 @@ json2xml_decode(cxobj     *x,
 }
 
 /*! Encode leaf/leaf_list identityref type from XML to JSON
+ *
  * @param[in]     x    XML body node
  * @param[in]     body body string
  * @param[in]     ys   Yang spec of parent
@@ -497,6 +513,7 @@ xml2json_encode_identityref(cxobj     *xb,
 }
 
 /*! Encode leaf/leaf_list types from XML to JSON
+ *
  * @param[in]     xb   XML body
  * @param[in]     xp   XML parent
  * @param[in]     yp   Yang spec of parent
@@ -663,6 +680,7 @@ nullchild(cbuf      *cb,
 }
 
 /*! Encode json metadata
+ *
  * This function could be more general, code based on two examples:
  * 1) ietf-list-pagination:remaining
  *  {
@@ -773,6 +791,7 @@ xml2json_encode_attr(cxobj     *xa,
 }
 
 /*! Do the actual work of translating XML to JSON 
+ *
  * @param[out]   cb        Cligen text buffer containing json on exit
  * @param[in]    x         XML tree structure containing XML to translate
  * @param[in]    arraytype Does x occur in a array (of its parent) and how?
@@ -1142,6 +1161,7 @@ clixon_json2cbuf(cbuf  *cb,
 }
 
 /*! Translate a vector of xml objects to JSON Cligen buffer.
+ *
  * This is done by adding a top pseudo-object, and add the vector as subs,
  * and then not printing the top pseudo-object using the 'flat' option.
  * @param[out] cb     Cligen buffer to write to
@@ -1224,6 +1244,7 @@ xml2json_cbuf_vec(cbuf      *cb,
 }
 
 /*! Translate from xml tree to JSON and print to file using a callback
+ *
  * @param[in]  f       File to print to
  * @param[in]  xn      XML tree to translate from
  * @param[in]  pretty  Set if output is pretty-printed
@@ -1280,6 +1301,7 @@ json_print(FILE  *f,
 }
 
 /*! Translate a vector of xml objects to JSON File.
+ *
  * This is done by adding a top pseudo-object, and add the vector as subs,
  * and then not pritning the top pseudo-.object using the 'flat' option.
  * @param[out] cb     Cligen buffer to write to
@@ -1320,6 +1342,7 @@ xml2json_vec(FILE             *f,
 }
 
 /*! Translate from JSON module:name to XML default ns: xmlns="uri" recursively
+ *
  * Assume an xml tree where prefix:name have been split into "module":"name"
  * In other words, from JSON to XML namespace trees
  * 

diff --git a/lib/src/clixon_json_parse.l b/lib/src/clixon_json_parse.l
@@ -32,6 +32,9 @@
 
   ***** END LICENSE BLOCK *****
 
+ * @see http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
+ *  and RFC 7951 JSON Encoding of Data Modeled with YANG
+ *  and RFC 8259 The JavaScript Object Notation (JSON) Data Interchange Format
  */
 
 %{
@@ -51,6 +54,7 @@
 #include "clixon_queue.h"
 #include "clixon_hash.h"
 #include "clixon_handle.h"
+#include "clixon_string.h"
 #include "clixon_yang.h"
 #include "clixon_log.h"
 #include "clixon_xml.h"
@@ -78,10 +82,12 @@ digit [0-9]
 integer {digit}+
 real ({digit}+[.]{digit}*)|({digit}*[.]{digit}+)
 exp ({integer}|{real})[eE][+-]{integer}
+hex [A-Fa-f0-9]
 
 %x START
 %s STRING
 %s ESCAPE
+%s HEXDIG
 
 %%
 
@@ -103,14 +109,31 @@ exp ({integer}|{real})[eE][+-]{integer}
 <START>.                { return -1; }
 <STRING>\"              { BEGIN(START); return J_DQ; }
 <STRING>\\              { BEGIN(ESCAPE); }
-<STRING>\n              { _JY->jy_linenum++; 
-                          clixon_json_parselval.string = strdup(yytext); 
-                          return J_CHAR;}
-<STRING>.               { clixon_json_parselval.string = strdup(yytext);
-                          return J_CHAR;}
-<ESCAPE>.               { BEGIN(STRING); 
-                          clixon_json_parselval.string = strdup(yytext); 
-                          return J_CHAR; }
+<STRING>[^\"\\\b\f\n\r\t]+ { BEGIN(STRING); clixon_json_parselval.string = yytext; return J_STRING; }
+<STRING>\n              { return -1; }
+<STRING>.               { return -1; }
+<ESCAPE>\"              { BEGIN(STRING); clixon_json_parselval.string = yytext; return J_STRING; }
+<ESCAPE>\\              { BEGIN(STRING); clixon_json_parselval.string = yytext; return J_STRING; }
+<ESCAPE>\/              { BEGIN(STRING); clixon_json_parselval.string = yytext; return J_STRING; }
+<ESCAPE>b               { BEGIN(STRING); clixon_json_parselval.string = "\b";   return J_STRING; }
+<ESCAPE>f               { BEGIN(STRING); clixon_json_parselval.string = "\f";   return J_STRING; }
+<ESCAPE>n               { BEGIN(STRING); clixon_json_parselval.string = "\n";   return J_STRING; }
+<ESCAPE>r               { BEGIN(STRING); clixon_json_parselval.string = "\r";   return J_STRING; }
+<ESCAPE>t               { BEGIN(STRING); clixon_json_parselval.string = "\t";   return J_STRING; }
+<ESCAPE>u               { BEGIN(HEXDIG); }
+<ESCAPE>\n              { return -1; }
+<ESCAPE>.               { return -1; }
+<HEXDIG>{hex}{hex}{hex}{hex} { 
+                               char buf[5] = {0, };
+                               BEGIN(STRING); 
+                               if (clixon_unicode2utf8(yytext, buf, 5) < 0)
+                                  return -1;
+                               strncpy(yytext, buf, 4);
+                               clixon_json_parselval.string = yytext; 
+                               return J_STRING;
+                              }
+<HEXDIG>\n              { return -1;}
+<HEXDIG>.               { return -1; }
 
 %%
 

diff --git a/lib/src/clixon_json_parse.y b/lib/src/clixon_json_parse.y
@@ -33,8 +33,9 @@
   ***** END LICENSE BLOCK *****
 
  * JSON Parser
- * From http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
- * And RFC7951 JSON Encoding of Data Modeled with YANG
+ * @see http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
+ *  and RFC 7951 JSON Encoding of Data Modeled with YANG
+ *  and RFC 8259 The JavaScript Object Notation (JSON) Data Interchange Format
 
 Structural tokens: 
  [   left square bracket 
@@ -87,7 +88,7 @@ object.
 %token <string> J_NULL
 %token <string> J_EOF
 %token <string> J_DQ
-%token <string> J_CHAR
+%token <string> J_STRING
 %token <string> J_NUMBER
 
 %type <cbuf>      string
@@ -151,7 +152,7 @@ void
 clixon_json_parseerror(void *_jy,
                        char *s) 
 { 
-    clicon_err(OE_JSON, XMLPARSE_ERRNO, "json_parse: line %d: %s at or before: '%s'", 
+    clicon_err(OE_JSON, 0, "json_parse: line %d: %s at or before: '%s'", 
                _JY->jy_linenum ,
                s, 
                clixon_json_parsetext); 
@@ -312,12 +313,16 @@ string        : J_DQ ustring J_DQ { _PARSE_DEBUG("string->\" ustring \"");$$=$2;
               ;
 
 /* unquoted string: can be optimized by reading whole string in lex */
-ustring       : ustring J_CHAR 
+ustring       : ustring J_STRING
                      {
-                         cbuf_append_str($1,$2); $$=$1; free($2);
+                         cbuf_append_str($1,$2); $$=$1;
                      }
-              | J_CHAR 
-              { cbuf *cb = cbuf_new(); cbuf_append_str(cb,$1); $$=cb; free($1);} 
+              | J_STRING
+                     {
+                         cbuf *cb = cbuf_new();
+                         cbuf_append_str(cb,$1);
+                         $$=cb;
+                     } 
               ;
 
 number        : J_NUMBER { $$ = $1; }