diff --git a/CHANGELOG.md b/CHANGELOG.md index 78575006c..52ee775bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ Developers may need to change their code ### Minor features +* JSON: Added unicode BMP support for unicode strings as part of fixing (https://github.com/clicon/clixon/issues/453) * Example cli pipe grep command quotes vertical bar for OR function * Added: [Feature request: node's alias for CLI](https://github.com/clicon/clixon/issues/434) * Note: "Skip" is for all nodes, but "Alias" is only for leafs @@ -85,7 +86,9 @@ Developers may need to change their code ### Corrected Bugs +* Fixed: [JSON backslash string decoding/encoding not correct](https://github.com/clicon/clixon/issues/453) * Fixed: [CLI show config | display exits over mountpoints with large YANGs](https://github.com/clicon/clixon-controller/issues/39) + * JSON string fixed according to RFC 8259: encoding/decoding of escape as defined in Section 8 * No need to bind for xml and json, only cli and text * Fixed several issues with extra-config files, including overwriting of structured sub-configs * including ``and m̀ ` diff --git a/apps/cli/cli_pipe.c b/apps/cli/cli_pipe.c index 08c9b3eda..c857dfd8f 100644 --- a/apps/cli/cli_pipe.c +++ b/apps/cli/cli_pipe.c @@ -297,6 +297,7 @@ pipe_showas_fn(clicon_handle h, switch (format){ case FORMAT_CLI: case FORMAT_TEXT: + case FORMAT_JSON: /* Requires binding. Note binding over mountpoints can cause rpc: extra latency */ if ((ret = xml_bind_yang(h, xt, YB_MODULE, yspec, &xerr)) < 0) goto done; diff --git a/lib/clixon/clixon_json.h b/lib/clixon/clixon_json.h index 5758aa6b9..3477df81a 100644 --- a/lib/clixon/clixon_json.h +++ b/lib/clixon/clixon_json.h @@ -33,9 +33,9 @@ ***** END LICENSE BLOCK ***** * JSON support functions. - * JSON syntax is according to: - * http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf - * RFC 7951 JSON Encoding of Data Modeled with YANG + * @see http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf + * and RFC 7951 JSON Encoding of Data Modeled with YANG + * and RFC 8259 The JavaScript Object Notation (JSON) Data Interchange Format */ #ifndef _CLIXON_JSON_H #define _CLIXON_JSON_H diff --git a/lib/clixon/clixon_string.h b/lib/clixon/clixon_string.h index 73956db47..c36d57eb9 100644 --- a/lib/clixon/clixon_string.h +++ b/lib/clixon/clixon_string.h @@ -106,6 +106,7 @@ int nodeid_split(char *nodeid, char **prefix, char **id); char *clixon_trim(char *str); char *clixon_trim2(char *str, char *trims); int clicon_strcmp(char *s1, char *s2); +int clixon_unicode2utf8(char *ucstr, char *utfstr, size_t utflen); #ifndef HAVE_STRNDUP char *clicon_strndup (const char *, size_t); diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c index 9c6bb5c8d..01865073d 100644 --- a/lib/src/clixon_json.c +++ b/lib/src/clixon_json.c @@ -33,9 +33,9 @@ ***** END LICENSE BLOCK ***** * JSON support functions. - * JSON syntax is according to: - * http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf - * RFC 7951 JSON Encoding of Data Modeled with YANG + * @see http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf + * and RFC 7951 JSON Encoding of Data Modeled with YANG + * and RFC 8259 The JavaScript Object Notation (JSON) Data Interchange Format * XXX: The complexity of xml2json1_cbuf() mapping from internal cxobj structure to JSON output * needs a rewrite due to complexity of lists/leaf-lists/null-values, etc. */ @@ -107,6 +107,7 @@ enum childtype{ }; /*! x is element and has exactly one child which in turn has none + * * remove attributes from x * @see tleaf in clixon_xml_map.c */ @@ -241,6 +242,7 @@ array_eval(cxobj *xprev, } /*! Escape a json string as well as decode xml cdata + * * @param[out] cb cbuf (encoded) * @param[in] str string (unencoded) */ @@ -255,15 +257,27 @@ json_str_escape_cdata(cbuf *cb, len = strlen(str); for (i=0; i. { return -1; } \" { BEGIN(START); return J_DQ; } \\ { BEGIN(ESCAPE); } -\n { _JY->jy_linenum++; - clixon_json_parselval.string = strdup(yytext); - return J_CHAR;} -. { clixon_json_parselval.string = strdup(yytext); - return J_CHAR;} -. { BEGIN(STRING); - clixon_json_parselval.string = strdup(yytext); - return J_CHAR; } +[^\"\\\b\f\n\r\t]+ { BEGIN(STRING); clixon_json_parselval.string = yytext; return J_STRING; } +\n { return -1; } +. { return -1; } +\" { BEGIN(STRING); clixon_json_parselval.string = yytext; return J_STRING; } +\\ { BEGIN(STRING); clixon_json_parselval.string = yytext; return J_STRING; } +\/ { BEGIN(STRING); clixon_json_parselval.string = yytext; return J_STRING; } +b { BEGIN(STRING); clixon_json_parselval.string = "\b"; return J_STRING; } +f { BEGIN(STRING); clixon_json_parselval.string = "\f"; return J_STRING; } +n { BEGIN(STRING); clixon_json_parselval.string = "\n"; return J_STRING; } +r { BEGIN(STRING); clixon_json_parselval.string = "\r"; return J_STRING; } +t { BEGIN(STRING); clixon_json_parselval.string = "\t"; return J_STRING; } +u { BEGIN(HEXDIG); } +\n { return -1; } +. { return -1; } +{hex}{hex}{hex}{hex} { + char buf[5] = {0, }; + BEGIN(STRING); + if (clixon_unicode2utf8(yytext, buf, 5) < 0) + return -1; + strncpy(yytext, buf, 4); + clixon_json_parselval.string = yytext; + return J_STRING; + } +\n { return -1;} +. { return -1; } %% diff --git a/lib/src/clixon_json_parse.y b/lib/src/clixon_json_parse.y index a570c61f0..8e815069a 100644 --- a/lib/src/clixon_json_parse.y +++ b/lib/src/clixon_json_parse.y @@ -33,8 +33,9 @@ ***** END LICENSE BLOCK ***** * JSON Parser - * From http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf - * And RFC7951 JSON Encoding of Data Modeled with YANG + * @see http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf + * and RFC 7951 JSON Encoding of Data Modeled with YANG + * and RFC 8259 The JavaScript Object Notation (JSON) Data Interchange Format Structural tokens: [ left square bracket @@ -87,7 +88,7 @@ object. %token J_NULL %token J_EOF %token J_DQ -%token J_CHAR +%token J_STRING %token J_NUMBER %type string @@ -151,7 +152,7 @@ void clixon_json_parseerror(void *_jy, char *s) { - clicon_err(OE_JSON, XMLPARSE_ERRNO, "json_parse: line %d: %s at or before: '%s'", + clicon_err(OE_JSON, 0, "json_parse: line %d: %s at or before: '%s'", _JY->jy_linenum , s, clixon_json_parsetext); @@ -312,12 +313,16 @@ string : J_DQ ustring J_DQ { _PARSE_DEBUG("string->\" ustring \"");$$=$2; ; /* unquoted string: can be optimized by reading whole string in lex */ -ustring : ustring J_CHAR +ustring : ustring J_STRING { - cbuf_append_str($1,$2); $$=$1; free($2); + cbuf_append_str($1,$2); $$=$1; } - | J_CHAR - { cbuf *cb = cbuf_new(); cbuf_append_str(cb,$1); $$=cb; free($1);} + | J_STRING + { + cbuf *cb = cbuf_new(); + cbuf_append_str(cb,$1); + $$=cb; + } ; number : J_NUMBER { $$ = $1; } diff --git a/lib/src/clixon_string.c b/lib/src/clixon_string.c index 737987d31..a208477eb 100644 --- a/lib/src/clixon_string.c +++ b/lib/src/clixon_string.c @@ -1051,6 +1051,105 @@ clicon_strcmp(char *s1, return strcmp(s1, s2); } +/*! Translate from unicode in hex form to utf-8 + * + * @param[in] uc16 Unicode as 2-byte hex int + * @param[out] utf8str UTF-8 string + * @param[out] utflen Length utf string + * @retval 0 OK + * @retval -1 Error + */ +static int +clixon_unicode2utf8_one(uint16_t uc16, + char *utfstr, + size_t utflen) +{ + int retval = -1; + + if (utflen < 5){ + clicon_err(OE_UNIX, EINVAL, "Length of utfstr is not >=4"); + goto done; + } + if (uc16<0x80) + *utfstr++=uc16; + else if (uc16<0x800){ + *utfstr++=192+uc16/64; + *utfstr++=128+uc16%64; + } + else if (uc16-0xd800u<0x800){ + clicon_err(OE_UNIX, EINVAL, "unicode2utf error"); + goto done; + } + else if (uc16<0x10000) { + *utfstr++=224+uc16/4096; + *utfstr++=128+uc16/64%64; + *utfstr++=128+uc16%64; + } + else if (uc16<0x110000) { + *utfstr++=240+uc16/262144; + *utfstr++=128+uc16/4096%64; + *utfstr++=128+uc16/64%64; + *utfstr++=128+uc16%64; + } + else{ + clicon_err(OE_UNIX, EINVAL, "unicode2utf error"); + goto done; + } + *utfstr++=0; + retval = 0; + done: + return retval; +} + +/*! Translate unicode hexstring on the form "ABCD" to UTF-8 in string form + * + * @param[in] unicode Unicode as string of 2-byte hex codes + * @param[out] utf8 UTF-8 character string must be length >=5 + * @retval 0 OK + * @retval -1 Error + */ +int +clixon_unicode2utf8(char *ucstr, + char *utfstr, + size_t utflen) +{ + int retval = -1; + size_t len; + int i; + char c; + int j; + uint16_t uc16 = 0; + + if (ucstr == NULL || utfstr == NULL){ + clicon_err(OE_UNIX, EINVAL, "input param is NULL"); + goto done; + } + if ((len = strlen(ucstr)) != 4){ + clicon_err(OE_UNIX, EINVAL, "Length of ucstr is not 4"); + goto done; + } + for (i=0; iruled over the shores of the Hreiðsea" + +# see https://github.com/clicon/clixon/issues/453 +JSON="{\"text\":\"one +two\"}" +new "json not escaped \n expect fail" +expecteofx "$clixon_util_json -j" 255 "$JSON" 2> /dev/null + +new "json not escaped \" expect fail" +expecteofx "$clixon_util_json -j" 255 "quote:\"" 2> /dev/null + +JSON='{"text":"cr:\nquote:\"tab:\tend"}' +new "json escaping \n\"\t to json" +expecteofx "$clixon_util_json -j" 0 "$JSON" "$JSON" + +new "json escaping \n\"\t to xml" +expecteofx "$clixon_util_json" 0 "$JSON" "quote:\"tab: end" + +JSON='{"text":"bmp:\u005E"}' +new "json escaping \u BMP circumflex" +#expecteofx "$clixon_util_json -j -D $DBG" 0 "$JSON" '{"text":"bmp:^"}' + +JSON='{"text":"bmp:\u00F0"}' +new "json escaping \u BMP latin eth" +expecteofx "$clixon_util_json -j -D $DBG" 0 "$JSON" '{"text":"bmp:ð"}' + +JSON='{"text":"bmp:\uFAIL"}' +new "json escaping unicode BMP fail" +expecteofx "$clixon_util_json -j -D $DBG" 255 "$JSON" 2> /dev/null + rm -rf $dir new "endtest" diff --git a/test/test_restconf.sh b/test/test_restconf.sh index ec8344b57..d498ccfcc 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -294,6 +294,7 @@ function testrun() new "start restconf daemon" # inline of start_restconf, cant make quotes to work echo "sudo -u $wwwstartuser -s $clixon_restconf $RCLOG -D $DBG -f $cfg -R $RESTCONFIG1" + STTYSETTINGS=$(stty -g) # reset in wait_restconf sudo -u $wwwstartuser -s $clixon_restconf $RCLOG -D $DBG -f $cfg -R "$RESTCONFIG1" /dev/null & if [ $? -ne 0 ]; then err1 "expected 0" "$?" @@ -308,14 +309,13 @@ function testrun() else HVER=2 fi + new "wait restconf" wait_restconf - + new "restconf root discovery. RFC 8040 3.1 (xml+xrd)" - echo "curl $CURLOPTS -X GET $proto://$addr/.well-known/host-meta" expectpart "$(curl $CURLOPTS -X GET $proto://$addr/.well-known/host-meta)" 0 "HTTP/$HVER 200" "" "" "" - echo "fcgi or native+http/1 or native+http/1+http/2" new "restconf GET http/1.1" expectpart "$(curl $CURLOPTS --http1.1 -X GET $proto://$addr/.well-known/host-meta)" 0 'HTTP/1.1 200 OK' "" "" "" @@ -334,7 +334,7 @@ function testrun() new "restconf GET https/2 prior-knowledge" expectpart "$(curl $CURLOPTS --http2-prior-knowledge -X GET $proto://$addr/.well-known/host-meta)" 0 "HTTP/$HVER 200" "" "" "" fi - + # Wrong protocol http when https or vice versa if [ $proto = http ]; then # see (2) https to http port in restconf_main_native.c new "Wrong proto=https on http port, expect err 35 wrong version number" @@ -354,10 +354,8 @@ function testrun() wait_restconf new "restconf root discovery. RFC 8040 3.1 (xml+xrd)" - echo "curl $CURLOPTS -X GET $proto://$addr/.well-known/host-meta" expectpart "$(curl $CURLOPTS -X GET $proto://$addr/.well-known/host-meta)" 0 "HTTP/$HVER 200" "" "" "" - echo "native + http/2 only" # Important here is robustness of restconf daemon, not a meaningful reply if [ $proto = http ]; then # see (2) https to http port in restconf_main_native.c # http protocol mismatch can just close the socket if assumed its http/2 @@ -405,10 +403,8 @@ function testrun() wait_restconf new "restconf root discovery. RFC 8040 3.1 (xml+xrd)" - echo "curl $CURLOPTS -X GET $proto://$addr/.well-known/host-meta" expectpart "$(curl $CURLOPTS -X GET $proto://$addr/.well-known/host-meta)" 0 "HTTP/$HVER 200" "" "" "" - echo "fcgi or native+http/1 or native+http/1+http/2" if [ "${WITH_RESTCONF}" = "native" ]; then # XXX does not work with nginx new "restconf GET http/1.0 - returns 1.0" expectpart "$(curl $CURLOPTS --http1.0 -X GET $proto://$addr/.well-known/host-meta)" 0 'HTTP/1.0 200 OK' "" "" "" @@ -439,7 +435,6 @@ function testrun() expectpart "$(curl $CURLOPTS -X GET http://$addr:443/.well-known/host-meta)" 0 "HTTP/" "400" fi fi # HTTP/2 - # Exact match new "restconf get restconf resource. RFC 8040 3.3 (json)" expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-data+json" $proto://$addr/restconf)" 0 "HTTP/$HVER 200" '{"ietf-restconf:restconf":{"data":{},"operations":{},"yang-library-version":"2019-01-04"}}' @@ -653,6 +648,24 @@ function testrun() new "restconf Add subtree with too many keys (expected error)" expectpart "$(curl $CURLOPTS -X PUT -H "Content-Type: application/yang-data+json" -d '{"ietf-interfaces:interface":{"name":"eth/0/0","type":"clixon-example:eth","enabled":true}}' $proto://$addr/restconf/data/ietf-interfaces:interfaces/interface=a,b)" 0 "HTTP/$HVER 400" '{"ietf-restconf:errors":{"error":{"error-type":"rpc","error-tag":"malformed-message","error-severity":"error","error-message":"List key interface length mismatch"}}}' +cat < $dir/input.json +{"clixon-example:parameter":{"name":"x","value":"foo +bar"}} +EOF + new "restconf JSON escape encoding with explicit \n expect fail" + expectpart "$(curl $CURLOPTS -X POST -H "Accept: application/yang-data+json" -H "Content-Type: application/yang-data+json" --data-binary @$dir/input.json $proto://$addr/restconf/data/clixon-example:table)" 0 "HTTP/$HVER 400" "malformed-message" + + JSON='{"clixon-example:parameter":[{"name":"x","value":"foo\nbar"}]}' + + new "restconf JSON escape encoding" + expectpart "$(curl $CURLOPTS -X POST -H "Accept: application/yang-data+json" -H "Content-Type: application/yang-data+json" --data-binary "$JSON" $proto://$addr/restconf/data/clixon-example:table)" 0 "HTTP/$HVER 201" "Location: $proto://$addr/restconf/data/clixon-example:table/parameter=x" + + new "Check restconf JSON escape encoding" + expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-data+json" $proto://$addr/restconf/data/clixon-example:table/parameter=x)" 0 "HTTP/$HVER 200" '{"clixon-example:parameter":\[{"name":"x","value":"foo\\nbar"}\]}' + + new "Check restconf JSON escape encoding in XML" + expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-data+xml" $proto://$addr/restconf/data/clixon-example:table/parameter=x)" 0 "HTTP/$HVER 200" "xfoo" "bar" + if [ $RC -ne 0 ]; then new "Kill restconf daemon" stop_restconf diff --git a/test/test_xml.sh b/test/test_xml.sh index f5634873b..c9003e65e 100755 --- a/test/test_xml.sh +++ b/test/test_xml.sh @@ -212,6 +212,19 @@ EOF ) expecteof "$clixon_util_xml -o" 0 "$XML" '^Cheaper by the Dozen1568491379$' +XML=$(cat < + +Theodoric the bold +chief of sea-warriors +ruled over the shores of the Hreiðsea + +EOF + ) + +new "utf-8 string" +expecteof "$clixon_util_xml -o" 0 "$XML" "^ruled over the shores of the Hreiðsea$" + rm -rf $dir new "endtest" diff --git a/util/clixon_util_json.c b/util/clixon_util_json.c index 585f6843c..386c2b50a 100644 --- a/util/clixon_util_json.c +++ b/util/clixon_util_json.c @@ -33,7 +33,9 @@ ***** END LICENSE BLOCK ***** * JSON utility command - * http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf + * @see http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf + * and RFC 7951 JSON Encoding of Data Modeled with YANG + * and RFC 8259 The JavaScript Object Notation (JSON) Data Interchange Format */ #ifdef HAVE_CONFIG_H