diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b75c3347..d4f880df2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,12 @@ Users may have to change how they access the system * Revert the creators attribute feature introduced in 6.2. It is now obsoleted. It is replaced with a configured `creators` and user/application semantics +* New `clixon-lib@2024-01-01.yang` revision + * Replaced container creators to grouping/uses +* New `clixon-config@2024-01-01.yang` revision + * Marked as obsolete: + * `CLICON_DATASTORE_CACHE` Replaced with enhanced datastore read API + * `CLICON_NETCONF_CREATOR_ATTR` reverting 6.5 functionality ### C/CLI-API changes on existing features Developers may need to change their code diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index 00900e540..2f27a9f39 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -366,9 +366,6 @@ startup_validate(clixon_handle h, goto fail; } plugin_transaction_end_all(h, td); - /* Clear cached trees from default values and marking */ - if (xmldb_get0_clear(h, td->td_target) < 0) - goto done; if (xtr){ *xtr = td->td_target; td->td_target = NULL; @@ -376,7 +373,6 @@ startup_validate(clixon_handle h, retval = 1; done: if (td){ - xmldb_get0_free(h, &td->td_target); transaction_free(td); } return retval; @@ -421,10 +417,9 @@ startup_commit(clixon_handle h, /* After commit, make a post-commit call (sure that all plugins have committed) */ if (plugin_transaction_commit_done_all(h, td) < 0) goto done; - /* Clear cached trees from default values and marking */ - if (xmldb_get0_clear(h, td->td_target) < 0) + /* Remove global defaults and empty non-presence containers */ + if (xml_defaults_nopresence(td->td_target, 2) < 0) goto done; - /* [Delete and] create running db */ if (xmldb_exists(h, "running") == 1){ if (xmldb_delete(h, "running") != 0 && errno != ENOENT) @@ -454,7 +449,6 @@ startup_commit(clixon_handle h, if (td){ if (retval < 1) plugin_transaction_abort_all(h, td); - xmldb_get0_free(h, &td->td_target); transaction_free(td); } return retval; @@ -622,10 +616,6 @@ candidate_validate(clixon_handle h, goto done; goto fail; } - if (xmldb_get0_clear(h, td->td_src) < 0 || - xmldb_get0_clear(h, td->td_target) < 0) - goto done; - plugin_transaction_end_all(h, td); retval = 1; done: @@ -634,8 +624,6 @@ candidate_validate(clixon_handle h, if (td){ if (retval < 1) plugin_transaction_abort_all(h, td); - xmldb_get0_free(h, &td->td_target); - xmldb_get0_free(h, &td->td_src); transaction_free(td); } return retval; @@ -715,13 +703,6 @@ candidate_commit(clixon_handle h, /* After commit, make a post-commit call (sure that all plugins have committed) */ if (plugin_transaction_commit_done_all(h, td) < 0) goto done; - - /* Clear cached trees from default values and marking */ - if (xmldb_get0_clear(h, td->td_target) < 0) - goto done; - if (xmldb_get0_clear(h, td->td_src) < 0) - goto done; - /* 8. Success: Copy candidate to running */ if (xmldb_copy(h, db, "running") < 0) @@ -746,8 +727,6 @@ candidate_commit(clixon_handle h, if (td){ if (retval < 1) plugin_transaction_abort_all(h, td); - xmldb_get0_free(h, &td->td_target); - xmldb_get0_free(h, &td->td_src); transaction_free(td); } if (xret) @@ -1043,7 +1022,6 @@ from_client_restart_one(clixon_handle h, retval = 1; done: if (td){ - xmldb_get0_free(h, &td->td_target); transaction_free(td); } return retval; diff --git a/apps/backend/backend_startup.c b/apps/backend/backend_startup.c index 637f2d85b..d7e06d900 100644 --- a/apps/backend/backend_startup.c +++ b/apps/backend/backend_startup.c @@ -93,7 +93,6 @@ db_merge(clixon_handle h, /* Merge xml into db2. Without commit */ retval = xmldb_put(h, (char*)db2, OP_MERGE, xt, clicon_username_get(h), cbret); done: - xmldb_get0_free(h, &xt); return retval; } @@ -334,7 +333,6 @@ startup_extraxml(clixon_handle h, done: if (xt0) xml_free(xt0); - xmldb_get0_free(h, &xt); if (xmldb_delete(h, tmp_db) != 0 && errno != ENOENT) return -1; return retval; diff --git a/lib/clixon/clixon_datastore.h b/lib/clixon/clixon_datastore.h index 096d18d12..fd04348bf 100644 --- a/lib/clixon/clixon_datastore.h +++ b/lib/clixon/clixon_datastore.h @@ -53,8 +53,6 @@ int xmldb_get(clixon_handle h, const char *db, cvec *nsc, char *xpath, cxobj **x int xmldb_get0(clixon_handle h, const char *db, yang_bind yb, cvec *nsc, const char *xpath, int copy, withdefaults_type wdef, cxobj **xtop, modstate_diff_t *msd, cxobj **xerr); -int xmldb_get0_clear(clixon_handle h, cxobj *x); -int xmldb_get0_free(clixon_handle h, cxobj **xp); int xmldb_put(clixon_handle h, const char *db, enum operation_type op, cxobj *xt, char *username, cbuf *cbret); /* in clixon_datastore_write.[ch] */ int xmldb_copy(clixon_handle h, const char *from, const char *to); int xmldb_lock(clixon_handle h, const char *db, uint32_t id); diff --git a/lib/clixon/clixon_options.h b/lib/clixon/clixon_options.h index 89969c19d..404c09863 100644 --- a/lib/clixon/clixon_options.h +++ b/lib/clixon/clixon_options.h @@ -85,16 +85,6 @@ enum nacm_credentials_t{ NC_EXCEPT /* Exact match except for root and www user */ }; -/*! Datastore cache behaviour, see clixon_datastore.[ch] - * - * See config option type datastore_cache in clixon-config.yang - */ -enum datastore_cache{ - DATASTORE_NOCACHE, - DATASTORE_CACHE, - DATASTORE_CACHE_ZEROCOPY -}; - /*! yang clixon regexp engine * * @see regexp_mode in clixon-config.yang @@ -204,7 +194,6 @@ enum priv_mode_t clicon_backend_privileges_mode(clixon_handle h); enum priv_mode_t clicon_restconf_privileges_mode(clixon_handle h); enum nacm_credentials_t clicon_nacm_credentials(clixon_handle h); -enum datastore_cache clicon_datastore_cache(clixon_handle h); enum regexp_mode clicon_yang_regexp(clixon_handle h); /*-- Specific option access functions for non-yang options --*/ int clicon_quiet_mode(clixon_handle h); diff --git a/lib/src/clixon_datastore.c b/lib/src/clixon_datastore.c index 884c72312..6d09a4958 100644 --- a/lib/src/clixon_datastore.c +++ b/lib/src/clixon_datastore.c @@ -190,42 +190,40 @@ xmldb_copy(clixon_handle h, clixon_debug(CLIXON_DBG_DATASTORE, "%s %s", from, to); /* XXX lock */ - if (clicon_datastore_cache(h) != DATASTORE_NOCACHE){ - /* Copy in-memory cache */ - /* 1. "to" xml tree in x1 */ - if ((de1 = clicon_db_elmnt_get(h, from)) != NULL) - x1 = de1->de_xml; - if ((de2 = clicon_db_elmnt_get(h, to)) != NULL) - x2 = de2->de_xml; - if (x1 == NULL && x2 == NULL){ - /* do nothing */ - } - else if (x1 == NULL){ /* free x2 and set to NULL */ - xml_free(x2); - x2 = NULL; - } - else if (x2 == NULL){ /* create x2 and copy from x1 */ - if ((x2 = xml_new(xml_name(x1), NULL, CX_ELMNT)) == NULL) - goto done; - xml_flag_set(x2, XML_FLAG_TOP); - if (xml_copy(x1, x2) < 0) - goto done; - } - else{ /* copy x1 to x2 */ - xml_free(x2); - if ((x2 = xml_new(xml_name(x1), NULL, CX_ELMNT)) == NULL) - goto done; - xml_flag_set(x2, XML_FLAG_TOP); - if (xml_copy(x1, x2) < 0) - goto done; - } - /* always set cache although not strictly necessary in case 1 - * above, but logic gets complicated due to differences with - * de and de->de_xml */ - if (de2) - de0 = *de2; - de0.de_xml = x2; /* The new tree */ + /* Copy in-memory cache */ + /* 1. "to" xml tree in x1 */ + if ((de1 = clicon_db_elmnt_get(h, from)) != NULL) + x1 = de1->de_xml; + if ((de2 = clicon_db_elmnt_get(h, to)) != NULL) + x2 = de2->de_xml; + if (x1 == NULL && x2 == NULL){ + /* do nothing */ + } + else if (x1 == NULL){ /* free x2 and set to NULL */ + xml_free(x2); + x2 = NULL; + } + else if (x2 == NULL){ /* create x2 and copy from x1 */ + if ((x2 = xml_new(xml_name(x1), NULL, CX_ELMNT)) == NULL) + goto done; + xml_flag_set(x2, XML_FLAG_TOP); + if (xml_copy(x1, x2) < 0) + goto done; + } + else{ /* copy x1 to x2 */ + xml_free(x2); + if ((x2 = xml_new(xml_name(x1), NULL, CX_ELMNT)) == NULL) + goto done; + xml_flag_set(x2, XML_FLAG_TOP); + if (xml_copy(x1, x2) < 0) + goto done; } + /* always set cache although not strictly necessary in case 1 + * above, but logic gets complicated due to differences with + * de and de->de_xml */ + if (de2) + de0 = *de2; + de0.de_xml = x2; /* The new tree */ clicon_db_elmnt_set(h, to, &de0); /* Copy the files themselves (above only in-memory cache) */ diff --git a/lib/src/clixon_datastore_read.c b/lib/src/clixon_datastore_read.c index d432341af..94c100705 100644 --- a/lib/src/clixon_datastore_read.c +++ b/lib/src/clixon_datastore_read.c @@ -677,162 +677,6 @@ xmldb_readfile(clixon_handle h, * @retval 1 OK * @retval 0 Parse OK but yang assigment not made (or only partial) and xerr set * @retval -1 Error - * @note Use of 1 for OK - * @see xmldb_get the generic API function - */ -static int -xmldb_get_nocache(clixon_handle h, - const char *db, - yang_bind yb, - cvec *nsc, - const char *xpath, - withdefaults_type wdef, - cxobj **xtop, - modstate_diff_t *msdiff, - cxobj **xerr) -{ - int retval = -1; - yang_stmt *yspec; - cxobj *xt = NULL; - cxobj *x; - int fd = -1; - cxobj **xvec = NULL; - size_t xlen; - int i; - int ret; - db_elmnt de0 = {0,}; - - if ((yspec = clicon_dbspec_yang(h)) == NULL){ - clixon_err(OE_YANG, ENOENT, "No yang spec"); - goto done; - } - /* xml looks like: ... where "x" is a top-level symbol in a module */ - if ((ret = xmldb_readfile(h, db, yb, yspec, &xt, &de0, msdiff, xerr)) < 0) - goto done; - if (ret == 0) - goto fail; - clicon_db_elmnt_set(h, db, &de0); /* Content is copied */ - /* Here xt looks like: ... */ - /* Given the xpath, return a vector of matches in xvec */ - if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, xpath?xpath:"/") < 0) - goto done; - - /* If vectors are specified then mark the nodes found with all ancestors - * and filter out everything else, - * otherwise return complete tree. - */ - if (xvec != NULL) - for (i=0; ide_xml == NULL){ /* Cache miss, read XML from file */ - /* If there is no xml x0 tree (in cache), then read it from file */ - /* xml looks like: ... where "x" is a top-level symbol in a module */ - if ((ret = xmldb_readfile(h, db, yb, yspec, &x0t, &de0, msdiff, xerr)) < 0) - goto done; - if (ret == 0) - goto fail; - /* Should we validate file if read from disk? - * No, argument against: we may want to have a semantically wrong file and wish to edit? - */ - de0.de_xml = x0t; - if (de) - de0.de_id = de->de_id; - clicon_db_elmnt_set(h, db, &de0); - } /* x0t == NULL */ - else - x0t = de->de_xml; - - /* Here xt looks like: ... */ - if (xpath_vec(x0t, nsc, "%s", &xvec, &xlen, xpath?xpath:"/") < 0) - goto done; - /* Iterate through the match vector - * For every node found in x0, mark the tree up to t1 - */ - for (i=0; ide_xml; /* XXX flag is not XML_FLAG_TOP */ + x0 = de->de_xml; /* XXX flag is not XML_FLAG_TOP */ } /* If there is no xml x0 tree (in cache), then read it from file */ if (x0 == NULL){ @@ -1294,7 +1294,7 @@ xmldb_put(clixon_handle h, clixon_log(h, LOG_NOTICE, "%s: verify failed #3", __FUNCTION__); #endif /* Write back to datastore cache if first time */ - if (clicon_datastore_cache(h) != DATASTORE_NOCACHE){ + { db_elmnt de0 = {0,}; if (de != NULL) de0 = *de; @@ -1327,7 +1327,7 @@ xmldb_put(clixon_handle h, goto done; } pretty = clicon_option_bool(h, "CLICON_XMLDB_PRETTY"); - if (strcmp(format,"json")==0){ + if (strcmp(format, "json")==0){ if (clixon_json2file(f, x0, pretty, fprintf, 0, 0) < 0) goto done; } @@ -1339,6 +1339,7 @@ xmldb_put(clixon_handle h, goto done; retval = 1; done: + clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "retval:%d", retval); if (f != NULL) fclose(f); if (xerr) @@ -1347,8 +1348,6 @@ xmldb_put(clixon_handle h, xml_nsctx_free(nsc); if (dbfile) free(dbfile); - if (x0 && clicon_datastore_cache(h) == DATASTORE_NOCACHE) - xml_free(x0); return retval; fail: retval = 0; diff --git a/lib/src/clixon_options.c b/lib/src/clixon_options.c index e16ac6469..ee001a94c 100644 --- a/lib/src/clixon_options.c +++ b/lib/src/clixon_options.c @@ -117,15 +117,6 @@ static const map_str2int nacm_credentials_map[] = { {NULL, -1} }; -/* Mapping between datastore cache string <--> constants, - * see clixon-config.yang type datastore_cache */ -static const map_str2int datastore_cache_map[] = { - {"nocache", DATASTORE_NOCACHE}, - {"cache", DATASTORE_CACHE}, - {"cache-zerocopy", DATASTORE_CACHE_ZEROCOPY}, - {NULL, -1} -}; - /* Mapping between regular expression type string <--> constants, * see clixon-config.yang type regexp_mode */ static const map_str2int yang_regexp_map[] = { @@ -997,23 +988,6 @@ clicon_nacm_credentials(clixon_handle h) return clicon_str2int(nacm_credentials_map, mode); } -/*! Which datastore cache method to use - * - * @param[in] h Clixon handle - * @retval method Datastore cache method - * @see clixon-config@.yang CLICON_DATASTORE_CACHE - */ -enum datastore_cache -clicon_datastore_cache(clixon_handle h) -{ - char *str; - - if ((str = clicon_option_str(h, "CLICON_DATASTORE_CACHE")) == NULL) - return DATASTORE_CACHE; - else - return clicon_str2int(datastore_cache_map, str); -} - /*! Which Yang regexp/pattern engine to use * * @param[in] h Clixon handle diff --git a/lib/src/clixon_xml_default.c b/lib/src/clixon_xml_default.c index 298a1b8a4..84ada2c1c 100644 --- a/lib/src/clixon_xml_default.c +++ b/lib/src/clixon_xml_default.c @@ -315,11 +315,11 @@ xml_default(yang_stmt *yt, /* If config parameter and local is config false */ if (!state && !yang_config(yc)) continue; + /* Want to add state defaults, but this is config */ + if (state && yang_config_ancestor(yc)) + continue; switch (yang_keyword_get(yc)){ case Y_LEAF: - /* Want to add state defaults, but this is config */ - if (state && yang_config_ancestor(yc)) - break; if ((cv = yang_cv_get(yc)) == NULL){ clixon_err(OE_YANG,0, "Internal error: yang leaf %s not populated with cv as it should", yang_argument_get(yc)); @@ -402,9 +402,10 @@ xml_default_recurse(cxobj *xn, cxobj *x; yang_stmt *y; - if ((yn = (yang_stmt*)xml_spec(xn)) != NULL) + if ((yn = (yang_stmt*)xml_spec(xn)) != NULL){ if (xml_default(yn, xn, state) < 0) goto done; + } x = NULL; while ((x = xml_child_each(xn, x, CX_ELMNT)) != NULL) { if ((y = (yang_stmt*)xml_spec(x)) != NULL){ @@ -536,6 +537,7 @@ xml_global_defaults(clixon_handle h, * @param[in] purge 0: Dont remove any nodes * 1: Remove config sub-nodes that are empty non-presence container or default leaf * 2: Remove all sub-nodes that are empty non-presence container or default leaf + * 3: Remove all sub-nodes that are empty non-presence containers * @retval 1 Node is an (recursive) empty non-presence container or default leaf * @retval 0 Other node * @retval -1 Error @@ -562,7 +564,8 @@ xml_defaults_nopresence(cxobj *xn, yang_find(yn, Y_PRESENCE, NULL) == NULL) rmx = 1; else if (keyw == Y_LEAF && - xml_flag(xn, XML_FLAG_DEFAULT)) + xml_flag(xn, XML_FLAG_DEFAULT) && + purge != 3) rmx = 1; config = yang_config_ancestor(yn); } @@ -570,6 +573,7 @@ xml_defaults_nopresence(cxobj *xn, x = NULL; xprev = NULL; while ((x = xml_child_each(xn, x, CX_ELMNT)) != NULL) { + /* 1: node is empty non-presence or default leaf (eg rmx) */ if ((ret = xml_defaults_nopresence(x, purge)) < 0) goto done; if (ret == 1){ @@ -582,6 +586,7 @@ xml_defaults_nopresence(cxobj *xn, break; /* fall thru */ case 2: /* purge all nodes */ + case 3: if (xml_purge(x) < 0) goto done; x = xprev; @@ -649,8 +654,6 @@ xml_flag_state_default_value(cxobj *x, goto done; if ((cv = yang_cv_get(y)) == NULL) goto done; - if ((cv = yang_cv_get(y)) == NULL) - goto done; if (cv_name_get(cv) == NULL) goto done; if ((yv = cv2str_dup(cv)) == NULL) diff --git a/lib/src/clixon_yang_schema_mount.c b/lib/src/clixon_yang_schema_mount.c index 3da01ab77..700c6b63d 100644 --- a/lib/src/clixon_yang_schema_mount.c +++ b/lib/src/clixon_yang_schema_mount.c @@ -441,7 +441,11 @@ find_schema_mounts(cxobj *x, * "ietf-yang-schema-mount" modules in the mounted schema and specifying * the schemas in exactly the same way as the top-level schema. * Alt: see snmp_yang2xml to get instances instead of brute force traverse of whole tree - * @note: Mountpoints must exist in xret on entry + * XXX Mountpoints must exist in xret on entry, which is problematic: + * XXX A get state may have an xpath not including their config, ie: + * XXX xpath=/top/mymount/yang-library does not include /top/mymount and therefore + * XXX the mountpoint will not be present in xret + * XXX see: https://github.com/clicon/clixon/issues/485 */ static int yang_schema_mount_statedata_yanglib(clixon_handle h, diff --git a/test/test_type.sh b/test/test_type.sh index 744d051b5..211e9c1b9 100755 --- a/test/test_type.sh +++ b/test/test_type.sh @@ -225,10 +225,7 @@ EOF # Type tests. # Parameters: -# 1: dbcache: cache, nocache, cache-zerocopy function testrun(){ - dbcache=$1 - new "test params: -f $cfg # dbcache: $dbcache" cat < $cfg @@ -242,7 +239,6 @@ function testrun(){ /usr/local/var/run/$APPNAME.sock /usr/local/var/run/$APPNAME.pidfile /usr/local/var/$APPNAME - $dbcache $format ${AUTOCLI} @@ -698,14 +694,8 @@ EOF fi } -# Run without db cache -testrun nocache - -# Run with db cache -testrun cache - -# Run with zero-copy -testrun cache-zerocopy +# Run test +testrun rm -rf $dir diff --git a/yang/clixon/clixon-config@2024-01-01.yang b/yang/clixon/clixon-config@2024-01-01.yang index 77ad139c4..8194b6a2b 100644 --- a/yang/clixon/clixon-config@2024-01-01.yang +++ b/yang/clixon/clixon-config@2024-01-01.yang @@ -52,6 +52,7 @@ module clixon-config { revision 2024-01-01 { description "Makred as obsolete: + CLICON_DATASTORE_CACHE CLICON_NETCONF_CREATOR_ATTR Released in Clixon 6.6"; } @@ -960,7 +961,9 @@ module clixon-config { "Clixon datastore cache behaviour. There are three values: no cache, cache with copy, or cache without copy. Note: 'cache' is default value and supported with regressions etc. - Others are experimental (in Clixon 5.5)"; + Others are experimental (in Clixon 5.5) + Note that from 6.6 this is OBSOLETED, only datastore_cache is supported"; + status obsolete; } leaf CLICON_XMLDB_FORMAT { type cl:datastore_format;