[Add device / profile ignore fields](#55)

Added ignore-compare extension
#21: combination of local load and NETCONF monitoring

CHANGELOG.md

@@ -5,18 +5,21 @@ Expected: December 2023
 
 ### New features
 
+* [Add device / profile ignore fields](https://github.com/clicon/clixon-controller/issues/55)
 * [Support a loadable mechanism for device profiles](https://github.com/clicon/clixon-controller/issues/21)
   * The mechanism loads a set of configured modules locally from disk,
   * Instead of using RFC 6022 NETCONF monitoring get-schema method to devices
   * Modified the controller yang with device-profile and device-common
-  * Remains: combination of local load and NETCONF monitoring
 
 ### API changes on existing protocol/config features
 
+* New `clixon-controller-config@2023-11-01.yang` revision
+  * Added CONTROLLER_YANG_SCHEMA_MOUNT_DIR
 * New `clixon-controller@2023-11-01.yang` revision
   * Added device-profile list
   * Added device-common grouping, for common device/device-profile fields
   * Added module-set to device-common
+  * Added ignore-compare extension
 
 ### Corrected Bugs
 

src/controller.h

@@ -44,29 +44,22 @@
 #define CONTROLLER_NAMESPACE "http://clicon.org/controller"
 
 /*! Skip junos-configuration-metadata.yang
+ *
  * cRPD gives error if you request it with get-schema:
  * <error-message>invalid schema identifier : junos-configuration-metadata</error-message>
  */
 #define CONTROLLER_JUNOS_SKIP_METADATA
 
 /*! Add grouping command-forwarding in junos-rpc yangs if not exists
+ *
  * cRPD YANGs do not have groupimg command-grouping in junos-rpc YANGs so that
  * uses command-grouping fails.
  * Insert an empty grouping command-forwarding if it does not exist
  */
 #define CONTROLLER_JUNOS_ADD_COMMAND_FORWARDING
 
-/*! XPATH to skip for junos
- * Primitive patch for single xpath to ignore when comparing remote and local device configs
- */
-#define CONTROLLER_JUNOS_XPATH_SKIP "/configuration/system/login/user/uid"
-
-/*! Where to write all yangs from devices 
- * Move to clixon as option?
- */
-#define YANG_SCHEMA_MOUNT_DIR "/usr/local/share/clixon/controller/mounts"
-
 /*! Top-symbol in clixon datastores
+ *
  * Duplicate of constant in clixon_custom.h
  */
 #define DATASTORE_TOP_SYMBOL "config"

src/controller.xml

@@ -1,7 +1,6 @@
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>/usr/local/etc/controller.xml</CLICON_CONFIGFILE>
   <CLICON_CONFIG_EXTEND>clixon-controller-config</CLICON_CONFIG_EXTEND>
-  <CLICON_CONFIGFILE>/usr/local/etc/controller.xml</CLICON_CONFIGFILE>
   <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
   <CLICON_FEATURE>clixon-restconf:allow-auth-none</CLICON_FEATURE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
@@ -49,7 +48,7 @@
        <operation>enable</operation>
      </rule>
      <rule>
-       <name>include arista/openconfig system</name>
+       <name>include openconfig system</name>
        <module-name>openconfig*</module-name>
        <operation>enable</operation>
      </rule>

src/controller_backend.c

@@ -57,9 +57,9 @@
 
 /*! Called to get state data from plugin by programmatically adding state
  *
- * @param[in]    h        Clicon handle
+ * @param[in]    h        Clixon handle
  * @param[in]    nsc      External XML namespace context, or NULL
- * @param[in]    xpath    String with XPATH syntax. or NULL for all
+ * @param[in]    xpath    String with XPath syntax. or NULL for all
  * @param[out]   xstate   XML tree, <config/> on entry. 
  * @retval       0        OK
  * @retval      -1        Error
@@ -500,7 +500,7 @@ action_daemon_register(clicon_handle h)
  * flags etc.
  * In particular for the controller, check if the services daemon is configured up and 
  * if so, ensure it is started.
- * @param[in]  h   Clicon handle
+ * @param[in]  h   Clixon handle
  * @param[in]  db  Database name (eg "running")
  * @retval     0   OK
  * @retval    -1   Fatal error

src/controller_cli.c

@@ -129,7 +129,7 @@ controller_cli_exit(clicon_handle h)
  * 4. Get yang specs of mountpoint from controller
  * 5. Parse YANGs locally from the yang specs
  * 6. Generate auto-cligen tree from the specs 
- * @param[in]  h         Clicon handle
+ * @param[in]  h         Clixon handle
  * @param[in]  xdev      XML device tree 
  * @param[in]  devname   Device name
  * @param[in]  treename  Autocli treename

src/controller_cli_callbacks.c

@@ -487,7 +487,7 @@ send_transaction_error(clicon_handle h,
 
 /*! Poll controller notification socket
  *
- * param[in]  h      Clicon handle
+ * param[in]  h      Clixon handle
  * param[in]  tidstr Transaction identifier
  * param[out] result
  * @retval    0      OK
@@ -1261,7 +1261,7 @@ send_pull_transient(clicon_handle h,
 
 /*! Compare device config types: running with last saved synced or current device (transient)
  *
- * @param[in]   h       Clicon handle
+ * @param[in]   h       Clixon handle
  * @param[in]   cvv     name: device pattern
  * @param[in]   argv    <format>        "text"|"xml"|"json"|"cli"|"netconf" (see format_enum)
  * @param[in]   dt1     First device config config 
@@ -1378,7 +1378,7 @@ compare_device_config_type(clicon_handle      h,
 /*! Compare datastores uses special diff rpc
  *
  * Use specialized rpc to reduce bandwidth
- * @param[in]   h     Clicon handle
+ * @param[in]   h     Clixon handle
  * @param[in]   cvv  
  * @param[in]   argv  <db1> <db2> <format>
  * @retval      0     OK
@@ -1464,7 +1464,7 @@ compare_dbs_rpc(clicon_handle h,
 
 /*! Compare device dbs: running with (last) synced db
  *
- * @param[in]   h     Clicon handle
+ * @param[in]   h     Clixon handle
  * @param[in]   cvv  
  * @param[in]   argv  arg: 0 as xml, 1: as text
  * @retval      0     OK
@@ -1491,7 +1491,7 @@ compare_device_db_sync(clicon_handle h,
 
 /*! Compare device dbs: running with current device (transient)
  *
- * @param[in] h     Clicon handle
+ * @param[in] h     Clixon handle
  * @param[in] cvv  : name pattern or NULL
  * @param[in] argv  arg: 0 as xml, 1: as text
  * @retval    0     OK
@@ -1549,7 +1549,7 @@ check_device_db(clixon_handle h,
 
 /*! Sub-routine for device dbxml: api-path to xml and send edit-config
  *
- * @param[in]  h     Clicon handle
+ * @param[in]  h     Clixon handle
  * @param[in]  cvv   Vector of cli string and instantiated variables 
  * @param[in]  op    Operation to perform on datastore
  * @param[in]  nsctx Namespace context for last value added
@@ -1656,7 +1656,7 @@ cli_dbxml_devs_sub(clicon_handle       h,
 
 /*! Modify xml datastore from a callback using xml key format strings
  *
- * @param[in]  h     Clicon handle
+ * @param[in]  h     Clixon handle
  * @param[in]  cvv   Vector of cli string and instantiated variables 
  * @param[in]  argv  Vector: <apipathfmt> [<mointpt>], eg "/aaa/%s"
  * @param[in]  op    Operation to perform on datastore
@@ -1782,7 +1782,7 @@ cli_dbxml_devs(clicon_handle       h,
 
 /*! CLI callback: set auto db item, specialization for controller devices
  *
- * @param[in]  h    Clicon handle
+ * @param[in]  h    Clixon handle
  * @param[in]  cvv  Vector of cli string and instantiated variables 
  * @param[in]  argv Vector. First element xml key format string, eg "/aaa/%s"
  * Format of argv:
@@ -1809,7 +1809,7 @@ cli_auto_set_devs(clicon_handle h,
 
 /*! Merge datastore xml entry, specialization for controller devices
  *
- * @param[in]  h    Clicon handle
+ * @param[in]  h    Clixon handle
  * @param[in]  cvv  Vector of cli string and instantiated variables 
  * @param[in]  argv Vector. First element xml key format string, eg "/aaa/%s"
  * @see cli_auto_merge  original callback
@@ -1834,7 +1834,7 @@ cli_auto_merge_devs(clicon_handle h,
 
 /*! Delete datastore xml, specialization for controller devices
  *
- * @param[in]  h    Clicon handle
+ * @param[in]  h    Clixon handle
  * @param[in]  cvv  Vector of cli string and instantiated variables 
  * @param[in]  argv Vector. First element xml key format string, eg "/aaa/%s"
  * @see cli_auto_del  original callback

src/controller_device_handle.c

@@ -723,19 +723,73 @@ device_handle_yang_lib_get(device_handle dh)
  * @param[in]  dh     Device handle
  * @param[in]  xylib  XML tree, is consumed
  * @retval     0      OK
+ * On the form: yang-library/module-set/name=<name>/module/name,revision,namespace  RFC 8525
  */
 int
 device_handle_yang_lib_set(device_handle dh,
                            cxobj        *xylib)
 {
     struct controller_device_handle *cdh = devhandle(dh);
 
+    /* Sanity check */
+    if (xylib)
+        assert(xml_find_type(xylib, NULL, "module-set", CX_ELMNT));
     if (cdh->cdh_yang_lib != NULL)
         xml_free(cdh->cdh_yang_lib);
     cdh->cdh_yang_lib = xylib;
     return 0;
 }
 
+/*! Set RFC 8525 yang library as xml tree
+ *
+ * @param[in]  dh     Device handle
+ * @param[in]  xylib  XML tree to append merge with existing if any
+ * @retval     0      OK
+ * @retval    -1      Error
+ * On the form: yang-library/module-set/name=<name>/module/name,revision,namespace  RFC 8525
+ */
+int
+device_handle_yang_lib_append(device_handle dh,
+                              cxobj        *xylib)
+{
+    int                              retval = -1;
+    struct controller_device_handle *cdh = devhandle(dh);
+    cxobj                           *x;
+    cxobj                           *xms0;
+    cxobj                           *xms = NULL;
+
+    /* Sanity check */
+    if (xylib){
+        if ((xms = xml_find_type(xylib, NULL, "module-set", CX_ELMNT)) == NULL){
+            clicon_err(OE_XML, EINVAL, "yang-lib top-level malformed: not module-set");
+            return -1;
+        }
+    }
+    if (cdh->cdh_yang_lib) {
+        //        cdh->cdh_yang_lib = xylib;
+        if (xylib){
+            if ((xms0 = xml_find_type(cdh->cdh_yang_lib, NULL, "module-set", CX_ELMNT)) == NULL){
+                clicon_err(OE_XML, EINVAL, "yang-lib top-level malformed: not module-set");
+                goto done;
+            }
+            x = NULL;
+            while ((x = xml_child_each(xms, x, CX_ELMNT)) != NULL) {
+                if (strcmp(xml_name(x), "module") != 0)
+                    continue;
+                xml_rm(x);
+                if (xml_addsub(xms0, x) < 0)
+                    goto done;
+                x = NULL; /* reset loop after removal */
+            }
+            free(xylib);
+        }
+    }
+    else
+        cdh->cdh_yang_lib = xylib;
+    retval = 0;
+ done:
+    return retval;
+}
 
 /*! Get sync timestamp
  *

src/controller_device_handle.h

@@ -83,7 +83,8 @@ cxobj *device_handle_capabilities_get(device_handle dh);
 int    device_handle_capabilities_set(device_handle dh, cxobj *xcaps);
 int    device_handle_capabilities_find(clixon_handle ch, const char *name);
 cxobj *device_handle_yang_lib_get(device_handle dh);
-int    device_handle_yang_lib_set(device_handle dh, cxobj *xschemas);
+int    device_handle_yang_lib_set(device_handle dh, cxobj *xylib);
+int    device_handle_yang_lib_append(device_handle dh, cxobj *xylib);
 int    device_handle_sync_time_get(device_handle dh, struct timeval *t);
 int    device_handle_sync_time_set(device_handle dh, struct timeval *t);
 yang_stmt *device_handle_yspec_get(device_handle dh);

src/controller_device_recv.c

@@ -414,7 +414,7 @@ device_state_recv_schema_list(device_handle dh,
     if (xml_rm(xschemas) < 0)
         goto done;
     /* "Wash" it from other elements: eg. junos may sneak in errors
-     * XXX Maybe this can be skipped ? v
+     * XXX Maybe this can be skipped ?
      */
     x = NULL;
     while ((x = xml_child_each(xschemas, x, CX_ELMNT)) != NULL) {
@@ -432,7 +432,7 @@ device_state_recv_schema_list(device_handle dh,
         goto done;
     if (xml_rootchild(xyanglib, 0, &xyanglib) < 0)
         goto done;
-    if (device_handle_yang_lib_set(dh, xyanglib) < 0)
+    if (device_handle_yang_lib_append(dh, xyanglib) < 0)
         goto done;
     retval = 1;
  done:
@@ -462,18 +462,21 @@ device_state_recv_get_schema(device_handle dh,
                              char         *rpcname,
                              conn_state    conn_state)
 {
-    int         retval = -1;
-    char       *ystr;
-    char       *ydec = NULL;
-    char       *modname;
-    char       *revision = NULL;
-    cbuf       *cb = cbuf_new();
-    FILE       *f = NULL;
-    size_t      sz;
-    yang_stmt  *yspec = NULL;
-    int         ret;
+    int           retval = -1;
+    clixon_handle h;
+    char         *ystr;
+    char         *ydec = NULL;
+    char         *modname;
+    char         *revision = NULL;
+    cbuf         *cb = cbuf_new();
+    FILE         *f = NULL;
+    size_t        sz;
+    yang_stmt    *yspec = NULL;
+    char         *dir;
+    int           ret;
 
     clicon_debug(1, "%s", __FUNCTION__);
+    h = device_handle_handle_get(dh);
     if ((ret = rpc_reply_sanity(dh, xmsg, rpcname, conn_state)) < 0)
         goto done;
     if (ret == 0)
@@ -492,7 +495,11 @@ device_state_recv_get_schema(device_handle dh,
         clicon_err(OE_UNIX, errno, "cbuf_new");
         goto done;
     }
-    cprintf(cb, "%s/%s", YANG_SCHEMA_MOUNT_DIR, modname);
+    if ((dir = clicon_option_str(h, "CONTROLLER_YANG_SCHEMA_MOUNT_DIR")) == NULL){
+        clicon_err(OE_YANG, 0, "schema mount dir not set");
+        goto done;
+    }
+    cprintf(cb, "%s/%s", dir, modname);
     if (revision)
         cprintf(cb, "@%s", revision);
     cprintf(cb, ".yang");