Merge pull request #42 from Juniper/eos_internal

added override control of eos in internal jtimon configs
Juniper · Jun 25, 2024 · c7a5d46 · c7a5d46
2 parents 5e8fd2d + dd154db
commit c7a5d46
Show file tree

Hide file tree

Showing 15 changed files with 113 additions and 75 deletions.
diff --git a/config.go b/config.go
@@ -323,6 +323,7 @@ func HandleConfigChange(jctx *JCtx, config Config, restart *bool) error {
 		jctx.config = config
 		logInit(jctx)
 		internalJtimonLogInit(jctx)
+		initInternalJtimon(jctx)
 		if restart != nil {
 			jLog(jctx, fmt.Sprintf("Restarting worker process to spawn new device connection for: %s", jctx.file))
 			*restart = true
@@ -348,6 +349,7 @@ func ConfigRead(jctx *JCtx, init bool, restart *bool) error {
 		jctx.config = config
 		logInit(jctx)
 		internalJtimonLogInit(jctx)
+		initInternalJtimon(jctx)
 		b, err := json.MarshalIndent(jctx.config, "", "    ")
 		if err != nil {
 			return fmt.Errorf("config parsing error (json marshal) for %s: %v", jctx.file, err)

diff --git a/internal_jtimon.go b/internal_jtimon.go
@@ -21,6 +21,19 @@ type InternalJtimonConfig struct {
 	logger        *log.Logger
 	preGnmiLogger *log.Logger
 	csvLogger     *log.Logger
+	GnmiEOS       bool `json:"gnmi-eos"`
+	PreGnmiEOS    bool `json:"pre-gnmi-eos"`
+}
+
+func initInternalJtimon(jctx *JCtx) {
+	// if Internal Jtimon EOS value is not set,
+	// then take the EOS value from parent config
+	if !jctx.config.InternalJtimon.GnmiEOS {
+		jctx.config.InternalJtimon.GnmiEOS = jctx.config.EOS
+	}
+	if !jctx.config.InternalJtimon.PreGnmiEOS {
+		jctx.config.InternalJtimon.PreGnmiEOS = jctx.config.EOS
+	}
 }
 
 func internalJtimonLogInit(jctx *JCtx) {

diff --git a/multi_vendor.go b/multi_vendor.go
@@ -84,12 +84,16 @@ func subscribePrePostGNMI(conn *grpc.ClientConn, jctx *JCtx, cfg Config, paths [
 	// Launch goroutines for each subscription function
 	go func() {
 		gnmiPaths := getGnmiPaths(cfg)
-		gnmiResultCh <- subscribegNMI(conn, jctx, cfg, gnmiPaths)
+		if len(gnmiPaths) > 0 {
+			gnmiResultCh <- subscribegNMI(conn, jctx, cfg, gnmiPaths)
+		}
 	}()
 
 	go func() {
 		preGnmiPaths := getPreGnmiPaths(cfg)
-		junosResultCh <- subscribeJunos(conn, jctx, cfg, preGnmiPaths)
+		if len(preGnmiPaths) > 0 {
+			junosResultCh <- subscribeJunos(conn, jctx, cfg, preGnmiPaths)
+		}
 	}()
 
 	// Use select to wait for the first result to be available

diff --git a/subscribe_gnmi.go b/subscribe_gnmi.go
@@ -291,7 +291,8 @@ func gnmiHandleResponse(jctx *JCtx, rsp *gnmi.SubscribeResponse) error {
 		parseOutput = &tmpParseOp
 		err         error
 
-		hostname = jctx.config.Host + ":" + strconv.Itoa(jctx.config.Port)
+		eosEnabled = false
+		hostname   = jctx.config.Host + ":" + strconv.Itoa(jctx.config.Port)
 	)
 
 	// Update packet stats
@@ -316,7 +317,11 @@ func gnmiHandleResponse(jctx *JCtx, rsp *gnmi.SubscribeResponse) error {
 	updateStatsKV(jctx, true, parseOutput.inKvs)
 
 	// Ignore all packets till sync response is received.
-	if !jctx.config.EOS {
+	eosEnabled = jctx.config.EOS
+	if isInternalJtimonLogging(jctx) {
+		eosEnabled = jctx.config.InternalJtimon.GnmiEOS
+	}
+	if !eosEnabled {
 		if !jctx.receivedSyncRsp {
 			if parseOutput.jHeader != nil {
 				// For juniper packets, ignore only the packets which are numbered in initial sync sequence range

diff --git a/subscribe_juniper_junos.go b/subscribe_juniper_junos.go
@@ -237,6 +237,10 @@ func subscribeJunos(conn *grpc.ClientConn, jctx *JCtx, cfg Config, paths []Paths
 		subReqM.PathList = append(subReqM.PathList, &pathM)
 	}
 	additionalConfigM.NeedEos = jctx.config.EOS
+	// Override EOS if InternalJtimon is configured
+	if isInternalJtimonLogging(jctx) {
+		additionalConfigM.NeedEos = jctx.config.InternalJtimon.PreGnmiEOS
+	}
 	subReqM.AdditionalConfig = &additionalConfigM
 
 	return subSendAndReceive(conn, jctx, subReqM)

diff --git a/tests/data/cisco-ios-xr/config/xr-all-influx.log b/tests/data/cisco-ios-xr/config/xr-all-influx.log
@@ -48,7 +48,8 @@ Running config of JTIMON:
     "internal-jtimon": {
         "data-log-file": "",
         "csv-log-file": "",
-        "csv-stats": false
+        "gnmi-eos": false,
+        "pre-gnmi-eos": false
     },
     "paths": [
         {

diff --git a/tests/data/cisco-ios-xr/config/xr-wdsysmon-influx.log b/tests/data/cisco-ios-xr/config/xr-wdsysmon-influx.log
@@ -48,7 +48,8 @@ Running config of JTIMON:
     "internal-jtimon": {
         "data-log-file": "",
         "csv-log-file": "",
-        "csv-stats": false
+        "gnmi-eos": false,
+        "pre-gnmi-eos": false
     },
     "paths": [
         {

diff --git a/tests/data/juniper-junos/config/jtisim-influx-alias.log b/tests/data/juniper-junos/config/jtisim-influx-alias.log
@@ -48,7 +48,8 @@ Running config of JTIMON:
     "internal-jtimon": {
         "data-log-file": "",
         "csv-log-file": "",
-        "csv-stats": false
+        "gnmi-eos": false,
+        "pre-gnmi-eos": false
     },
     "paths": [
         {
@@ -91,22 +92,22 @@ Receiving telemetry data from 127.0.0.1:50051
 +------------------------------+--------------------+--------------------+--------------------+--------------------+
 |         Timestamp            |        KV          |      Packets       |       Bytes        |     Bytes(wire)    |
 +------------------------------+--------------------+--------------------+--------------------+--------------------+
-| Fri May 31 02:22:48 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:38:31 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 Batch processing: #packets:40 #points:40
 Batch write successful! Post batch write available points:  0
 
-| Fri May 31 02:22:50 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:38:33 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:22:52 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:38:35 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:22:54 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:38:37 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
 
-Collector Stats for 127.0.0.1:50051 (Run time : 8.007675042s)
+Collector Stats for 127.0.0.1:50051 (Run time : 8.010581416s)
 40           : in-packets
 1980         : data points (KV pairs)
 25           : in-header wirelength (bytes)

diff --git a/tests/data/juniper-junos/config/jtisim-influx.log b/tests/data/juniper-junos/config/jtisim-influx.log
@@ -48,7 +48,8 @@ Running config of JTIMON:
     "internal-jtimon": {
         "data-log-file": "",
         "csv-log-file": "",
-        "csv-stats": false
+        "gnmi-eos": false,
+        "pre-gnmi-eos": false
     },
     "paths": [
         {
@@ -85,57 +86,57 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory
 compression = none
 Connecting to 127.0.0.1:50051
 gRPC headers from host 127.0.0.1:50051
-  content-type: [application/grpc]
   jtisim: [yes]
+  content-type: [application/grpc]
 Receiving telemetry data from 127.0.0.1:50051
 
 +------------------------------+--------------------+--------------------+--------------------+--------------------+
 |         Timestamp            |        KV          |      Packets       |       Bytes        |     Bytes(wire)    |
 +------------------------------+--------------------+--------------------+--------------------+--------------------+
-| Fri May 31 02:22:23 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:38:06 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 Batch processing: #packets:40 #points:40
 Batch write successful! Post batch write available points:  0
 
-| Fri May 31 02:22:25 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:38:08 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:22:27 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:38:10 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:22:29 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:38:12 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:22:31 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:38:14 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 Batch processing: #packets:40 #points:40
 
-| Fri May 31 02:22:33 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:38:16 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 Batch write successful! Post batch write available points:  0
 
-| Fri May 31 02:22:35 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:38:18 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:22:37 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:38:20 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:22:39 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:38:22 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:22:41 PDT 2024 |               5367 |                109 |             236698 |             237243 |
+| Tue Jun 25 11:38:24 PDT 2024 |               4040 |                 82 |             178196 |             178606 |
 
 Batch processing: #packets:40 #points:40
 
-| Fri May 31 02:22:43 PDT 2024 |               5940 |                120 |             262258 |             262858 |
+| Tue Jun 25 11:38:26 PDT 2024 |               5940 |                120 |             262258 |             262858 |
 
 Batch write successful! Post batch write available points:  0
 
-| Fri May 31 02:22:45 PDT 2024 |               5940 |                120 |             262258 |             262858 |
+| Tue Jun 25 11:38:28 PDT 2024 |               5940 |                120 |             262258 |             262858 |
 
 
 
-Collector Stats for 127.0.0.1:50051 (Run time : 25.004155208s)
+Collector Stats for 127.0.0.1:50051 (Run time : 25.004049042s)
 120          : in-packets
 5940         : data points (KV pairs)
 25           : in-header wirelength (bytes)

diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-1.log b/tests/data/juniper-junos/config/jtisim-interfaces-1.log
@@ -48,7 +48,8 @@ Running config of JTIMON:
     "internal-jtimon": {
         "data-log-file": "",
         "csv-log-file": "",
-        "csv-stats": false
+        "gnmi-eos": false,
+        "pre-gnmi-eos": false
     },
     "paths": [
         {
@@ -82,51 +83,51 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory
 compression = none
 Connecting to 127.0.0.1:50051
 gRPC headers from host 127.0.0.1:50051
-  content-type: [application/grpc]
   jtisim: [yes]
+  content-type: [application/grpc]
 Receiving telemetry data from 127.0.0.1:50051
 
 +------------------------------+--------------------+--------------------+--------------------+--------------------+
 |         Timestamp            |        KV          |      Packets       |       Bytes        |     Bytes(wire)    |
 +------------------------------+--------------------+--------------------+--------------------+--------------------+
-| Fri May 31 02:23:21 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:39:04 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:23:23 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:39:06 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:23:25 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:39:08 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:23:27 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:39:10 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:23:29 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:39:12 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:23:31 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:39:14 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:23:33 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:39:16 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:23:35 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:39:18 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:23:37 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:39:20 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:23:39 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:39:22 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:23:41 PDT 2024 |               5940 |                120 |             262258 |             262858 |
+| Tue Jun 25 11:39:24 PDT 2024 |               5940 |                120 |             262258 |             262858 |
 
 
-| Fri May 31 02:23:43 PDT 2024 |               5940 |                120 |             262258 |             262858 |
+| Tue Jun 25 11:39:26 PDT 2024 |               5940 |                120 |             262258 |             262858 |
 
 
 
-Collector Stats for 127.0.0.1:50051 (Run time : 25.002534292s)
+Collector Stats for 127.0.0.1:50051 (Run time : 25.001662209s)
 120          : in-packets
 5940         : data points (KV pairs)
 25           : in-header wirelength (bytes)

diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-2.log b/tests/data/juniper-junos/config/jtisim-interfaces-2.log
@@ -48,7 +48,8 @@ Running config of JTIMON:
     "internal-jtimon": {
         "data-log-file": "",
         "csv-log-file": "",
-        "csv-stats": false
+        "gnmi-eos": false,
+        "pre-gnmi-eos": false
     },
     "paths": [
         {
@@ -89,44 +90,44 @@ Receiving telemetry data from 127.0.0.1:50051
 +------------------------------+--------------------+--------------------+--------------------+--------------------+
 |         Timestamp            |        KV          |      Packets       |       Bytes        |     Bytes(wire)    |
 +------------------------------+--------------------+--------------------+--------------------+--------------------+
-| Fri May 31 02:23:21 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:39:04 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:23:23 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:39:06 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:23:25 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:39:08 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:23:27 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:39:10 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:23:29 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
+| Tue Jun 25 11:39:12 PDT 2024 |               1980 |                 40 |              87418 |              87618 |
 
 
-| Fri May 31 02:23:31 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:39:14 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:23:33 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:39:16 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:23:35 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:39:18 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:23:37 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:39:20 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:23:39 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
+| Tue Jun 25 11:39:22 PDT 2024 |               3960 |                 80 |             174838 |             175238 |
 
 
-| Fri May 31 02:23:41 PDT 2024 |               5940 |                120 |             262258 |             262858 |
+| Tue Jun 25 11:39:24 PDT 2024 |               5940 |                120 |             262258 |             262858 |
 
 
-| Fri May 31 02:23:43 PDT 2024 |               5940 |                120 |             262258 |             262858 |
+| Tue Jun 25 11:39:26 PDT 2024 |               5940 |                120 |             262258 |             262858 |
 
 
 
-Collector Stats for 127.0.0.1:50051 (Run time : 25.00119025s)
+Collector Stats for 127.0.0.1:50051 (Run time : 25.000787458s)
 120          : in-packets
 5940         : data points (KV pairs)
 25           : in-header wirelength (bytes)