feat(inputs.modbus): Optimise grouped requests

plugins/inputs/modbus/README.md

@@ -146,11 +146,19 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
     ##  |                 to reduce the number of requested registers by keeping
     ##  |                 the number of requests.
     ##  |---aggressive -- Rearrange request boundaries similar to "rearrange" but
-    ##                    allow to request registers not specified by the user to
-    ##                    fill gaps. This usually reduces the number of requests at the
-    ##                    cost of more requested registers.
+    ##  |                 allow to request registers not specified by the user to
+    ##  |                 fill gaps. This usually reduces the number of requests at the
+    ##  |                 cost of more requested registers.
+    ##  |---max_insert -- Rearrange request keeping the number of extra fields below the value
+    ##                    provided in "max_extra_registers". It is not necessary to define 'omitted'
+    ##                    fields as the optimisation will add such field only where needed.
     # optimization = "none"
 
+    ## Max extra register. For the 'max_insert' optimization only. Maximum size by which a request 
+    # can be extended in order to reduce the number of requests. This needs to be
+    # a positive integer lower than the maximum number of fields per query.
+    # max_extra_registers = 50
+
     ## Field definitions
     ## Analog Variables, Input Registers and Holding Registers
     ## address        - address of the register to query. For coil and discrete inputs this is the bit address.
@@ -377,6 +385,42 @@ interested in but want to minimize the number of requests sent to the device.
 __Please note:__ This optimization might take long in case of many
 non-consecutive, non-omitted fields!
 
+addresses are filled automatically. This usually reduces the number of
+requests, but will increase the number of registers read due to larger requests.
+This algorithm might be usefull if you only want to specify the fields you are
+interested in but want to minimize the number of requests sent to the device.
+Requests are processed similar to `rearrange` but user-defined gaps in the
+field addresses are filled automatically.
+
+__Please note:__ This optimization might take long in case of many
+non-consecutive, non-ommitted fields!
+
+##### `max_insert`
+
+With this optimization, user defined omitted fields are ignored, which
+allows composing shorter configuration files in the case of devices with
+many available registers. Every request is built considering the cost of
+adding a new register (including the gap between this one and the previous
+one of the request) compared to the cost of creating a new request.
+
+__Please note:__ The optimal value for `max_extra_registers` will depend
+on the network and the queried device. It is hence recommended to test
+several values and assess performance in order to find the best value.
+When running telegraf with the `--debug` flag, you can check the number
+of requests and the number of touched registers that your configuration
+results to.
+You can use the following strategy to find the most suitable value of
+`max_extra_registers` for your setup:
+
+1. use the `--test --debug` flags to identify a list of
+`max_extra_registers` values leading to unique number of requests
+2. set the acquisition interval to a very low value (_e.g._ 10ms)
+3. acquire the data for 5min with each value of `max_extra_registers`
+4. check the actually recorded data period, for instance in influx call
+    `SELECT MEAN(dt) FROM (SELECT ELAPSED(<oneSignaName>,1ms) AS dt FROM <MeasurementName>)`
+5. select the `max_extra_registers` that gives the lowest actually
+    recorded data period
+
 #### Field definitions
 
 Each `request` can contain a list of fields to collect from the modbus device.

plugins/inputs/modbus/configuration_register.go

@@ -81,7 +81,7 @@ func (c *ConfigurationOriginal) initRequests(fieldDefs []fieldDefinition, maxQua
 	if err != nil {
 		return nil, err
 	}
-	return groupFieldsToRequests(fields, nil, maxQuantity, "none"), nil
+	return groupFieldsToRequests(fields, nil, maxQuantity, "none", 0), nil
 }
 
 func (c *ConfigurationOriginal) initFields(fieldDefs []fieldDefinition) ([]field, error) {

plugins/inputs/modbus/configuration_request.go

@@ -5,8 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"hash/maphash"
-
-	"github.com/influxdata/telegraf/internal/choice"
 )
 
 //go:embed sample_request.conf
@@ -23,13 +21,14 @@ type requestFieldDefinition struct {
 }
 
 type requestDefinition struct {
-	SlaveID      byte                     `toml:"slave_id"`
-	ByteOrder    string                   `toml:"byte_order"`
-	RegisterType string                   `toml:"register"`
-	Measurement  string                   `toml:"measurement"`
-	Optimization string                   `toml:"optimization"`
-	Fields       []requestFieldDefinition `toml:"fields"`
-	Tags         map[string]string        `toml:"tags"`
+	SlaveID           byte                     `toml:"slave_id"`
+	ByteOrder         string                   `toml:"byte_order"`
+	RegisterType      string                   `toml:"register"`
+	Measurement       string                   `toml:"measurement"`
+	Optimization      string                   `toml:"optimization"`
+	MaxExtraRegisters uint16                   `toml:"max_extra_registers"`
+	Fields            []requestFieldDefinition `toml:"fields"`
+	Tags              map[string]string        `toml:"tags"`
 }
 
 type ConfigurationPerRequest struct {
@@ -46,11 +45,15 @@ func (c *ConfigurationPerRequest) Check() error {
 
 	for _, def := range c.Requests {
 		// Check for valid optimization
-		validOptimizations := []string{"", "none", "shrink", "rearrange", "aggressive"}
-		if !choice.Contains(def.Optimization, validOptimizations) {
+		switch def.Optimization {
+		case "", "none", "shrink", "rearrange", "aggressive":
+		case "max_insert":
+			if def.MaxExtraRegisters <= 0 || def.MaxExtraRegisters > maxQuantityHoldingRegisters {
+				return fmt.Errorf("max_extra_registers has to be between 1 and %d", maxQuantityHoldingRegisters)
+			}
+		default:
 			return fmt.Errorf("unknown optimization %q", def.Optimization)
 		}
-
 		// Check byte order of the data
 		switch def.ByteOrder {
 		case "":
@@ -162,16 +165,16 @@ func (c *ConfigurationPerRequest) Process() (map[byte]requestSet, error) {
 
 		switch def.RegisterType {
 		case "coil":
-			requests := groupFieldsToRequests(fields, def.Tags, maxQuantityCoils, def.Optimization)
+			requests := groupFieldsToRequests(fields, def.Tags, maxQuantityCoils, def.Optimization, def.MaxExtraRegisters)
 			set.coil = append(set.coil, requests...)
 		case "discrete":
-			requests := groupFieldsToRequests(fields, def.Tags, maxQuantityDiscreteInput, def.Optimization)
+			requests := groupFieldsToRequests(fields, def.Tags, maxQuantityDiscreteInput, def.Optimization, def.MaxExtraRegisters)
 			set.discrete = append(set.discrete, requests...)
 		case "holding":
-			requests := groupFieldsToRequests(fields, def.Tags, maxQuantityHoldingRegisters, def.Optimization)
+			requests := groupFieldsToRequests(fields, def.Tags, maxQuantityHoldingRegisters, def.Optimization, def.MaxExtraRegisters)
 			set.holding = append(set.holding, requests...)
 		case "input":
-			requests := groupFieldsToRequests(fields, def.Tags, maxQuantityInputRegisters, def.Optimization)
+			requests := groupFieldsToRequests(fields, def.Tags, maxQuantityInputRegisters, def.Optimization, def.MaxExtraRegisters)
 			set.input = append(set.input, requests...)
 		default:
 			return nil, fmt.Errorf("unknown register type %q", def.RegisterType)

plugins/inputs/modbus/modbus.go

@@ -135,7 +135,24 @@ func (m *Modbus) Init() error {
 	if err := m.initClient(); err != nil {
 		return fmt.Errorf("initializing client failed: %v", err)
 	}
-
+	numberOfRequests := 0
+	totalTouchedFields := uint16(0)
+	for k := range m.requests {
+		numberOfRequests += len(m.requests[k].coil) + len(m.requests[k].discrete) + len(m.requests[k].holding) + len(m.requests[k].input)
+		for _, r := range m.requests[k].coil {
+			totalTouchedFields += r.length / r.fields[0].length
+		}
+		for _, r := range m.requests[k].discrete {
+			totalTouchedFields += r.length / r.fields[0].length
+		}
+		for _, r := range m.requests[k].holding {
+			totalTouchedFields += r.length / r.fields[0].length
+		}
+		for _, r := range m.requests[k].input {
+			totalTouchedFields += r.length / r.fields[0].length
+		}
+	}
+	m.Log.Debugf("For %v: sending %d request(s), touching %d registers", m.Name, numberOfRequests, totalTouchedFields)
 	return nil
 }