backports for ActivityLog and Reporting 1.11.x (#21210)

* backport of commit 8dedb31

* backport of commit 50a1f9e

* backport of commit 840abfb

* backport of commit 87aa644

* backport of commit b8de2c2

* backport of commit cb61488

* backport of commit d641bbc

* backport of commit 2a5a07e

* backport of commit 771bd8b

* backport of commit b5d7d47

* backport of commit a9e17c2

* backport of commit 9f7f8d5

* backport of commit e3c5977

* backport of commit b4fab6a

* backport of commit d70c17f

* backport of commit 4b6ec40

* backport of commit 05ba6bb

* backport of commit 002a59a

* backport of commit 77f83d9

* backport of commit 730d0e2

* backport of commit 35e2c16

* backport of commit 810d504

* Add ListMounts method backport for activitylog generation

* backport of commit 5b23dd5

* backport of commit 018ea84

* backport of commit 541f18e

* backport of commit b4e2751

* backport of commit dc5dd71

* backport of commit 5002489

---------

Co-authored-by: Nick Cabatoff <ncabatoff@hashicorp.com>
Co-authored-by: Chris Capurso <1036769+ccapurso@users.noreply.github.com>
Co-authored-by: Josh Black <raskchanky@users.noreply.github.com>
Co-authored-by: Peter Wilson <peter.wilson@hashicorp.com>
Co-authored-by: miagilepner <mia.epner@hashicorp.com>

changelog/17028.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core: Activity log goroutine management improvements to allow tests to be more deterministic.
+```

changelog/17856.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+core/activity: fix the end_date returned from the activity log endpoint when partial counts are computed
+```

changelog/17935.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+core/activity: return partial month counts when querying a historical date range and no historical data exists.
+```

changelog/18452.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+core/activity: de-duplicate namespaces when historical and current month data are mixed
+```

changelog/18598.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+core/activity: include mount counts when de-duplicating current and historical month data
+```

changelog/18766.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+core/activity: add namespace breakdown for new clients when date range spans multiple months, including the current month.
+```

changelog/18916.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+core/activity: report mount paths (rather than mount accessors) in current month activity log counts and include deleted mount paths in precomputed queries.
+```

changelog/19625.txt

@@ -0,0 +1,4 @@
+```release-note:feature
+core (enterprise): Add background worker for automatic reporting of billing
+information.
+```

changelog/19891.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core (enterprise): add configuration for license reporting 
+```

changelog/20073.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core/activity: refactor the activity log's generation of precomputed queries 
+```

changelog/20078.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core/activity: error when attempting to update retention configuration below the minimum 
+```

changelog/20086.txt

@@ -0,0 +1,4 @@
+```release-note:improvement
+api: `/sys/internal/counters/config` endpoint now contains read-only
+`reporting_enabled` and `billing_start_timestamp` fields.
+```

changelog/20150.txt

@@ -0,0 +1,4 @@
+```release-note:improvement
+api: `/sys/internal/counters/config` endpoint now contains read-only
+`minimum_retention_months`.
+```

changelog/20680.txt

@@ -0,0 +1,6 @@
+```release-note:improvement
+core (enterprise): support reloading configuration for automated reporting via SIGHUP
+```
+```release-note:improvement
+core (enterprise): license updates trigger a reload of reporting and the activity log 
+```

changelog/20694.txt

@@ -0,0 +1,4 @@
+```release-note:improvement
+api: GET ... /sys/internal/counters/activity?current_billing_period=true now
+results in a response which contains the full billing period
+```

command/server.go

@@ -1644,6 +1644,9 @@ func (c *ServerCommand) Run(args []string) int {
 				c.UI.Error(err.Error())
 			}
 
+			if err := core.ReloadCensus(); err != nil {
+				c.UI.Error(err.Error())
+			}
 			select {
 			case c.licenseReloadedCh <- err:
 			default:

command/server/config_test_helpers.go

@@ -955,6 +955,7 @@ func testParseSeals(t *testing.T) {
 			},
 		},
 	}
+	addExpectedDefaultEntConfig(expected)
 	config.Prune()
 	require.Equal(t, config, expected)
 }

command/server/config_test_helpers_util.go

@@ -3,4 +3,5 @@
 package server
 
 func addExpectedEntConfig(c *Config, sentinelModules []string)                         {}
+func addExpectedDefaultEntConfig(c *Config)                                            {}
 func addExpectedEntSanitizedConfig(c map[string]interface{}, sentinelModules []string) {}

helper/metricsutil/gauge_process.go

@@ -7,24 +7,9 @@ import (
 	"time"
 
 	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/timeutil"
 )
 
-// This interface allows unit tests to substitute in a simulated clock.
-type clock interface {
-	Now() time.Time
-	NewTicker(time.Duration) *time.Ticker
-}
-
-type defaultClock struct{}
-
-func (_ defaultClock) Now() time.Time {
-	return time.Now()
-}
-
-func (_ defaultClock) NewTicker(d time.Duration) *time.Ticker {
-	return time.NewTicker(d)
-}
-
 // GaugeLabelValues is one gauge in a set sharing a single key, that
 // are measured in a batch.
 type GaugeLabelValues struct {
@@ -69,7 +54,7 @@ type GaugeCollectionProcess struct {
 	ticker           *time.Ticker
 
 	// time source
-	clock clock
+	clock timeutil.Clock
 }
 
 // NewGaugeCollectionProcess creates a new collection process for the callback
@@ -88,7 +73,7 @@ func (m *ClusterMetricSink) NewGaugeCollectionProcess(
 		id,
 		collector,
 		logger,
-		defaultClock{},
+		timeutil.DefaultClock{},
 	)
 }
 
@@ -98,7 +83,7 @@ func (m *ClusterMetricSink) newGaugeCollectionProcessWithClock(
 	id []Label,
 	collector GaugeCollector,
 	logger log.Logger,
-	clock clock,
+	clock timeutil.Clock,
 ) (*GaugeCollectionProcess, error) {
 	process := &GaugeCollectionProcess{
 		stop:             make(chan struct{}, 1),

helper/metricsutil/gauge_process_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/armon/go-metrics"
 	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/timeutil"
 )
 
 // SimulatedTime maintains a virtual clock so the test isn't
@@ -21,9 +22,10 @@ import (
 type SimulatedTime struct {
 	now           time.Time
 	tickerBarrier chan *SimulatedTicker
+	timeutil.DefaultClock
 }
 
-var _ clock = &SimulatedTime{}
+var _ timeutil.Clock = &SimulatedTime{}
 
 type SimulatedTicker struct {
 	ticker   *time.Ticker
@@ -118,7 +120,7 @@ func TestGauge_Creation(t *testing.T) {
 		t.Fatalf("Error creating collection process: %v", err)
 	}
 
-	if _, ok := p.clock.(defaultClock); !ok {
+	if _, ok := p.clock.(timeutil.DefaultClock); !ok {
 		t.Error("Default clock not installed.")
 	}
 

helper/timeutil/timeutil.go

@@ -139,3 +139,26 @@ func SkipAtEndOfMonth(t *testing.T) {
 		t.Skip("too close to end of month")
 	}
 }
+
+// This interface allows unit tests to substitute in a simulated Clock.
+type Clock interface {
+	Now() time.Time
+	NewTicker(time.Duration) *time.Ticker
+	NewTimer(time.Duration) *time.Timer
+}
+
+type DefaultClock struct{}
+
+var _ Clock = (*DefaultClock)(nil)
+
+func (_ DefaultClock) Now() time.Time {
+	return time.Now()
+}
+
+func (_ DefaultClock) NewTicker(d time.Duration) *time.Ticker {
+	return time.NewTicker(d)
+}
+
+func (_ DefaultClock) NewTimer(d time.Duration) *time.Timer {
+	return time.NewTimer(d)
+}

vault/activity/generation/generate_data.proto

@@ -48,10 +48,9 @@ message Clients {
 message Client {
   string id = 1;
   int32 count = 2;
-  int32 times_seen = 3;
-  bool repeated = 4;
-  int32 repeated_from_month = 5;
-  string namespace = 6;
-  string mount = 7;
-  bool non_entity = 8;
+  bool repeated = 3;
+  int32 repeated_from_month = 4;
+  string namespace = 5;
+  string mount = 6;
+  bool non_entity = 7;
 }