feat(enginenetx): enable the stats-based policy (ooni#1313)

This commit modifies the https-dialer policy we create to take into
account stats to generate tactics. As of this commit, the overall policy
is as follows:

1. if `$OONI_HOME/$engine_dir/httpsdialerstatic.conf` exists and
contains entries for the endpoint's domain (e.g.,
"www.example.com:443"), then we unconditionally use it to generate
tactics, otherwise;

2. we generate tactics using existing stats and filter only the tactics
that are less than one week old (filtering done when loading from the
kvstore) and have worked at least once, otherwise;

3. we generate tactics using known beacons and candidate SNIs with the
extra caveat that we're not going to generate a tactic we have generate
already in the previous step, otherwise;

4. we use whatever resolver is configured to lookup for the domain name
and generate tactics doing the boring thing of using the resolved IP
addrs along with the SNI being equal to the original domain.

Note that this diff fixes a previously untested for bug caused by trying
to obtain statistics for an unknow endpoint domain.

Part of ooni/probe#2531

internal/enginenetx/network.go

@@ -96,7 +96,7 @@ func NewNetwork(
 	httpsDialer := newHTTPSDialer(
 		logger,
 		&netxlite.Netx{Underlying: nil}, // nil means using netxlite's singleton
-		newHTTPSDialerPolicy(kvStore, logger, resolver),
+		newHTTPSDialerPolicy(kvStore, logger, resolver, stats),
 		stats,
 	)
 
@@ -139,10 +139,16 @@ func NewNetwork(
 }
 
 // newHTTPSDialerPolicy contains the logic to select the [HTTPSDialerPolicy] to use.
-func newHTTPSDialerPolicy(kvStore model.KeyValueStore, logger model.Logger, resolver model.Resolver) httpsDialerPolicy {
+func newHTTPSDialerPolicy(
+	kvStore model.KeyValueStore,
+	logger model.Logger,
+	resolver model.Resolver,
+	stats *statsManager,
+) httpsDialerPolicy {
 	// create a composed fallback TLS dialer policy
-	fallback := &beaconsPolicy{
-		Fallback: &dnsPolicy{logger, resolver},
+	fallback := &statsPolicy{
+		Fallback: &beaconsPolicy{Fallback: &dnsPolicy{logger, resolver}},
+		Stats:    stats,
 	}
 
 	// make sure we honor a user-provided policy

internal/enginenetx/statsmanager.go

@@ -403,17 +403,22 @@ func (mt *statsManager) Close() error {
 
 // LookupTacticsStats returns stats about tactics for a given domain and port. The returned
 // list is a clone of the one stored by [*statsManager] so, it can easily be modified.
-func (mt *statsManager) LookupTactics(domain string, port string) []*statsTactic {
+func (mt *statsManager) LookupTactics(domain string, port string) ([]*statsTactic, bool) {
 	out := []*statsTactic{}
 
 	// get exclusive access
 	defer mt.mu.Unlock()
 	mt.mu.Lock()
 
+	// check whether we have information on this endpoint
+	domainEpnts, good := mt.container.DomainEndpoints[net.JoinHostPort(domain, port)]
+	if !good {
+		return out, len(out) > 0
+	}
+
 	// return a copy of each entry
-	domainEpnts := mt.container.DomainEndpoints[net.JoinHostPort(domain, port)]
 	for _, entry := range domainEpnts.Tactics {
 		out = append(out, entry.Clone())
 	}
-	return out
+	return out, len(out) > 0
 }

internal/enginenetx/statsmanager_test.go

@@ -848,24 +848,40 @@ func TestStatsManagerLookupTacticsStats(t *testing.T) {
 	// create the stats manager
 	stats := newStatsManager(kvStore, log.Log)
 
-	// obtain tactics
-	tactics := stats.LookupTactics("api.ooni.io", "443")
-	if len(tactics) != 3 {
-		t.Fatal("unexpected tactics length")
-	}
+	t.Run("when we're searching for a domain endpoint we know about", func(t *testing.T) {
+		// obtain tactics
+		tactics, good := stats.LookupTactics("api.ooni.io", "443")
+		if !good {
+			t.Fatal("expected good")
+		}
+		if len(tactics) != 3 {
+			t.Fatal("unexpected tactics length")
+		}
+
+		// sort obtained tactics lexicographically
+		sort.SliceStable(tactics, func(i, j int) bool {
+			return tactics[i].Tactic.tacticSummaryKey() < tactics[j].Tactic.tacticSummaryKey()
+		})
 
-	// sort obtained tactics lexicographically
-	sort.SliceStable(tactics, func(i, j int) bool {
-		return tactics[i].Tactic.tacticSummaryKey() < tactics[j].Tactic.tacticSummaryKey()
-	})
+		// sort the initial tactics as well
+		sort.SliceStable(expectTactics, func(i, j int) bool {
+			return expectTactics[i].Tactic.tacticSummaryKey() < expectTactics[j].Tactic.tacticSummaryKey()
+		})
 
-	// sort the initial tactics as well
-	sort.SliceStable(expectTactics, func(i, j int) bool {
-		return expectTactics[i].Tactic.tacticSummaryKey() < expectTactics[j].Tactic.tacticSummaryKey()
+		// compare once we have sorted
+		if diff := cmp.Diff(expectTactics, tactics); diff != "" {
+			t.Fatal(diff)
+		}
 	})
 
-	// compare once we have sorted
-	if diff := cmp.Diff(expectTactics, tactics); diff != "" {
-		t.Fatal(diff)
-	}
+	t.Run("when we don't have information about a domain endpoint", func(t *testing.T) {
+		// obtain tactics
+		tactics, good := stats.LookupTactics("api.ooni.io", "444") // note: different port!
+		if good {
+			t.Fatal("expected !good")
+		}
+		if len(tactics) != 0 {
+			t.Fatal("unexpected tactics length")
+		}
+	})
 }

internal/enginenetx/statspolicy.go

@@ -65,7 +65,10 @@ func (p *statsPolicy) LookupTactics(ctx context.Context, domain string, port str
 }
 
 func (p *statsPolicy) statsLookupTactics(domain string, port string) (out []*httpsDialerTactic) {
-	tactics := p.Stats.LookupTactics(domain, port)
+	tactics, good := p.Stats.LookupTactics(domain, port)
+	if !good {
+		return
+	}
 
 	successRate := func(t *statsTactic) (rate float64) {
 		if t.CountStarted > 0 {