From 619d53645c85efe75eddcd87c3b9df561a4782a7 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 26 Sep 2023 18:15:41 +0200 Subject: [PATCH] feat(enginenetx): support getting stats on a domain endpoint (#1311) We need this functionality to ask the stats questions like "what are all the available tactics for www.example.com:443 along with their stats?". In turn, this information is needed to implement a new policy that uses existing stats to choose what to do. While there, rename files to make it clear which files contain policies and which contain state trackers. Useful because upcoming changes will add files containing state-based policy. While there, stop being paranoid about cloning a nil map to a nil map, acknowledge this is not important and making it round trip correctly would require more code we don't need, and simply adjust tests to accept nil and empty map as equal. Part of https://github.com/ooni/probe/issues/2531 --- .../{beacons.go => beaconspolicy.go} | 0 ...{beacons_test.go => beaconspolicy_test.go} | 0 .../enginenetx/{static.go => staticpolicy.go} | 0 .../{static_test.go => staticpolicy_test.go} | 0 .../enginenetx/{stats.go => statstracker.go} | 22 +++- .../{stats_test.go => statstracker_test.go} | 111 ++++++++++++++++++ 6 files changed, 130 insertions(+), 3 deletions(-) rename internal/enginenetx/{beacons.go => beaconspolicy.go} (100%) rename internal/enginenetx/{beacons_test.go => beaconspolicy_test.go} (100%) rename internal/enginenetx/{static.go => staticpolicy.go} (100%) rename internal/enginenetx/{static_test.go => staticpolicy_test.go} (100%) rename internal/enginenetx/{stats.go => statstracker.go} (95%) rename internal/enginenetx/{stats_test.go => statstracker_test.go} (86%) diff --git a/internal/enginenetx/beacons.go b/internal/enginenetx/beaconspolicy.go similarity index 100% rename from internal/enginenetx/beacons.go rename to internal/enginenetx/beaconspolicy.go diff --git a/internal/enginenetx/beacons_test.go b/internal/enginenetx/beaconspolicy_test.go similarity index 100% rename from internal/enginenetx/beacons_test.go rename to internal/enginenetx/beaconspolicy_test.go diff --git a/internal/enginenetx/static.go b/internal/enginenetx/staticpolicy.go similarity index 100% rename from internal/enginenetx/static.go rename to internal/enginenetx/staticpolicy.go diff --git a/internal/enginenetx/static_test.go b/internal/enginenetx/staticpolicy_test.go similarity index 100% rename from internal/enginenetx/static_test.go rename to internal/enginenetx/staticpolicy_test.go diff --git a/internal/enginenetx/stats.go b/internal/enginenetx/statstracker.go similarity index 95% rename from internal/enginenetx/stats.go rename to internal/enginenetx/statstracker.go index 707e93a8d6..f05539fd7b 100644 --- a/internal/enginenetx/stats.go +++ b/internal/enginenetx/statstracker.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "net" "sync" "time" @@ -87,10 +88,8 @@ type statsTactic struct { } func statsCloneMapStringInt64(input map[string]int64) (output map[string]int64) { + output = make(map[string]int64) for key, value := range input { - if output == nil { - output = make(map[string]int64) // the idea here is to clone a nil map to a nil map - } output[key] = value } return @@ -401,3 +400,20 @@ func (mt *statsManager) Close() error { // write updated stats into the underlying key-value store return mt.kvStore.Set(statsKey, runtimex.Try1(json.Marshal(mt.container))) } + +// LookupTacticsStats returns stats about tactics for a given domain and port. The returned +// list is a clone of the one stored by [*statsManager] so, it can easily be modified. +func (mt *statsManager) LookupTactics(domain string, port string) []*statsTactic { + out := []*statsTactic{} + + // get exclusive access + defer mt.mu.Unlock() + mt.mu.Lock() + + // return a copy of each entry + domainEpnts := mt.container.DomainEndpoints[net.JoinHostPort(domain, port)] + for _, entry := range domainEpnts.Tactics { + out = append(out, entry.Clone()) + } + return out +} diff --git a/internal/enginenetx/stats_test.go b/internal/enginenetx/statstracker_test.go similarity index 86% rename from internal/enginenetx/stats_test.go rename to internal/enginenetx/statstracker_test.go index da67454728..e3c017f844 100644 --- a/internal/enginenetx/stats_test.go +++ b/internal/enginenetx/statstracker_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "sort" "testing" "time" @@ -746,6 +747,7 @@ func TestStatsManagerCallbacks(t *testing.T) { // make sure the stats are the ones we expect diffOptions := []cmp.Option{ cmpopts.IgnoreFields(statsTactic{}, "LastUpdated"), + cmpopts.EquateEmpty(), } if diff := cmp.Diff(tc.expectRoot, root, diffOptions...); diff != "" { t.Fatal(diff) @@ -758,3 +760,112 @@ func TestStatsManagerCallbacks(t *testing.T) { }) } } + +// Make sure that we can safely obtain statistics for a domain and a port. +func TestStatsManagerLookupTacticsStats(t *testing.T) { + + // prepare the content of the stats + twentyMinutesAgo := time.Now().Add(-20 * time.Minute) + + expectTactics := []*statsTactic{{ + CountStarted: 5, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 5, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: twentyMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.repubblica.it", + VerifyHostname: "api.ooni.io", + }, + }, { + CountStarted: 1, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 1, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: twentyMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.kernel.org", + VerifyHostname: "api.ooni.io", + }, + }, { + CountStarted: 3, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 3, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: twentyMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "theconversation.com", + VerifyHostname: "api.ooni.io", + }, + }} + + expectContainer := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": { + Tactics: map[string]*statsTactic{}, + }, + }, + Version: statsContainerVersion, + } + + for _, tactic := range expectTactics { + expectContainer.DomainEndpoints["api.ooni.io:443"].Tactics[tactic.Tactic.tacticSummaryKey()] = tactic + } + + // configure the initial value of the stats + kvStore := &kvstore.Memory{} + if err := kvStore.Set(statsKey, runtimex.Try1(json.Marshal(expectContainer))); err != nil { + t.Fatal(err) + } + + // create the stats manager + stats := newStatsManager(kvStore, log.Log) + + // obtain tactics + tactics := stats.LookupTactics("api.ooni.io", "443") + if len(tactics) != 3 { + t.Fatal("unexpected tactics length") + } + + // sort obtained tactics lexicographically + sort.SliceStable(tactics, func(i, j int) bool { + return tactics[i].Tactic.tacticSummaryKey() < tactics[j].Tactic.tacticSummaryKey() + }) + + // sort the initial tactics as well + sort.SliceStable(expectTactics, func(i, j int) bool { + return expectTactics[i].Tactic.tacticSummaryKey() < expectTactics[j].Tactic.tacticSummaryKey() + }) + + // compare once we have sorted + if diff := cmp.Diff(expectTactics, tactics); diff != "" { + t.Fatal(diff) + } +}