feat: issue reverse lookups in the test helper

This query collects extra information useful to further
disambiguate possible DNSDiff cases.

internal/archival/archival.go

@@ -206,7 +206,7 @@ func (ev *FlatDNSLookupEvent) ToArchival(begin time.Time) []model.ArchivalDNSLoo
 	case DNSLookupTypeNS:
 		return ev.toArchivalNS(begin)
 	default:
-		log.Printf("ToArchivalDNSLookupResultList: unhandled record: %+v", ev)
+		log.Printf("[BUG] ToArchivalDNSLookupResultList: unhandled record: %+v", ev)
 		return []model.ArchivalDNSLookupResult{}
 	}
 }
@@ -504,8 +504,10 @@ func (ev *FlatDNSRoundTripEvent) fillHostnameAndQueryType(out *model.ArchivalDNS
 		out.QueryType = "CNAME"
 	case dns.TypeANY:
 		out.QueryType = "ANY"
+	case dns.TypePTR:
+		out.QueryType = "PTR"
 	default:
-		// nothing
+		log.Printf("[BUG] fillHostnameAndQueryType: unhandled query type: %d", q0.Qtype)
 	}
 }
 
@@ -570,8 +572,20 @@ func (ev *FlatDNSRoundTripEvent) fillAnswers(out *model.ArchivalDNSLookupResult)
 				NS:         "",
 				TTL:        ev.ttl(v.Hdr.Ttl),
 			})
+		case *dns.PTR:
+			out.Answers = append(out.Answers, model.ArchivalDNSAnswer{
+				ALPN:       "",
+				ASN:        0,
+				ASOrgName:  "",
+				AnswerType: "PTR",
+				Hostname:   v.Ptr, // ooniprobe-legacy probably did this
+				IPv4:       "",
+				IPv6:       "",
+				NS:         "",
+				TTL:        ev.ttl(v.Hdr.Ttl),
+			})
 		default:
-			// nothing
+			log.Printf("[BUG] fillAnswers: unhandled record type %T", answer)
 		}
 	}
 }

internal/archival/flat.go

@@ -27,6 +27,7 @@ type FlatDNSLookupEvent struct {
 	Finished        time.Time
 	LookupType      DNSLookupType
 	NS              []string `json:",omitempty"`
+	PTRs            []string `json:",omitempty"`
 	ResolverAddress string   `json:",omitempty"`
 	ResolverNetwork NetworkType
 	Started         time.Time
@@ -44,7 +45,8 @@ func NewFakeFlatDNSLookupEvent(resolverNetwork NetworkType, resolverAddress stri
 		Failure:         "",
 		Finished:        now,
 		LookupType:      lookupType,
-		NS:              []string{},
+		NS:              nil,
+		PTRs:            nil,
 		ResolverAddress: resolverAddress,
 		ResolverNetwork: resolverNetwork,
 		Started:         now,

internal/archival/resolver.go

@@ -27,13 +27,16 @@ var (
 
 	// DNSLookupTypeNS indicates we're performing a NS lookup type.
 	DNSLookupTypeNS = DNSLookupType("ns")
+
+	// DNSLookupTypeReverse indicates we're performing a reverse lookup.
+	DNSLookupTypeReverse = DNSLookupType("reverse")
 )
 
 // WrapResolver wraps a resolver to use the saver.
 func (s *Saver) WrapResolver(reso model.Resolver) model.Resolver {
 	return &resolverSaver{
-		Resolver: reso,
-		s:        s,
+		r: reso,
+		s: s,
 	}
 }
 
@@ -46,20 +49,38 @@ func (s *Saver) WrapDNSTransport(txp model.DNSTransport) model.DNSTransport {
 }
 
 type resolverSaver struct {
-	model.Resolver
+	r model.Resolver
 	s *Saver
 }
 
+var _ model.Resolver = &resolverSaver{}
+
 func (r *resolverSaver) LookupHost(ctx context.Context, domain string) ([]string, error) {
-	return r.s.lookupHost(ctx, r.Resolver, domain)
+	return r.s.lookupHost(ctx, r.r, domain)
 }
 
 func (r *resolverSaver) LookupHTTPS(ctx context.Context, domain string) (*model.HTTPSSvc, error) {
-	return r.s.lookupHTTPS(ctx, r.Resolver, domain)
+	return r.s.lookupHTTPS(ctx, r.r, domain)
 }
 
 func (r *resolverSaver) LookupNS(ctx context.Context, domain string) ([]*net.NS, error) {
-	return r.s.lookupNS(ctx, r.Resolver, domain)
+	return r.s.lookupNS(ctx, r.r, domain)
+}
+
+func (r *resolverSaver) LookupPTR(ctx context.Context, domain string) ([]string, error) {
+	return r.s.lookupPTR(ctx, r.r, domain)
+}
+
+func (r *resolverSaver) Address() string {
+	return r.r.Address()
+}
+
+func (r *resolverSaver) Network() string {
+	return r.r.Network()
+}
+
+func (r *resolverSaver) CloseIdleConnections() {
+	r.r.CloseIdleConnections()
 }
 
 func (s *Saver) lookupHost(ctx context.Context, reso model.Resolver, domain string) ([]string, error) {
@@ -74,6 +95,7 @@ func (s *Saver) lookupHost(ctx context.Context, reso model.Resolver, domain stri
 		Finished:        time.Now(),
 		LookupType:      DNSLookupTypeGetaddrinfo,
 		NS:              nil,
+		PTRs:            nil,
 		ResolverAddress: reso.Address(),
 		ResolverNetwork: NetworkType(reso.Network()),
 		Started:         started,
@@ -99,6 +121,7 @@ func (s *Saver) lookupHTTPS(ctx context.Context, reso model.Resolver, domain str
 		Finished:        time.Now(),
 		LookupType:      DNSLookupTypeHTTPS,
 		NS:              nil,
+		PTRs:            nil,
 		ResolverAddress: reso.Address(),
 		ResolverNetwork: NetworkType(reso.Network()),
 		Started:         started,
@@ -133,6 +156,7 @@ func (s *Saver) lookupNS(ctx context.Context, reso model.Resolver, domain string
 		Finished:        time.Now(),
 		LookupType:      DNSLookupTypeNS,
 		NS:              s.ns(ns),
+		PTRs:            nil,
 		ResolverAddress: reso.Address(),
 		ResolverNetwork: NetworkType(reso.Network()),
 		Started:         started,
@@ -147,6 +171,26 @@ func (s *Saver) ns(ns []*net.NS) (out []string) {
 	return
 }
 
+func (s *Saver) lookupPTR(ctx context.Context, reso model.Resolver, domain string) ([]string, error) {
+	started := time.Now()
+	domains, err := reso.LookupPTR(ctx, domain)
+	s.appendDNSLookupEvent(&FlatDNSLookupEvent{
+		ALPNs:           nil,
+		Addresses:       nil,
+		CNAME:           "",
+		Domain:          domain,
+		Failure:         NewFlatFailure(err),
+		Finished:        time.Now(),
+		LookupType:      DNSLookupTypeReverse,
+		NS:              []string{},
+		PTRs:            domains,
+		ResolverAddress: reso.Address(),
+		ResolverNetwork: NetworkType(reso.Network()),
+		Started:         started,
+	})
+	return domains, err
+}
+
 type dnsTransportSaver struct {
 	model.DNSTransport
 	s *Saver

internal/dnsping/urladdress.go

@@ -48,6 +48,7 @@ func (spr *SinglePingResult) DNSLookupMeasurementList(
 		out = append(out, &measurex.DNSLookupMeasurement{
 			ID:               entry.ID,
 			URLMeasurementID: urlMeasurementID,
+			ReverseAddress:   "",
 			Lookup: &archival.FlatDNSLookupEvent{
 				ALPNs:           entry.ALPNs,
 				Addresses:       entry.Addresses,

internal/engine/experiment/websteps/cache.go

@@ -99,6 +99,7 @@ func (sc *stepsCache) dnsLookup(mx measurex.AbstractMeasurer,
 	o := &measurex.DNSLookupMeasurement{
 		ID:               mx.NextID(),
 		URLMeasurementID: urlMeasurementID,
+		ReverseAddress:   "",
 		Lookup: &archival.FlatDNSLookupEvent{
 			ALPNs:           alpns,
 			Addresses:       addrs,
@@ -108,6 +109,7 @@ func (sc *stepsCache) dnsLookup(mx measurex.AbstractMeasurer,
 			Finished:        now,
 			LookupType:      archival.DNSLookupTypeHTTPS,
 			NS:              []string{},
+			PTRs:            []string{},
 			ResolverAddress: "dnscache",
 			ResolverNetwork: "",
 			Started:         now,

internal/engine/experiment/websteps/client.go

@@ -398,6 +398,7 @@ func (c *Client) importTHMeasurement(mx measurex.AbstractMeasurer, in *THRespons
 		dns := &measurex.DNSLookupMeasurement{
 			ID:               mx.NextID(),
 			URLMeasurementID: cur.ID,
+			ReverseAddress:   e.ReverseAddress,
 			Lookup: &archival.FlatDNSLookupEvent{
 				ALPNs:           e.ALPNs(),
 				Addresses:       e.Addresses(),
@@ -407,6 +408,7 @@ func (c *Client) importTHMeasurement(mx measurex.AbstractMeasurer, in *THRespons
 				Finished:        now,
 				LookupType:      e.LookupType(),
 				NS:              e.NS(),
+				PTRs:            e.PTRs(),
 				ResolverAddress: e.ResolverAddress(),
 				ResolverNetwork: e.ResolverNetwork(),
 				Started:         now,

internal/engine/experiment/websteps/th.go

@@ -549,6 +549,7 @@ func (thr *THRequestHandler) step(
 		um.DNS = append(um.DNS, m)
 	}
 	probeAddrs := thr.addProbeDNS(mx, um, req.Plan)
+	revch := thr.reverseDNSLookupAsync(ctx, mx, um, probeAddrs)
 	// Implementation note: of course it doesn't make sense here for the
 	// test helper to follow bogons discovered by the client :^)
 	epplan, _ := um.NewEndpointPlan(thr.logger(), measurex.EndpointPlanningExcludeBogons)
@@ -562,10 +563,45 @@ func (thr *THRequestHandler) step(
 	for m := range mx.MeasureEndpoints(ctx, epplan...) {
 		um.Endpoint = append(um.Endpoint, m)
 	}
-	thr.saver().Save(um) // allows saving the measurement for analysis
+	um.DNS = append(um.DNS, <-revch...) // merge async results of the reverse lookup
+	thr.saver().Save(um)                // allows saving the measurement for analysis
 	return thr.serialize(um), nil
 }
 
+// reverseDNSLookupAsync performs a reverse DNS lookup for all the IP addresses we know.
+func (thr *THRequestHandler) reverseDNSLookupAsync(ctx context.Context, mx measurex.AbstractMeasurer,
+	um *measurex.URLMeasurement, probeAddrs []string) <-chan []*measurex.DNSLookupMeasurement {
+	out := make(chan []*measurex.DNSLookupMeasurement)
+	go func() {
+		// 0. close channel when done, we'll return a nil list in the worst case
+		defer close(out)
+		// 1. build a list of unique IP addresses to reverse lookup
+		uniqm := map[string]int{}
+		for _, dns := range um.DNS {
+			for _, addr := range dns.Addresses() {
+				uniqm[addr]++
+			}
+		}
+		for _, addr := range probeAddrs {
+			uniqm[addr]++
+		}
+		uniq := []string{}
+		for addr := range uniqm {
+			uniq = append(uniq, addr)
+		}
+		// 2. generate a reverse lookup plan
+		plan := um.NewDNSReverseLookupPlans(uniq, thr.resolvers()...)
+		// 3. collect results
+		v := []*measurex.DNSLookupMeasurement{}
+		for m := range mx.DNSLookups(ctx, plan...) {
+			v = append(v, m)
+		}
+		// 4. return results to the caller.
+		out <- v
+	}()
+	return out
+}
+
 // patchEndpointPlan returns a modified endpoint plan where:
 //
 // 1. we include cookies from the probe (if any);
@@ -684,6 +720,7 @@ func (thr *THRequestHandler) simplifyDNS(
 		out = append(out, &measurex.DNSLookupMeasurement{
 			ID:               0,
 			URLMeasurementID: 0,
+			ReverseAddress:   entry.ReverseAddress,
 			Lookup: &archival.FlatDNSLookupEvent{
 				ALPNs:           entry.ALPNs(),
 				Addresses:       entry.Addresses(),
@@ -693,6 +730,7 @@ func (thr *THRequestHandler) simplifyDNS(
 				Finished:        thhResponseTime,
 				LookupType:      entry.LookupType(),
 				NS:              entry.NS(),
+				PTRs:            entry.PTRs(),
 				ResolverAddress: entry.ResolverAddress(),
 				ResolverNetwork: entry.ResolverNetwork(),
 				Started:         thhResponseTime,

internal/measurex/archival.go

@@ -21,6 +21,10 @@ type ArchivalDNSLookupMeasurement struct {
 	// Domain is the domain this lookup refers to.
 	Domain string `json:"domain"`
 
+	// ReverseAddress is a convenience field to help analysis that is only
+	// set when we're performing a reverse DNS lookup.
+	ReverseAddress string `json:"reverse_address,omitempty"`
+
 	// ResolverNetwork is the network used by this resolver.
 	ResolverNetwork string `json:"resolver_network"`
 
@@ -42,6 +46,7 @@ func (m *DNSLookupMeasurement) ToArchival(begin time.Time) ArchivalDNSLookupMeas
 	return ArchivalDNSLookupMeasurement{
 		ID:              m.ID,
 		Domain:          m.Domain(),
+		ReverseAddress:  m.ReverseAddress,
 		ResolverNetwork: string(m.ResolverNetwork()),
 		ResolverAddress: m.ResolverAddress(),
 		Failure:         m.Failure().ToArchivalFailure(),