refactor -dns flag

go.mod

@@ -90,7 +90,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
 	github.com/projectdiscovery/cdncheck v1.0.9 // indirect
-	github.com/projectdiscovery/hmap v0.0.22
+	github.com/projectdiscovery/hmap v0.0.22 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.6 // indirect
 	github.com/projectdiscovery/retryabledns v1.0.38 // indirect
 	github.com/projectdiscovery/retryablehttp-go v1.0.31

internal/runner/runner.go

@@ -2,7 +2,6 @@ package runner
 
 import (
 	"bufio"
-	"bytes"
 	"net"
 	"net/url"
 	"os"
@@ -16,7 +15,6 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/formatter"
 	"github.com/projectdiscovery/gologger/levels"
-	"github.com/projectdiscovery/hmap/store/hybrid"
 	"github.com/projectdiscovery/mapcidr"
 	"github.com/projectdiscovery/mapcidr/asn"
 	"github.com/projectdiscovery/tlsx/pkg/output"
@@ -26,13 +24,10 @@ import (
 	"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	iputil "github.com/projectdiscovery/utils/ip"
-	mapsutil "github.com/projectdiscovery/utils/maps"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	updateutils "github.com/projectdiscovery/utils/update"
 )
 
-var hostnamesHm *hybrid.HybridMap
-
 // Runner is a client for running the enumeration process
 type Runner struct {
 	hasStdin     bool
@@ -113,11 +108,6 @@ func New(options *clients.Options) (*Runner, error) {
 	}
 	runner.outputWriter = outputWriter
 
-	hostnamesHm, err = hybrid.New(hybrid.DefaultDiskOptions)
-	if err != nil {
-		return nil, errorutil.NewWithErr(err).Msgf("could not create hmap for hostnames")
-	}
-
 	return runner, nil
 }
 
@@ -157,17 +147,6 @@ func (r *Runner) Execute() error {
 	close(inputs)
 	wg.Wait()
 
-	//FIXME: this is a hack to print deduplicated hostnames
-	if r.options.DisplayDns && !r.options.JSON {
-		builder := &bytes.Buffer{}
-		hostnamesHm.Scan(func(k, _ []byte) error {
-			builder.WriteString(string(k))
-			builder.WriteString("\n")
-			return nil
-		})
-		_, _ = os.Stdout.Write(builder.Bytes())
-	}
-
 	// Print the stats if auto fallback mode is used
 	if r.options.ScanMode == "auto" {
 		gologger.Info().Msgf("Connections made using crypto/tls: %d, zcrypto/tls: %d, openssl: %d", stats.LoadCryptoTLSConnections(), stats.LoadZcryptoTLSConnections(), stats.LoadOpensslTLSConnections())
@@ -206,40 +185,13 @@ func (r *Runner) processInputElementWorker(inputs chan taskInput, wg *sync.WaitG
 			continue
 		}
 
-		if r.options.DisplayDns && response.CertificateResponse != nil {
-			uniqueHostnames := getUniqueHostnamesPerInput(response.CertificateResponse)
-			response.CertificateResponse.Hostname = uniqueHostnames
-			for _, hostname := range uniqueHostnames {
-				_ = hostnamesHm.Set(hostname, nil)
-			}
-			if !r.options.JSON {
-				continue
-			}
-		}
-
 		if err := r.outputWriter.Write(response); err != nil {
 			gologger.Warning().Msgf("Could not write output %s: %s", task.Address(), err)
 			continue
 		}
 	}
 }
 
-func getUniqueHostnamesPerInput(certResponse *clients.CertificateResponse) []string {
-	hostnameSet := map[string]struct{}{}
-	if certResponse.SubjectCN != "" {
-		hostnameSet[trimWildcardPrefix(certResponse.SubjectCN)] = struct{}{}
-	}
-	for _, hostname := range certResponse.SubjectAN {
-		hostnameSet[trimWildcardPrefix(hostname)] = struct{}{}
-	}
-
-	return mapsutil.GetKeys(hostnameSet)
-}
-
-func trimWildcardPrefix(hostname string) string {
-	return strings.TrimPrefix(hostname, "*.")
-}
-
 // normalizeAndQueueInputs normalizes the inputs and queues them for execution
 func (r *Runner) normalizeAndQueueInputs(inputs chan taskInput) error {
 	// Process Normal Inputs

pkg/output/output.go

@@ -12,9 +12,18 @@ import (
 	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	mapsutil "github.com/projectdiscovery/utils/maps"
 	"golang.org/x/exp/maps"
 )
 
+var (
+	// when unique domains are displayed with `-dns` flag. tlsx json/struct already
+	// contains unique domains for each certificate
+	// globalDedupe is meant to be used when running in cli mode with multiple inputs
+	// ex: google.com and youtube.com may have same wildcard certificate or some overlapping domains
+	globalDedupe = mapsutil.NewSyncLockMap[string, struct{}]()
+)
+
 // Writer is an interface which writes output to somewhere for katana events.
 type Writer interface {
 	// Close closes the output writer interface
@@ -69,6 +78,10 @@ func (w *StandardWriter) Write(event *clients.Response) error {
 		return errorutil.NewWithErr(err).Msgf("could not format output")
 	}
 	data = bytes.TrimSuffix(data, []byte("\n")) // remove last newline
+	if len(data) == 0 {
+		// this happens when -dns flag is used and two domains have same certificate hence deduped
+		return nil
+	}
 
 	w.outputMutex.Lock()
 	defer w.outputMutex.Unlock()
@@ -108,9 +121,22 @@ func (w *StandardWriter) formatStandard(output *clients.Response) ([]byte, error
 	if output.CertificateResponse == nil {
 		return nil, errorutil.New("empty leaf certificate")
 	}
-
+	cert := output.CertificateResponse
 	builder := &bytes.Buffer{}
 
+	if w.options.DisplayDns {
+		for _, hname := range cert.Domains {
+			if _, ok := globalDedupe.Get(hname); ok {
+				continue
+			}
+			_ = globalDedupe.Set(hname, struct{}{})
+			builder.WriteString(hname)
+			builder.WriteString("\n")
+		}
+		outputdata := builder.Bytes()
+		return outputdata, nil
+	}
+
 	if !w.options.RespOnly {
 		builder.WriteString(output.Host)
 		builder.WriteString(":")
@@ -124,8 +150,6 @@ func (w *StandardWriter) formatStandard(output *clients.Response) ([]byte, error
 	outputPrefix := builder.String()
 	builder.Reset()
 
-	cert := output.CertificateResponse
-
 	var names []string
 	if w.options.SAN {
 		names = append(names, cert.SubjectAN...)

pkg/tlsx/clients/clients.go

@@ -271,8 +271,8 @@ type CertificateResponse struct {
 	SubjectOrg []string `json:"subject_org,omitempty"`
 	// SubjectAN is a list of Subject Alternative Names for the certificate
 	SubjectAN []string `json:"subject_an,omitempty"`
-	// Hostname is list of  deduplicated subject_cn + subject_an
-	Hostname []string `json:"hostname,omitempty"`
+	// Domains is list of  deduplicated subject_cn + subject_an
+	Domains []string `json:"domains,omitempty"`
 	//Serial is the certificate serial number
 	Serial string `json:"serial,omitempty"`
 	// IssuerDN is the distinguished name for cert

pkg/tlsx/clients/utils.go

@@ -11,6 +11,7 @@ import (
 
 	errorutil "github.com/projectdiscovery/utils/errors"
 	iputil "github.com/projectdiscovery/utils/ip"
+	mapsutil "github.com/projectdiscovery/utils/maps"
 )
 
 func Convertx509toResponse(options *Options, hostname string, cert *x509.Certificate, showcert bool) *CertificateResponse {
@@ -42,9 +43,28 @@ func Convertx509toResponse(options *Options, hostname string, cert *x509.Certifi
 	if showcert {
 		response.Certificate = PemEncode(cert.Raw)
 	}
+	if options.DisplayDns {
+		response.Domains = GetUniqueDomainsFromCert(response)
+	}
 	return response
 }
 
+// GetUniqueDomainsFromCert returns unique domains extracted from certificate response
+func GetUniqueDomainsFromCert(resp *CertificateResponse) []string {
+	domains := map[string]struct{}{}
+	for _, domain := range resp.SubjectAN {
+		domains[trimWildcardPrefix(domain)] = struct{}{}
+	}
+	if resp.SubjectCN != "" {
+		domains[trimWildcardPrefix(resp.SubjectCN)] = struct{}{}
+	}
+	return mapsutil.GetKeys(domains)
+}
+
+func trimWildcardPrefix(hostname string) string {
+	return strings.TrimPrefix(hostname, "*.")
+}
+
 // IntersectStringSlices returns intersection of two string slices
 func IntersectStringSlices(s1 []string, s2 []string) []string {
 	res := []string{}

pkg/tlsx/ztls/utils.go

@@ -63,6 +63,9 @@ func ConvertCertificateToResponse(options *clients.Options, hostname string, cer
 		},
 		Serial: clients.FormatToSerialNumber(cert.SerialNumber),
 	}
+	if options.DisplayDns {
+		response.Domains = clients.GetUniqueDomainsFromCert(response)
+	}
 	if options.Cert {
 		response.Certificate = clients.PemEncode(cert.Raw)
 	}