diff --git a/go.mod b/go.mod index 5cf97a86..a693d20a 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/cdncheck v1.0.9 // indirect - github.com/projectdiscovery/hmap v0.0.22 + github.com/projectdiscovery/hmap v0.0.22 // indirect github.com/projectdiscovery/networkpolicy v0.0.6 // indirect github.com/projectdiscovery/retryabledns v1.0.38 // indirect github.com/projectdiscovery/retryablehttp-go v1.0.31 diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 2c1c8f99..35bf414c 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -2,7 +2,6 @@ package runner import ( "bufio" - "bytes" "net" "net/url" "os" @@ -16,7 +15,6 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" - "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/mapcidr" "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/tlsx/pkg/output" @@ -26,13 +24,10 @@ import ( "github.com/projectdiscovery/tlsx/pkg/tlsx/openssl" errorutil "github.com/projectdiscovery/utils/errors" iputil "github.com/projectdiscovery/utils/ip" - mapsutil "github.com/projectdiscovery/utils/maps" sliceutil "github.com/projectdiscovery/utils/slice" updateutils "github.com/projectdiscovery/utils/update" ) -var hostnamesHm *hybrid.HybridMap - // Runner is a client for running the enumeration process type Runner struct { hasStdin bool @@ -113,11 +108,6 @@ func New(options *clients.Options) (*Runner, error) { } runner.outputWriter = outputWriter - hostnamesHm, err = hybrid.New(hybrid.DefaultDiskOptions) - if err != nil { - return nil, errorutil.NewWithErr(err).Msgf("could not create hmap for hostnames") - } - return runner, nil } @@ -157,17 +147,6 @@ func (r *Runner) Execute() error { close(inputs) wg.Wait() - //FIXME: this is a hack to print deduplicated hostnames - if r.options.DisplayDns && !r.options.JSON { - builder := &bytes.Buffer{} - hostnamesHm.Scan(func(k, _ []byte) error { - builder.WriteString(string(k)) - builder.WriteString("\n") - return nil - }) - _, _ = os.Stdout.Write(builder.Bytes()) - } - // Print the stats if auto fallback mode is used if r.options.ScanMode == "auto" { gologger.Info().Msgf("Connections made using crypto/tls: %d, zcrypto/tls: %d, openssl: %d", stats.LoadCryptoTLSConnections(), stats.LoadZcryptoTLSConnections(), stats.LoadOpensslTLSConnections()) @@ -206,17 +185,6 @@ func (r *Runner) processInputElementWorker(inputs chan taskInput, wg *sync.WaitG continue } - if r.options.DisplayDns && response.CertificateResponse != nil { - uniqueHostnames := getUniqueHostnamesPerInput(response.CertificateResponse) - response.CertificateResponse.Hostname = uniqueHostnames - for _, hostname := range uniqueHostnames { - _ = hostnamesHm.Set(hostname, nil) - } - if !r.options.JSON { - continue - } - } - if err := r.outputWriter.Write(response); err != nil { gologger.Warning().Msgf("Could not write output %s: %s", task.Address(), err) continue @@ -224,22 +192,6 @@ func (r *Runner) processInputElementWorker(inputs chan taskInput, wg *sync.WaitG } } -func getUniqueHostnamesPerInput(certResponse *clients.CertificateResponse) []string { - hostnameSet := map[string]struct{}{} - if certResponse.SubjectCN != "" { - hostnameSet[trimWildcardPrefix(certResponse.SubjectCN)] = struct{}{} - } - for _, hostname := range certResponse.SubjectAN { - hostnameSet[trimWildcardPrefix(hostname)] = struct{}{} - } - - return mapsutil.GetKeys(hostnameSet) -} - -func trimWildcardPrefix(hostname string) string { - return strings.TrimPrefix(hostname, "*.") -} - // normalizeAndQueueInputs normalizes the inputs and queues them for execution func (r *Runner) normalizeAndQueueInputs(inputs chan taskInput) error { // Process Normal Inputs diff --git a/pkg/output/output.go b/pkg/output/output.go index ff990e78..0ad65237 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -12,9 +12,18 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" errorutil "github.com/projectdiscovery/utils/errors" + mapsutil "github.com/projectdiscovery/utils/maps" "golang.org/x/exp/maps" ) +var ( + // when unique domains are displayed with `-dns` flag. tlsx json/struct already + // contains unique domains for each certificate + // globalDedupe is meant to be used when running in cli mode with multiple inputs + // ex: google.com and youtube.com may have same wildcard certificate or some overlapping domains + globalDedupe = mapsutil.NewSyncLockMap[string, struct{}]() +) + // Writer is an interface which writes output to somewhere for katana events. type Writer interface { // Close closes the output writer interface @@ -69,6 +78,10 @@ func (w *StandardWriter) Write(event *clients.Response) error { return errorutil.NewWithErr(err).Msgf("could not format output") } data = bytes.TrimSuffix(data, []byte("\n")) // remove last newline + if len(data) == 0 { + // this happens when -dns flag is used and two domains have same certificate hence deduped + return nil + } w.outputMutex.Lock() defer w.outputMutex.Unlock() @@ -108,9 +121,22 @@ func (w *StandardWriter) formatStandard(output *clients.Response) ([]byte, error if output.CertificateResponse == nil { return nil, errorutil.New("empty leaf certificate") } - + cert := output.CertificateResponse builder := &bytes.Buffer{} + if w.options.DisplayDns { + for _, hname := range cert.Domains { + if _, ok := globalDedupe.Get(hname); ok { + continue + } + _ = globalDedupe.Set(hname, struct{}{}) + builder.WriteString(hname) + builder.WriteString("\n") + } + outputdata := builder.Bytes() + return outputdata, nil + } + if !w.options.RespOnly { builder.WriteString(output.Host) builder.WriteString(":") @@ -124,8 +150,6 @@ func (w *StandardWriter) formatStandard(output *clients.Response) ([]byte, error outputPrefix := builder.String() builder.Reset() - cert := output.CertificateResponse - var names []string if w.options.SAN { names = append(names, cert.SubjectAN...) diff --git a/pkg/tlsx/clients/clients.go b/pkg/tlsx/clients/clients.go index d0aff9a3..d4c487a1 100644 --- a/pkg/tlsx/clients/clients.go +++ b/pkg/tlsx/clients/clients.go @@ -271,8 +271,8 @@ type CertificateResponse struct { SubjectOrg []string `json:"subject_org,omitempty"` // SubjectAN is a list of Subject Alternative Names for the certificate SubjectAN []string `json:"subject_an,omitempty"` - // Hostname is list of deduplicated subject_cn + subject_an - Hostname []string `json:"hostname,omitempty"` + // Domains is list of deduplicated subject_cn + subject_an + Domains []string `json:"domains,omitempty"` //Serial is the certificate serial number Serial string `json:"serial,omitempty"` // IssuerDN is the distinguished name for cert diff --git a/pkg/tlsx/clients/utils.go b/pkg/tlsx/clients/utils.go index 451d984a..0f177f0f 100644 --- a/pkg/tlsx/clients/utils.go +++ b/pkg/tlsx/clients/utils.go @@ -11,6 +11,7 @@ import ( errorutil "github.com/projectdiscovery/utils/errors" iputil "github.com/projectdiscovery/utils/ip" + mapsutil "github.com/projectdiscovery/utils/maps" ) func Convertx509toResponse(options *Options, hostname string, cert *x509.Certificate, showcert bool) *CertificateResponse { @@ -42,9 +43,28 @@ func Convertx509toResponse(options *Options, hostname string, cert *x509.Certifi if showcert { response.Certificate = PemEncode(cert.Raw) } + if options.DisplayDns { + response.Domains = GetUniqueDomainsFromCert(response) + } return response } +// GetUniqueDomainsFromCert returns unique domains extracted from certificate response +func GetUniqueDomainsFromCert(resp *CertificateResponse) []string { + domains := map[string]struct{}{} + for _, domain := range resp.SubjectAN { + domains[trimWildcardPrefix(domain)] = struct{}{} + } + if resp.SubjectCN != "" { + domains[trimWildcardPrefix(resp.SubjectCN)] = struct{}{} + } + return mapsutil.GetKeys(domains) +} + +func trimWildcardPrefix(hostname string) string { + return strings.TrimPrefix(hostname, "*.") +} + // IntersectStringSlices returns intersection of two string slices func IntersectStringSlices(s1 []string, s2 []string) []string { res := []string{} diff --git a/pkg/tlsx/ztls/utils.go b/pkg/tlsx/ztls/utils.go index 3b600f5c..85afce4b 100644 --- a/pkg/tlsx/ztls/utils.go +++ b/pkg/tlsx/ztls/utils.go @@ -63,6 +63,9 @@ func ConvertCertificateToResponse(options *clients.Options, hostname string, cer }, Serial: clients.FormatToSerialNumber(cert.SerialNumber), } + if options.DisplayDns { + response.Domains = clients.GetUniqueDomainsFromCert(response) + } if options.Cert { response.Certificate = clients.PemEncode(cert.Raw) }