Merge pull request #56 from projectdiscovery/dev

Dev

.github/workflows/build-test.yml

@@ -0,0 +1,26 @@
+name: 🔨 Build Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:  
+  build:
+    name: Test Builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Test
+        run: go test ./...
+
+      # Todo: create example folder
+      # - name: Build
+      #   run: go build .

.github/workflows/codeql-analysis.yml

@@ -2,7 +2,6 @@ name: 🚨 CodeQL Analysis
 
 on:
   workflow_dispatch:
-  push:
   pull_request:
     branches:
       - dev
@@ -24,16 +23,16 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
 
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/lint-test.yml

@@ -10,10 +10,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3.2.0
         with:
           version: latest
           args: --timeout 5m
-          working-directory: .
+          working-directory: .

.gitignore

@@ -0,0 +1 @@
+vendor/

fastdialer/dialer.go

@@ -79,7 +79,10 @@ func NewDialer(options Options) (*Dialer, error) {
 		// nolint:errcheck // if they cannot be loaded it's not a hard failure
 		loadHostsFile(hm)
 	}
-	dnsclient := retryabledns.New(resolvers, options.MaxRetries)
+	dnsclient, err := retryabledns.New(resolvers, options.MaxRetries)
+	if err != nil {
+		return nil, err
+	}
 
 	var npOptions networkpolicy.Options
 	// Populate deny list if necessary
@@ -235,6 +238,9 @@ func (d *Dialer) dial(ctx context.Context, network, address string, shouldUseTLS
 					return nil, setErr
 				}
 			}
+			if d.options.OnDialCallback != nil {
+				d.options.OnDialCallback(hostname, ip)
+			}
 			if d.options.WithTLSData && shouldUseTLS {
 				if connTLS, ok := conn.(*tls.Conn); ok {
 					var data bytes.Buffer
@@ -367,8 +373,10 @@ func (d *Dialer) GetDNSData(hostname string) (*retryabledns.DNSData, error) {
 		if data == nil {
 			return nil, ResolveHostError
 		}
-		b, _ := data.Marshal()
-		err = d.hm.Set(hostname, b)
+		if len(data.A)+len(data.AAAA) > 0 {
+			b, _ := data.Marshal()
+			err = d.hm.Set(hostname, b)
+		}
 		if err != nil {
 			return nil, err
 		}

fastdialer/dialer_test.go

@@ -23,7 +23,8 @@ func TestDialer(t *testing.T) {
 	options.CacheType = Hybrid
 	options.CacheMemoryMaxItems = 100
 	testDialer(t, options)
-	testDialerIpv6(t, options)
+
+	// testDialerIpv6(t, options) not supported by GitHub VMs
 }
 
 func testDialer(t *testing.T, options Options) {
@@ -52,24 +53,25 @@ func testDialer(t *testing.T, options Options) {
 	fd.Close()
 }
 
+// nolint
 func testDialerIpv6(t *testing.T, options Options) {
 	// disk based
 	fd, err := NewDialer(options)
 	if err != nil {
-		t.Errorf("couldn't create fastdialer instance: %s", err)
+		t.Fatalf("couldn't create fastdialer instance: %s", err)
 	}
 
 	// valid resolution + cache
 	ctx := context.Background()
 	conn, err := fd.Dial(ctx, "tcp", "ipv6.google.com:80")
 	if err != nil || conn == nil {
-		t.Errorf("couldn't connect to target: %s", err)
+		t.Fatalf("couldn't connect to target: %s", err)
 	}
 	conn.Close()
 	// retrieve cached data
 	data, err := fd.GetDNSData("ipv6.google.com")
 	if err != nil || data == nil {
-		t.Errorf("couldn't retrieve dns data: %s", err)
+		t.Fatalf("couldn't retrieve dns data: %s", err)
 	}
 	if len(data.AAAA) == 0 {
 		t.Error("no AAAA results found")
@@ -79,11 +81,11 @@ func testDialerIpv6(t *testing.T, options Options) {
 	// this test passes, but will fail if the hard-coded ipv6 address changes
 	// need to find a better way to test this
 	/*
-	    conn, err = fd.Dial(ctx, "tcp", "ipv6.google.com:80:[2607:f8b0:4006:807::200e]")
-		if err != nil || conn == nil {
-			t.Errorf("couldn't connect to target: %s", err)
-		}
-		conn.Close()
+		    conn, err = fd.Dial(ctx, "tcp", "ipv6.google.com:80:[2607:f8b0:4006:807::200e]")
+			if err != nil || conn == nil {
+				t.Errorf("couldn't connect to target: %s", err)
+			}
+			conn.Close()
 	*/
 
 	// cleanup

fastdialer/options.go

@@ -47,6 +47,7 @@ type Options struct {
 	Dialer              *net.Dialer
 	WithZTLS            bool
 	SNIName             string
+	OnDialCallback      func(hostname, IP string)
 }
 
 // DefaultOptions of the cache

go.mod

@@ -7,18 +7,18 @@ require (
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
-	github.com/miekg/dns v1.1.43 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345
 	github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa
-	github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619
+	github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8
 	github.com/projectdiscovery/networkpolicy v0.0.1
-	github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a
+	github.com/projectdiscovery/retryabledns v1.0.15
 	github.com/rogpeppe/go-internal v1.8.0 // indirect
-	github.com/stretchr/testify v1.7.4
+	github.com/stretchr/testify v1.8.0
 	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6
 	github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b
 	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4
+	golang.org/x/net v0.0.0-20220728211354-c7608f3a8462
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 )