Add custom provider

README.md

@@ -63,11 +63,11 @@ CONFIGURATION:
    -pc, -provider-config string  provider config file (default "$HOME/.config/cloudlist/provider-config.yaml")
 
 FILTERS:
-   -p, -provider value    display results for given providers (comma-separated) (default linode,fastly,heroku,terraform,digitalocean,consul,cloudflare,hetzner,nomad,do,scw,openstack,alibaba,aws,gcp,namecheap,kubernetes,azure)
+   -p, -provider value    display results for given providers (comma-separated) (default linode,fastly,heroku,terraform,digitalocean,consul,cloudflare,hetzner,nomad,do,scw,openstack,alibaba,aws,gcp,namecheap,kubernetes,azure, custom)
    -id string[]           display results for given ids (comma-separated)
    -host                  display only hostnames in results
    -ip                    display only ips in results
-   -s, -service value     query and display results from given service (comma-separated)) (default cloudfront,gke,domain,compute,ec2,instance,cloud-function,app,eks,consul,droplet,vm,ecs,fastly,alb,s3,lambda,elb,cloud-run,route53,publicip,dns,service,nomad,lightsail,ingress,apigateway)
+   -s, -service value     query and display results from given service (comma-separated)) (default cloudfront,gke,domain,compute,ec2,instance,cloud-function,app,eks,custom,consul,droplet,vm,ecs,fastly,alb,s3,lambda,elb,cloud-run,route53,publicip,dns,service,nomad,lightsail,ingress,apigateway)
    -ep, -exclude-private  exclude private ips in cli output
 
 UPDATE:

go.mod

@@ -184,11 +184,33 @@ require (
 )
 
 require (
+	github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect
+	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
+	github.com/akrylysov/pogreb v0.10.1 // indirect
 	github.com/alecthomas/chroma/v2 v2.14.0 // indirect
+	github.com/bits-and-blooms/bitset v1.13.0 // indirect
 	github.com/charmbracelet/lipgloss v0.13.0 // indirect
 	github.com/charmbracelet/x/ansi v0.3.2 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/gaissmai/bart v0.9.5 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/projectdiscovery/fastdialer v0.2.13 // indirect
+	github.com/projectdiscovery/hmap v0.0.70 // indirect
+	github.com/projectdiscovery/networkpolicy v0.0.9 // indirect
+	github.com/projectdiscovery/retryabledns v1.0.88 // indirect
+	github.com/projectdiscovery/retryablehttp-go v1.0.91 // indirect
+	github.com/refraction-networking/utls v1.6.7 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/syndtr/goleveldb v1.0.0 // indirect
+	github.com/tidwall/btree v1.4.3 // indirect
+	github.com/tidwall/buntdb v1.3.0 // indirect
+	github.com/tidwall/grect v0.1.4 // indirect
+	github.com/tidwall/rtred v0.1.2 // indirect
+	github.com/tidwall/tinyqueue v0.1.1 // indirect
+	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
+	github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect
+	go.etcd.io/bbolt v1.3.7 // indirect
 	golang.org/x/sync v0.10.0 // indirect
 )

internal/runner/options.go

@@ -336,4 +336,17 @@ const defaultProviderConfigFile = `#  #Provider configuration file for cloudlist
 #  # username is Openstack username used to authenticate
 #  username: <openstack-username>
 #  # password is Openstack password used to authenticate
-#  password: <openstack-password>`
+#  password: <openstack-password>
+
+# - # provider is the name of the provider
+#   provider: custom
+#   id: test
+#   # urls is a list of API endpoints or resources to be accessed
+#   urls:
+#     - https://api.example.com/resource1
+#     - https://api.example.com/resource2
+#   # headers is a map of custom headers to be included in requests
+#   headers:
+#     Authorization: $CUSTOM_AUTH_TOKEN
+#     Content-Type: application/json
+#     X-Custom-Header: $CUSTOM_HEADER`

pkg/inventory/inventory.go

@@ -9,6 +9,7 @@ import (
 	"github.com/projectdiscovery/cloudlist/pkg/providers/azure"
 	"github.com/projectdiscovery/cloudlist/pkg/providers/cloudflare"
 	"github.com/projectdiscovery/cloudlist/pkg/providers/consul"
+	"github.com/projectdiscovery/cloudlist/pkg/providers/custom"
 	"github.com/projectdiscovery/cloudlist/pkg/providers/digitalocean"
 	"github.com/projectdiscovery/cloudlist/pkg/providers/fastly"
 	"github.com/projectdiscovery/cloudlist/pkg/providers/gcp"
@@ -69,6 +70,7 @@ var Providers = map[string][]string{
 	"hetzner":      hetzner.Services,
 	"openstack":    openstack.Services,
 	"kubernetes":   k8s.Services,
+	"custom":       custom.Services,
 }
 
 func GetProviders() []string {
@@ -124,6 +126,8 @@ func nameToProvider(value string, block schema.OptionBlock) (schema.Provider, er
 		return openstack.New(block)
 	case "kubernetes":
 		return k8s.New(block)
+	case "custom":
+		return custom.New(block)
 	default:
 		return nil, fmt.Errorf("invalid provider name found: %s", value)
 	}

pkg/providers/custom/custom.go

@@ -0,0 +1,133 @@
+package custom
+
+import (
+	"context"
+	"net/url"
+	"strings"
+
+	"github.com/projectdiscovery/cloudlist/pkg/schema"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/networkpolicy"
+	"github.com/projectdiscovery/retryablehttp-go"
+	sliceutil "github.com/projectdiscovery/utils/slice"
+)
+
+var Services = []string{"custom"}
+
+type ProviderOptions struct {
+	Id       string
+	URLs     []string
+	Headers  map[string]string
+	Services schema.ServiceMap
+}
+
+const (
+	urls         = "urls"
+	headers      = "headers"
+	providerName = "custom"
+)
+
+// Provider is a data provider for custom URLs
+type Provider struct {
+	client     *retryablehttp.Client
+	id         string
+	urlList    []string
+	headerList map[string]string
+	services   schema.ServiceMap
+}
+
+// New creates a new provider client for custom URLs
+func New(block schema.OptionBlock) (*Provider, error) {
+	options := &ProviderOptions{}
+	if err := options.ParseOptionBlock(block); err != nil {
+		return nil, err
+	}
+
+	supportedServicesMap := make(map[string]struct{})
+	for _, s := range Services {
+		supportedServicesMap[s] = struct{}{}
+	}
+
+	services := make(schema.ServiceMap)
+	for _, s := range Services {
+		services[s] = struct{}{}
+	}
+	client := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
+	return &Provider{client: client, id: options.Id, urlList: options.URLs, headerList: options.Headers, services: services}, nil
+}
+
+// Name returns the name of the provider
+func (p *Provider) Name() string {
+	return providerName
+}
+
+// ID returns the name of the provider id
+func (p *Provider) ID() string {
+	return p.id
+}
+
+// Services returns the provider services
+func (p *Provider) Services() []string {
+	return p.services.Keys()
+}
+
+// Resources returns the provider for an resource deployment source.
+func (p *Provider) Resources(ctx context.Context) (*schema.Resources, error) {
+	finalResources := schema.NewResources()
+	serviceProvider := &serviceProvider{client: p.client, id: p.id, urlList: p.urlList, headerList: p.headerList}
+	if services, err := serviceProvider.GetResource(ctx); err == nil {
+		finalResources.Merge(services)
+	}
+	return finalResources, nil
+}
+
+func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error {
+	p.Id, _ = block.GetMetadata("id")
+
+	supportedServicesMap := make(map[string]struct{})
+	for _, s := range Services {
+		supportedServicesMap[s] = struct{}{}
+	}
+	services := make(schema.ServiceMap)
+	if ss, ok := block.GetMetadata("services"); ok {
+		for _, s := range strings.Split(ss, ",") {
+			if _, ok := supportedServicesMap[s]; ok {
+				services[s] = struct{}{}
+			}
+		}
+	}
+	// if no services provided from -service flag, includes all services
+	if len(services) == 0 {
+		for _, s := range Services {
+			services[s] = struct{}{}
+		}
+	}
+
+	np, err := networkpolicy.New(networkpolicy.DefaultOptions)
+	if err != nil {
+		return err
+	}
+
+	if urlListStr, ok := block.GetMetadata(urls); ok {
+		for _, urlStr := range sliceutil.Dedupe(strings.Split(urlListStr, ",")) {
+			if parsedUrl, err := url.Parse(urlStr); err != nil || !np.Validate(parsedUrl.Hostname()) {
+				gologger.Warning().Msgf("Invalid URL: %s\n", urlStr)
+				continue
+			}
+			p.URLs = append(p.URLs, urlStr)
+		}
+	}
+
+	if headerListStr, ok := block.GetMetadata(headers); ok {
+		p.Headers = make(map[string]string)
+		for _, header := range sliceutil.Dedupe(strings.Split(headerListStr, ",")) {
+			if parts := strings.SplitN(header, ":", 2); len(parts) == 2 {
+				key, value := strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])
+				if key != "" && value != "" {
+					p.Headers[key] = value
+				}
+			}
+		}
+	}
+	return nil
+}

pkg/providers/custom/services.go

@@ -0,0 +1,71 @@
+package custom
+
+import (
+	"bufio"
+	"context"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/projectdiscovery/cloudlist/pkg/schema"
+	"github.com/projectdiscovery/retryablehttp-go"
+)
+
+type serviceProvider struct {
+	client     *retryablehttp.Client
+	id         string
+	urlList    []string
+	headerList map[string]string
+}
+
+var re = regexp.MustCompile(`(?i)\b(?:https?://)?((?:[a-z0-9-]+\.)+)([a-z0-9-]+\.[a-z]{2,})(\.[a-z]{2,})?\b`)
+
+// GetResource returns all the resources in the store for a provider.
+func (d *serviceProvider) GetResource(ctx context.Context) (*schema.Resources, error) {
+	list := schema.NewResources()
+
+	for _, urlStr := range d.urlList {
+		req, err := retryablehttp.NewRequest("GET", urlStr, nil)
+		if err != nil {
+			continue
+		}
+
+		for k, v := range d.headerList {
+			req.Header.Set(k, v)
+		}
+
+		response, err := d.client.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		defer response.Body.Close()
+
+		if response.StatusCode != http.StatusOK {
+			continue
+		}
+
+		subdomainSet := make(map[string]struct{})
+		scanner := bufio.NewScanner(response.Body)
+		for scanner.Scan() {
+			line := strings.TrimSpace(scanner.Text())
+			if line == "" {
+				continue
+			}
+			line, _ = url.QueryUnescape(line)
+			for _, subdomain := range re.FindAllString(line, -1) {
+				subdomainSet[strings.ToLower(subdomain)] = struct{}{}
+			}
+		}
+
+		for subdomain := range subdomainSet {
+			list.Append(&schema.Resource{
+				Provider: providerName,
+				ID:       d.id,
+				DNSName:  subdomain,
+				Service:  providerName,
+			})
+		}
+	}
+	return list, nil
+}

pkg/schema/schema.go

@@ -157,7 +157,7 @@ func (ob *OptionBlock) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	// Convert raw map to OptionBlock and handle special cases
 	for key, value := range rawMap {
 		switch key {
-		case "account_ids":
+		case "account_ids", "urls":
 			if valueArr, ok := value.([]interface{}); ok {
 				var strArr []string
 				for _, v := range valueArr {
@@ -172,6 +172,14 @@ func (ob *OptionBlock) UnmarshalYAML(unmarshal func(interface{}) error) error {
 				}
 				(*ob)[key] = strings.Join(strArr, ",")
 			}
+		case "headers":
+			if valueMap, ok := value.(map[interface{}]interface{}); ok {
+				var strArr []string
+				for k, v := range valueMap {
+					strArr = append(strArr, fmt.Sprintf("%s: %s", k, v))
+				}
+				(*ob)[key] = strings.Join(strArr, ",")
+			}
 		default:
 			(*ob)[key] = fmt.Sprint(value)
 		}