feat: auto wildcard domain detection for multi-domain runs (#924)

internal/runner/options.go

@@ -59,6 +59,7 @@ type Options struct {
 	TraceMaxRecursion     int
 	WildcardThreshold     int
 	WildcardDomain        string
+	AutoWildcard          bool
 	ShowStatistics        bool
 	rcodes                map[int]struct{}
 	RCode                 string
@@ -189,6 +190,7 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"),
 		flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"),
 		flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"),
+		flagSet.BoolVar(&options.AutoWildcard, "auto-wildcard", false, "auto-detect wildcard root domain per host (multi-domain mode)"),
 		flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"),
 	)
 
@@ -304,9 +306,7 @@ func (options *Options) validateOptions() {
 		if options.Resume {
 			gologger.Fatal().Msgf("resume not supported in stream mode")
 		}
-		if options.WildcardDomain != "" {
-			gologger.Fatal().Msgf("wildcard not supported in stream mode")
-		}
+		// wildcard filtering is handled via buffered execution path even when stream is enabled
 		if options.ShowStatistics {
 			gologger.Fatal().Msgf("stats not supported in stream mode")
 		}

internal/runner/runner.go

@@ -28,9 +28,16 @@ import (
 	iputil "github.com/projectdiscovery/utils/ip"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	sliceutil "github.com/projectdiscovery/utils/slice"
+	"golang.org/x/net/publicsuffix"
 )
 
 // Runner is a client for running the enumeration process.
+// wildcardTask carries the input host and the wildcard root domain used for matching.
+type wildcardTask struct {
+	host   string
+	domain string
+}
+
 type Runner struct {
 	options             *Options
 	dnsx                *dnsx.DNSX
@@ -39,7 +46,7 @@ type Runner struct {
 	wgwildcardworker    *sync.WaitGroup
 	workerchan          chan string
 	outputchan          chan string
-	wildcardworkerchan  chan string
+	wildcardworkerchan  chan wildcardTask
 	wildcards           *mapsutil.SyncLockMap[string, struct{}]
 	wildcardscache      map[string][]string
 	wildcardscachemutex sync.Mutex
@@ -50,6 +57,10 @@ type Runner struct {
 	aurora              aurora.Aurora
 }
 
+func normalizeHostname(host string) string {
+	return strings.TrimSuffix(strings.TrimSpace(host), ".")
+}
+
 func New(options *Options) (*Runner, error) {
 	retryabledns.CheckInternalIPs = true
 
@@ -115,7 +126,7 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	// If no option is specified or wildcard filter has been requested use query type A
-	if len(questionTypes) == 0 || options.WildcardDomain != "" {
+	if len(questionTypes) == 0 || options.WildcardDomain != "" || options.AutoWildcard {
 		options.A = true
 		questionTypes = append(questionTypes, dns.TypeA)
 	}
@@ -156,7 +167,7 @@ func New(options *Options) (*Runner, error) {
 		wgresolveworkers:   &sync.WaitGroup{},
 		wgwildcardworker:   &sync.WaitGroup{},
 		workerchan:         make(chan string),
-		wildcardworkerchan: make(chan string),
+		wildcardworkerchan: make(chan wildcardTask),
 		wildcards:          mapsutil.NewSyncLockMap[string, struct{}](),
 		wildcardscache:     make(map[string][]string),
 		limiter:            limiter,
@@ -437,6 +448,10 @@ func (r *Runner) SaveResumeConfig() error {
 
 func (r *Runner) Run() error {
 	if r.options.Stream {
+		if r.options.WildcardDomain != "" || r.options.AutoWildcard {
+			gologger.Warning().Msgf("Wildcard filtering enabled in stream mode: falling back to buffered execution")
+			return r.run()
+		}
 		return r.runStream()
 	}
 
@@ -467,7 +482,7 @@ func (r *Runner) run() error {
 	close(r.outputchan)
 	r.wgoutputworker.Wait()
 
-	if r.options.WildcardDomain != "" {
+	if r.options.WildcardDomain != "" || r.options.AutoWildcard {
 		gologger.Print().Msgf("Starting to filter wildcard subdomains\n")
 		ipDomain := make(map[string]map[string]struct{})
 		listIPs := []string{}
@@ -503,13 +518,22 @@ func (r *Runner) run() error {
 		}
 
 		seen := make(map[string]struct{})
+		hostToWildcardDomain := make(map[string]string)
 		for _, a := range listIPs {
 			hosts := ipDomain[a]
 			if len(hosts) >= r.options.WildcardThreshold {
 				for host := range hosts {
+					wildcardDomain, ok := r.getWildcardDomainForHost(host)
+					if !ok {
+						continue
+					}
+					hostToWildcardDomain[host] = wildcardDomain
+					if r.isWildcardApexHost(host, wildcardDomain) {
+						continue
+					}
 					if _, ok := seen[host]; !ok {
 						seen[host] = struct{}{}
-						r.wildcardworkerchan <- host
+						r.wildcardworkerchan <- wildcardTask{host: host, domain: wildcardDomain}
 					}
 				}
 			}
@@ -524,7 +548,8 @@ func (r *Runner) run() error {
 		numRemovedSubdomains := 0
 		for _, A := range listIPs {
 			for host := range ipDomain[A] {
-				if host == r.options.WildcardDomain {
+				wildcardDomain, hasDomain := hostToWildcardDomain[host]
+				if hasDomain && r.isWildcardApexHost(host, wildcardDomain) {
 					if _, ok := seen[host]; !ok {
 						seen[host] = struct{}{}
 						_ = r.lookupAndOutput(host)
@@ -551,6 +576,38 @@ func (r *Runner) run() error {
 	return nil
 }
 
+// getWildcardDomainForHost resolves the wildcard root domain for a host.
+//
+// Resolution order:
+//   - explicit options.WildcardDomain
+//   - auto-derived eTLD+1 when options.AutoWildcard is enabled
+func (r *Runner) getWildcardDomainForHost(host string) (string, bool) {
+	if domain := normalizeHostname(r.options.WildcardDomain); domain != "" {
+		return domain, true
+	}
+	if !r.options.AutoWildcard {
+		return "", false
+	}
+
+	h := normalizeHostname(host)
+	switch {
+	case h == "":
+		return "", false
+	case strings.Contains(h, ":"):
+		return "", false
+	}
+
+	domain, err := publicsuffix.EffectiveTLDPlusOne(h)
+	if err != nil || domain == "" {
+		return "", false
+	}
+	return domain, true
+}
+
+func (r *Runner) isWildcardApexHost(host, wildcardDomain string) bool {
+	return normalizeHostname(host) == normalizeHostname(wildcardDomain)
+}
+
 func (r *Runner) lookupAndOutput(host string) error {
 	if r.options.JSON {
 		if data, ok := r.hm.Get(host); ok {
@@ -731,7 +788,7 @@ func (r *Runner) worker() {
 			}
 		}
 		// if wildcard filtering just store the data
-		if r.options.WildcardDomain != "" {
+		if r.options.WildcardDomain != "" || r.options.AutoWildcard {
 			if err := r.storeDNSData(dnsData.DNSData); err != nil {
 				gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err)
 			}
@@ -935,13 +992,13 @@ func (r *Runner) wildcardWorker() {
 	defer r.wgwildcardworker.Done()
 
 	for {
-		host, more := <-r.wildcardworkerchan
+		task, more := <-r.wildcardworkerchan
 		if !more {
 			break
 		}
-		if r.IsWildcard(host) {
+		if r.IsWildcard(task.host, task.domain) {
 			// mark this host as a wildcard subdomain
-			_ = r.wildcards.Set(host, struct{}{})
+			_ = r.wildcards.Set(task.host, struct{}{})
 		}
 	}
 }

internal/runner/runner_test.go

@@ -141,6 +141,62 @@ func TestRunner_fileInput_prepareInput(t *testing.T) {
 	require.ElementsMatch(t, expected, got, "could not match expected output")
 }
 
+func TestRunner_getWildcardDomainForHost(t *testing.T) {
+	t.Run("uses explicit wildcard domain when provided", func(t *testing.T) {
+		r := Runner{options: &Options{WildcardDomain: "example.com"}}
+		domain, ok := r.getWildcardDomainForHost("api.foo.bar")
+		require.True(t, ok)
+		require.Equal(t, "example.com", domain)
+	})
+
+	t.Run("derives effective tld+1 when auto wildcard enabled", func(t *testing.T) {
+		r := Runner{options: &Options{AutoWildcard: true}}
+		domain, ok := r.getWildcardDomainForHost("api.dev.example.co.uk")
+		require.True(t, ok)
+		require.Equal(t, "example.co.uk", domain)
+	})
+
+	t.Run("returns false for ip input", func(t *testing.T) {
+		r := Runner{options: &Options{AutoWildcard: true}}
+		_, ok := r.getWildcardDomainForHost("1.1.1.1")
+		require.False(t, ok)
+	})
+
+	t.Run("returns false when feature is disabled", func(t *testing.T) {
+		r := Runner{options: &Options{}}
+		_, ok := r.getWildcardDomainForHost("api.example.com")
+		require.False(t, ok)
+	})
+
+	t.Run("trims whitespace before deriving domain", func(t *testing.T) {
+		r := Runner{options: &Options{AutoWildcard: true}}
+		domain, ok := r.getWildcardDomainForHost("  www.projectdiscovery.io  ")
+		require.True(t, ok)
+		require.Equal(t, "projectdiscovery.io", domain)
+	})
+
+	t.Run("trims trailing dot before deriving domain", func(t *testing.T) {
+		r := Runner{options: &Options{AutoWildcard: true}}
+		domain, ok := r.getWildcardDomainForHost("api.projectdiscovery.io.")
+		require.True(t, ok)
+		require.Equal(t, "projectdiscovery.io", domain)
+	})
+
+	t.Run("returns false for host with port", func(t *testing.T) {
+		r := Runner{options: &Options{AutoWildcard: true}}
+		_, ok := r.getWildcardDomainForHost("api.example.com:443")
+		require.False(t, ok)
+	})
+}
+
+func TestRunner_isWildcardApexHost(t *testing.T) {
+	r := Runner{}
+
+	require.True(t, r.isWildcardApexHost("example.com", "example.com"))
+	require.True(t, r.isWildcardApexHost("example.com.", "example.com"))
+	require.False(t, r.isWildcardApexHost("api.example.com", "example.com"))
+}
+
 func TestRunner_InputWorkerStream(t *testing.T) {
 	options := &Options{
 		Hosts: "tests/stream_input.txt",

internal/runner/wildcard.go

@@ -6,8 +6,8 @@ import (
 	"github.com/rs/xid"
 )
 
-// IsWildcard checks if a host is wildcard
-func (r *Runner) IsWildcard(host string) bool {
+// IsWildcard checks if a host is wildcard for a specific root domain
+func (r *Runner) IsWildcard(host, wildcardDomain string) bool {
 	orig := make(map[string]struct{})
 	wildcards := make(map[string]struct{})
 
@@ -19,19 +19,19 @@ func (r *Runner) IsWildcard(host string) bool {
 		orig[A] = struct{}{}
 	}
 
-	subdomainPart := strings.TrimSuffix(host, "."+r.options.WildcardDomain)
+	subdomainPart := strings.TrimSuffix(host, "."+wildcardDomain)
 	subdomainTokens := strings.Split(subdomainPart, ".")
 
 	// Build an array by preallocating a slice of a length
 	// and create the wildcard generation prefix.
 	// We use a rand prefix at the beginning like %rand%.domain.tld
 	// A permutation is generated for each level of the subdomain.
 	var hosts []string
-	hosts = append(hosts, r.options.WildcardDomain)
+	hosts = append(hosts, wildcardDomain)
 
 	if len(subdomainTokens) > 0 {
 		for i := 1; i < len(subdomainTokens); i++ {
-			newhost := strings.Join(subdomainTokens[i:], ".") + "." + r.options.WildcardDomain
+			newhost := strings.Join(subdomainTokens[i:], ".") + "." + wildcardDomain
 			hosts = append(hosts, newhost)
 		}
 	}