When using Nuclei to send HTTP requests to multiple hosts, only one of them can be sent successfully.

[jsfengsdu]

When I use my own method to call the nuclei core method for scanning, it can send HTTP requests normally when scanning vulnerabilities for only one host. However, if I use a for loop to send HTTP requests to multiple hosts, it can only successfully send HTTP requests to the first host. For other machines, only TCP requests are sent. Could you please help me identify the problem? I am using Nuclei version 2.9.13. Below is my code.

`
func main(){
items := map[string]int{"1.7.20.21":9152,"1.7.7.76":12345}
for ip, port := range items{
YamlScan(vulcommon.HostInfo{
Host: ip,
Ports: port,
}, "tmp", apache-dolphinscheduler-default-login.yaml)
}
}

// scan vul use yaml file
func YamlScan(hostinfo vulcommon.HostInfo, templatePath, templateFileName string) error {
//step1: parse and compile yaml
//parse yaml file to template struct
template, err := Yaml2template(templatePath, templateFileName)
if err != nil {
return err
}

//TODO
//set target
target := ""
if len(template.RequestsHTTP) > 0 {
	if hostinfo.Ports == "443" ||
		hostinfo.Ports == "8443" ||
		hostinfo.Ports == "1443" ||
		hostinfo.Ports == "18443" ||
		hostinfo.Ports == "28443" {
		target = fmt.Sprintf("https://%s:%s", hostinfo.Host, hostinfo.Ports)
	} else {
		target = fmt.Sprintf("http://%s:%s", hostinfo.Host, hostinfo.Ports)
	}
} else if len(template.RequestsDNS) > 0 {
	target = hostinfo.Host
} else {
	target = fmt.Sprintf("%s:%s", hostinfo.Host, hostinfo.Ports)
}

typeOptions := &types.Options{
	Targets:                    []string{target},
	StoreResponseDir:           "output",
	Silent:                     true,
	NoColor:                    true,
	JSONRequests:               true,
	FollowRedirects:            true,
	DisableClustering:          true,
	OutputLimit:                100,
	MetricsPort:                9092,
	StatsInterval:              5,
	PageTimeout:                20,
	InputReadTimeout:           time.Duration(3 * time.Minute),
	MaxHostError:               30,
	Retries:                    1,
	Timeout:                    10,
	HeadlessTemplateThreads:    10,
	HeadlessBulkSize:           25,
	TemplateThreads:            25,
	RateLimitMinute:            0,
	RateLimit:                  150,
	UncoverRateLimit:           60,
	UncoverLimit:               100,
	InteractionsCoolDownPeriod: 5,
	InteractionsPollDuration:   5,
	InteractionsEviction:       60,
	InteractionsCacheSize:      5000,
	ResponseSaveSize:           1 * 1024 * 1024,
	ResponseReadSize:           10 * 1024 * 1024,
	MaxRedirects:               3,
	IPVersion:                  []string{"4"},
	BulkSize:                   25,
	NoInteractsh:               true,
	ProjectPath:                os.TempDir(),
	ScanStrategy:               "host-spray",
	GitLabServerURL:            "https://gitlab.com",
	CloudURL:                   "https://cloud-dev.nuclei.sh",
}

//init http client etc...
err = initProtocol(typeOptions)
if err != nil {
	return err
}

outputWriter, err := output.NewStandardWriter(typeOptions)
if err != nil {
	return err
}

hmapInput, err := hybrid.New(&hybrid.Options{
	Options: typeOptions,
})

if err != nil {
	return err
}

progressTmp, _ := progress.NewStatsTicker(-1, false, false, false, false, 0)
options := protocols.ExecutorOptions{
	Options:          typeOptions,
	TemplateID:       template.ID,
	TemplateInfo:     template.Info,
	StopAtFirstMatch: template.StopAtFirstMatch,
	Variables:        template.Variables,
	Constants:        template.Constants,
	TemplatePath:     filepath.Join(templatePath, templateFileName),
	Output:           outputWriter,
	Catalog:          disk.NewCatalog(templatePath),
	RateLimiter:      ratelimit.NewUnlimited(context.Background()),
	Colorizer:        aurora.NewAurora(false),
	InputHelper:      input.NewHelper(),
	Progress:         progressTmp,
	ResumeCfg:        types.NewResumeCfg(),
	ExcludeMatchers:  excludematchers.New(nil),
}

if err != nil {
	return err
}

//compile template
template, err = parse(templatePath, templateFileName, options, template)
if err != nil {
	return err
}

//step2:scan vul
//begin use template to scan
results := &atomic.Bool{}
wp := sizedwaitgroup.New(1)

//scan target
hmapInput.Scan(func(value *contextargs.MetaInput) bool {
	wp.Add()
	go func(targetval *contextargs.MetaInput) {
		defer wp.Done()
		executeTemplatesOnTarget(hostinfo, template, targetval, results)
	}(value)
	return true
})
wp.Wait()

//step3: close connection ...
options.Output.Close()
hmapInput.Close()
protocolstate.Dialer.Close()
options.RateLimiter.Stop()
options.InputHelper.Close()
options.Progress.Stop()

return nil

}

// parse yaml file to template struct & compile
func parse(templatePath, templateFileName string, options protocols.ExecutorOptions, template *templates.Template) (*templates.Template, error) {
if template.Requests() == 0 {
return nil, fmt.Errorf("no requests defined for %s", templateFileName)
}

if err := compileProtocolRequests(options, template); err != nil {
	return nil, err
}

if template.Executer != nil {
	if err := template.Executer.Compile(); err != nil {
		return nil, err
	}
	template.TotalRequests = template.Executer.Requests()
} else {
	return nil, fmt.Errorf("cant set executer for %s", templateFileName)
}

return template, nil

}

// get template struct from yaml
func Yaml2template(templatePath, templateFileName string) (*templates.Template, error) {
catalog := disk.NewCatalog(templatePath)
template := &templates.Template{}

reader, err := catalog.OpenFile(templateFileName)
if err != nil {
	return nil, err
}
data, err := io.ReadAll(reader)

if err != nil {
	return nil, err
}

data = expandPreprocessors(data)

if err := yaml.Unmarshal(data, template); err != nil {
	return nil, err
}

return template, nil

}

// when get vul result from yaml, write result to mongodb
func callback(hostinfo vulcommon.HostInfo, result *output.ResultEvent) {
vulcommon.INTLogSuccess(hostinfo, fmt.Sprintf(" %s \n，IP:%s port：%s \ninfo：%s", result.Info.Name, hostinfo.Host, hostinfo.Ports, result.Matched), map[string]string{
"name": result.Info.Name,
"info": result.Info.Description,
"level": result.Info.SeverityHolder.Severity.String(),
"author": result.Info.Authors.String(),
"url": result.Info.Reference.String(),
"keyword": result.Info.Tags.String(),
})
}

// compileProtocolRequests compiles all the protocol requests for the template
func compileProtocolRequests(options protocols.ExecutorOptions, template *templates.Template) error {
templateRequests := template.Requests()

if templateRequests == 0 {
	return fmt.Errorf("no requests defined for %s", template.ID)
}

var requests []protocols.Request

if len(template.RequestsDNS) > 0 {
	requests = append(requests, convertRequestToProtocolsRequest(template.RequestsDNS)...)
}
if len(template.RequestsFile) > 0 {
	requests = append(requests, convertRequestToProtocolsRequest(template.RequestsFile)...)
}
if len(template.RequestsNetwork) > 0 {
	requests = append(requests, convertRequestToProtocolsRequest(template.RequestsNetwork)...)
}
if len(template.RequestsHTTP) > 0 {
	requests = append(requests, convertRequestToProtocolsRequest(template.RequestsHTTP)...)
}
if len(template.RequestsHeadless) > 0 {
	return fmt.Errorf("unsupport protocol for %s", template.Info.Name)
}
if len(template.RequestsSSL) > 0 {
	requests = append(requests, convertRequestToProtocolsRequest(template.RequestsSSL)...)
}
if len(template.RequestsWebsocket) > 0 {
	requests = append(requests, convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
}
if len(template.RequestsWHOIS) > 0 {
	requests = append(requests, convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
}

template.Executer = executer.NewExecuter(requests, &options)

return nil

}

func initProtocol(options *types.Options) error {
uarand.Default = uarand.NewWithCustomList(userAgents)

if err := protocolstate.Init(options); err != nil {
	return err
}
if err := dnsclientpool.Init(options); err != nil {
	return err
}
if err := httpclientpool.Init(options); err != nil {
	return err
}
if err := signerpool.Init(options); err != nil {
	return err
}
if err := networkclientpool.Init(options); err != nil {
	return err
}
if err := rdapclientpool.Init(options); err != nil {
	return err
}
return nil

}

// convert request
func convertRequestToProtocolsRequest(requests interface{}) []protocols.Request {
switch reflect.TypeOf(requests).Kind() {
case reflect.Slice:
s := reflect.ValueOf(requests)

	requestSlice := make([]protocols.Request, s.Len())
	for i := 0; i < s.Len(); i++ {
		value := s.Index(i)
		valueInterface := value.Interface()
		requestSlice[i] = valueInterface.(protocols.Request)
	}
	return requestSlice
}
return nil

}

// expandPreprocessors expands the pre-processors if any for a template data.
func expandPreprocessors(data []byte) []byte {
preprocessorRegex := regexp.MustCompile({{([a-z0-9_]+)}})
foundMap := make(map[string]struct{})

for _, expression := range preprocessorRegex.FindAllStringSubmatch(string(data), -1) {
	if len(expression) != 2 {
		continue
	}
	value := expression[1]
	if strings.Contains(value, "(") || strings.Contains(value, ")") {
		continue
	}

	if _, ok := foundMap[value]; ok {
		continue
	}
	foundMap[value] = struct{}{}
	if strings.EqualFold(value, "randstr") || strings.HasPrefix(value, "randstr_") {
		data = bytes.ReplaceAll(data, []byte(expression[0]), []byte(ksuid.New().String()))
	}
}
return data

}

// execute given template on given single target
func executeTemplatesOnTarget(hostinfo vulcommon.HostInfo, template *templates.Template, target *contextargs.MetaInput, results *atomic.Bool) {
wp := core.NewWorkPool(core.WorkPoolConfig{
InputConcurrency: 1,
TypeConcurrency: 1,
HeadlessInputConcurrency: 1,
HeadlessTypeConcurrency: 1,
})

var sg *sizedwaitgroup.SizedWaitGroup
sg = wp.Default
sg.Add()
go func(template *templates.Template, value *contextargs.MetaInput, wg *sizedwaitgroup.SizedWaitGroup) {
	defer wg.Done()
	ctxArgs := contextargs.New()
	ctxArgs.MetaInput = value
	template.Executer.ExecuteWithResults(ctxArgs, func(event *output.InternalWrappedEvent) {
		for _, result := range event.Results {
			callback(hostinfo, result)
		}
	})
}(template, target, sg)

wp.Wait()

}
`

Here are the templates involved:

`
id: apache-dolphinscheduler-default-login

info:
name: Apache Dolphinscheduler default login
author: XIukD
severity: high
description: Apache Dolphinscheduler default login
reference:
- https://dolphinscheduler.apache.org/zh-cn/docs/3.2.1/guide/start/quick-start
tags: dolphinscheduler
metadata:
max-request: 4

http:

• raw:

  • |
    POST /dolphinscheduler/login HTTP/1.1
    Host: {{Hostname}}
    Accept: application/json, text/plain, /
    Content-Type: application/x-www-form-urlencoded

    userName={{username}}&userPassword={{password}}

  payloads:
  username:
  - admin
  password:
  - dolphinscheduler123
  - admin
  - dolphinscheduler123456
  - dolphinscheduler
  attack: clusterbomb
  matchers:

  • type: word
    part: body
    words:
    - "sessionId"
    - "PASSWORD"
    condition: and
    `

[dogancanbakir]

This discussion closed automatically due to inactivity. Feel free to reopen or start new if still relevant.