From a8d1393e961d5db72c45efa840a841854e96a2c1 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 3 Apr 2024 17:50:57 +0200
Subject: [PATCH 01/16] init- using resizable components

---
 examples/advanced/advanced.go                 |  7 +++-
 go.mod                                        |  2 +
 go.sum                                        |  3 ++
 internal/runner/inputs.go                     | 13 +++---
 pkg/core/execute_options.go                   |  7 ++--
 pkg/core/executors.go                         | 14 +++----
 pkg/core/workflow_execute.go                  |  9 ++--
 pkg/core/workpool.go                          | 26 +++++-------
 pkg/js/compiler/non-pool.go                   |  6 +--
 pkg/js/compiler/pool.go                       |  6 +--
 .../common/automaticscan/automaticscan.go     |  9 ++--
 pkg/protocols/dns/request.go                  |  7 +++-
 pkg/protocols/file/request.go                 |  9 ++--
 pkg/protocols/headless/engine/page_actions.go | 28 ++++++-------
 pkg/protocols/http/httputils/spm.go           |  9 ++--
 pkg/protocols/javascript/js.go                |  5 ++-
 .../network/networkclientpool/clientpool.go   |  2 +-
 pkg/protocols/network/request.go              |  7 +++-
 pkg/protocols/offlinehttp/request.go          | 11 +++--
 pkg/tmplexec/flow/vm.go                       | 41 +++++++++----------
 20 files changed, 121 insertions(+), 100 deletions(-)

diff --git a/examples/advanced/advanced.go b/examples/advanced/advanced.go
index 5ce579b3d6..110160f9a1 100644
--- a/examples/advanced/advanced.go
+++ b/examples/advanced/advanced.go
@@ -2,7 +2,7 @@ package main
 
 import (
 	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 func main() {
@@ -12,7 +12,10 @@ func main() {
 		panic(err)
 	}
 	// setup sizedWaitgroup to handle concurrency
-	sg := sizedwaitgroup.New(10)
+	sg, err := syncutil.New(syncutil.WithSize(10))
+	if err != nil {
+		panic(err)
+	}
 
 	// scan 1 = run dns templates on scanme.sh
 	sg.Add()
diff --git a/go.mod b/go.mod
index 7eeaf57386..6a0e8bfccf 100644
--- a/go.mod
+++ b/go.mod
@@ -142,6 +142,8 @@ require (
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/eapache/channels v1.1.0 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
diff --git a/go.sum b/go.sum
index cefd9ae445..83551eb1c2 100644
--- a/go.sum
+++ b/go.sum
@@ -296,8 +296,11 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
+github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go
index 8dc27a7a9e..75a869912b 100644
--- a/internal/runner/inputs.go
+++ b/internal/runner/inputs.go
@@ -12,7 +12,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	stringsutil "github.com/projectdiscovery/utils/strings"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 const probeBulkSize = 50
@@ -45,8 +45,11 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 
 	// Probe the non-standard URLs and store them in cache
-	swg := sizedwaitgroup.New(bulkSize)
-	count := int32(0)
+	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create adaptive group")
+	}
+	var count atomic.Int32
 	r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
 		if stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
 			return true
@@ -57,7 +60,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 			defer swg.Done()
 
 			if result := utils.ProbeURL(input.Input, httpxClient); result != "" {
-				atomic.AddInt32(&count, 1)
+				count.Add(1)
 				_ = hm.Set(input.Input, []byte(result))
 			}
 		}(value)
@@ -65,6 +68,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	})
 	swg.Wait()
 
-	gologger.Info().Msgf("Found %d URL from httpx", atomic.LoadInt32(&count))
+	gologger.Info().Msgf("Found %d URL from httpx", count.Load())
 	return hm, nil
 }
diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go
index fd1fadae3d..580b8b0a9d 100644
--- a/pkg/core/execute_options.go
+++ b/pkg/core/execute_options.go
@@ -4,8 +4,6 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/remeh/sizedwaitgroup"
-
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -14,6 +12,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // Execute takes a list of templates/workflows that have been compiled
@@ -111,7 +110,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 	for _, template := range templatesList {
 		templateType := template.Type()
 
-		var wg *sizedwaitgroup.SizedWaitGroup
+		var wg *syncutil.AdaptiveWaitGroup
 		if templateType == types.HeadlessProtocol {
 			wg = wp.Headless
 		} else {
@@ -134,7 +133,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 // executeHostSpray executes scan using host spray strategy where templates are iterated over each target
 func (e *Engine) executeHostSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
 	results := &atomic.Bool{}
-	wp := sizedwaitgroup.New(e.options.BulkSize + e.options.HeadlessBulkSize)
+	wp, _ := syncutil.New(syncutil.WithSize(e.options.BulkSize + e.options.HeadlessBulkSize))
 
 	target.Iterate(func(value *contextargs.MetaInput) bool {
 		wp.Add()
diff --git a/pkg/core/executors.go b/pkg/core/executors.go
index b491bd8e0b..ace7acb208 100644
--- a/pkg/core/executors.go
+++ b/pkg/core/executors.go
@@ -11,7 +11,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	generalTypes "github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // Executors are low level executors that deals with template execution on a target
@@ -104,9 +104,9 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 			return true
 		}
 
-		wg.WaitGroup.Add()
+		wg.Add()
 		go func(index uint32, skip bool, value *contextargs.MetaInput) {
-			defer wg.WaitGroup.Done()
+			defer wg.Done()
 			defer cleanupInFlight(index)
 			if skip {
 				return
@@ -140,7 +140,7 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 		index++
 		return true
 	})
-	wg.WaitGroup.Wait()
+	wg.Wait()
 
 	// on completion marks the template as completed
 	currentInfo.Lock()
@@ -158,14 +158,14 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	wp := e.GetWorkPool()
 
 	for _, tpl := range alltemplates {
-		var sg *sizedwaitgroup.SizedWaitGroup
+		var sg *syncutil.AdaptiveWaitGroup
 		if tpl.Type() == types.HeadlessProtocol {
 			sg = wp.Headless
 		} else {
 			sg = wp.Default
 		}
 		sg.Add()
-		go func(template *templates.Template, value *contextargs.MetaInput, wg *sizedwaitgroup.SizedWaitGroup) {
+		go func(template *templates.Template, value *contextargs.MetaInput, wg *syncutil.AdaptiveWaitGroup) {
 			defer wg.Done()
 
 			var match bool
@@ -213,7 +213,7 @@ func (e *ChildExecuter) Close() *atomic.Bool {
 func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
 	templateType := template.Type()
 
-	var wg *sizedwaitgroup.SizedWaitGroup
+	var wg *syncutil.AdaptiveWaitGroup
 	if templateType == types.HeadlessProtocol {
 		wg = e.e.workPool.Headless
 	} else {
diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go
index cb877cc60e..19d6f0d69d 100644
--- a/pkg/core/workflow_execute.go
+++ b/pkg/core/workflow_execute.go
@@ -5,13 +5,12 @@ import (
 	"net/http/cookiejar"
 	"sync/atomic"
 
-	"github.com/remeh/sizedwaitgroup"
-
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/workflows"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n"
@@ -32,7 +31,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 	if templateThreads == 1 {
 		templateThreads++
 	}
-	swg := sizedwaitgroup.New(templateThreads)
+	swg, _ := syncutil.New(syncutil.WithSize(templateThreads))
 
 	for _, template := range w.Workflows {
 		swg.Add()
@@ -40,7 +39,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 		func(template *workflows.WorkflowTemplate) {
 			defer swg.Done()
 
-			if err := e.runWorkflowStep(template, ctx, results, &swg, w); err != nil {
+			if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil {
 				gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
 			}
 		}(template)
@@ -51,7 +50,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 
 // runWorkflowStep runs a workflow step for the workflow. It executes the workflow
 // in a recursive manner running all subtemplates and matchers.
-func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error {
+func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, w *workflows.Workflow) error {
 	var firstMatched bool
 	var err error
 	var mainErr error
diff --git a/pkg/core/workpool.go b/pkg/core/workpool.go
index 0711759582..cd17a0b367 100644
--- a/pkg/core/workpool.go
+++ b/pkg/core/workpool.go
@@ -1,9 +1,8 @@
 package core
 
 import (
-	"github.com/remeh/sizedwaitgroup"
-
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // WorkPool implements an execution pool for executing different
@@ -12,8 +11,8 @@ import (
 // It also allows Configuration of such requirements. This is used
 // for per-module like separate headless concurrency etc.
 type WorkPool struct {
-	Headless *sizedwaitgroup.SizedWaitGroup
-	Default  *sizedwaitgroup.SizedWaitGroup
+	Headless *syncutil.AdaptiveWaitGroup
+	Default  *syncutil.AdaptiveWaitGroup
 	config   WorkPoolConfig
 }
 
@@ -31,13 +30,13 @@ type WorkPoolConfig struct {
 
 // NewWorkPool returns a new WorkPool instance
 func NewWorkPool(config WorkPoolConfig) *WorkPool {
-	headlessWg := sizedwaitgroup.New(config.HeadlessTypeConcurrency)
-	defaultWg := sizedwaitgroup.New(config.TypeConcurrency)
+	headlessWg, _ := syncutil.New(syncutil.WithSize(config.HeadlessTypeConcurrency))
+	defaultWg, _ := syncutil.New(syncutil.WithSize(config.TypeConcurrency))
 
 	return &WorkPool{
 		config:   config,
-		Headless: &headlessWg,
-		Default:  &defaultWg,
+		Headless: headlessWg,
+		Default:  defaultWg,
 	}
 }
 
@@ -47,19 +46,14 @@ func (w *WorkPool) Wait() {
 	w.Headless.Wait()
 }
 
-// InputWorkPool is a work pool per-input
-type InputWorkPool struct {
-	WaitGroup *sizedwaitgroup.SizedWaitGroup
-}
-
 // InputPool returns a work pool for an input type
-func (w *WorkPool) InputPool(templateType types.ProtocolType) *InputWorkPool {
+func (w *WorkPool) InputPool(templateType types.ProtocolType) *syncutil.AdaptiveWaitGroup {
 	var count int
 	if templateType == types.HeadlessProtocol {
 		count = w.config.HeadlessInputConcurrency
 	} else {
 		count = w.config.InputConcurrency
 	}
-	swg := sizedwaitgroup.New(count)
-	return &InputWorkPool{WaitGroup: &swg}
+	swg, _ := syncutil.New(syncutil.WithSize(count))
+	return swg
 }
diff --git a/pkg/js/compiler/non-pool.go b/pkg/js/compiler/non-pool.go
index 8057c49608..218b89b82a 100644
--- a/pkg/js/compiler/non-pool.go
+++ b/pkg/js/compiler/non-pool.go
@@ -4,13 +4,13 @@ import (
 	"sync"
 
 	"github.com/dop251/goja"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var (
-	ephemeraljsc    = sizedwaitgroup.New(NonPoolingVMConcurrency)
+	ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
 	lazyFixedSgInit = sync.OnceFunc(func() {
-		ephemeraljsc = sizedwaitgroup.New(NonPoolingVMConcurrency)
+		ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
 	})
 )
 
diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go
index 6dba600f70..fc0b61639b 100644
--- a/pkg/js/compiler/pool.go
+++ b/pkg/js/compiler/pool.go
@@ -36,7 +36,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	stringsutil "github.com/projectdiscovery/utils/strings"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 const (
@@ -51,9 +51,9 @@ var (
 		// autoregister console node module with default printer it uses gologger backend
 		require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter()))
 	})
-	pooljsc    sizedwaitgroup.SizedWaitGroup
+	pooljsc    *syncutil.AdaptiveWaitGroup
 	lazySgInit = sync.OnceFunc(func() {
-		pooljsc = sizedwaitgroup.New(PoolingJsVmConcurrency)
+		pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency))
 	})
 )
 
diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go
index 8193947190..7119377b01 100644
--- a/pkg/protocols/common/automaticscan/automaticscan.go
+++ b/pkg/protocols/common/automaticscan/automaticscan.go
@@ -30,8 +30,8 @@ import (
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	syncutil "github.com/projectdiscovery/utils/sync"
 	wappalyzer "github.com/projectdiscovery/wappalyzergo"
-	"github.com/remeh/sizedwaitgroup"
 	"gopkg.in/yaml.v2"
 )
 
@@ -128,7 +128,10 @@ func (s *Service) Close() bool {
 func (s *Service) Execute() error {
 	gologger.Info().Msgf("Executing Automatic scan on %d target[s]", s.target.Count())
 	// setup host concurrency
-	sg := sizedwaitgroup.New(s.opts.Options.BulkSize)
+	sg, err := syncutil.New(syncutil.WithSize(s.opts.Options.BulkSize))
+	if err != nil {
+		return err
+	}
 	s.target.Iterate(func(value *contextargs.MetaInput) bool {
 		sg.Add()
 		go func(input *contextargs.MetaInput) {
@@ -246,7 +249,7 @@ func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) (
 	// execute tech detection templates on target
 	tags := map[string]struct{}{}
 	m := &sync.Mutex{}
-	sg := sizedwaitgroup.New(s.opts.Options.TemplateThreads)
+	sg, _ := syncutil.New(syncutil.WithSize(s.opts.Options.TemplateThreads))
 	counter := atomic.Uint32{}
 
 	for _, t := range s.techTemplates {
diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index 280e8161fe..ba9dd66621 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -9,7 +9,6 @@ import (
 
 	"github.com/miekg/dns"
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/multierr"
 	"golang.org/x/exp/maps"
 
@@ -27,6 +26,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/projectdiscovery/retryabledns"
 	iputil "github.com/projectdiscovery/utils/ip"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -64,7 +64,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
-		swg := sizedwaitgroup.New(request.Threads)
+		swg, err := syncutil.New(syncutil.WithSize(request.Threads))
+		if err != nil {
+			return err
+		}
 		var multiErr error
 		m := &sync.Mutex{}
 
diff --git a/pkg/protocols/file/request.go b/pkg/protocols/file/request.go
index eb73544f9a..f13f08d1ab 100644
--- a/pkg/protocols/file/request.go
+++ b/pkg/protocols/file/request.go
@@ -11,7 +11,6 @@ import (
 	"github.com/docker/go-units"
 	"github.com/mholt/archiver"
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
@@ -24,6 +23,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	sliceutil "github.com/projectdiscovery/utils/slice"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -47,8 +47,11 @@ var errEmptyResult = errors.New("Empty result")
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
-	err := request.getInputPaths(input.MetaInput.Input, func(filePath string) {
+	wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
+	if err != nil {
+		return err
+	}
+	err = request.getInputPaths(input.MetaInput.Input, func(filePath string) {
 		wg.Add()
 		func(filePath string) {
 			defer wg.Done()
diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go
index 7338db44ab..348cab0aec 100644
--- a/pkg/protocols/headless/engine/page_actions.go
+++ b/pkg/protocols/headless/engine/page_actions.go
@@ -214,7 +214,7 @@ func geTimeParameter(p *Page, act *Action, parameterName string, defaultValue ti
 }
 
 // ActionAddHeader executes a AddHeader action.
-func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ActionAddHeader(act *Action, out map[string]string) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -225,7 +225,7 @@ func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review
 }
 
 // ActionSetHeader executes a SetHeader action.
-func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ActionSetHeader(act *Action, out map[string]string) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -236,7 +236,7 @@ func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review
 }
 
 // ActionDeleteHeader executes a DeleteHeader action.
-func (p *Page) ActionDeleteHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -343,7 +343,7 @@ func (p *Page) RunScript(action *Action, out map[string]string) error {
 }
 
 // ClickElement executes click actions for an element.
-func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ClickElement(act *Action, out map[string]string) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -358,12 +358,12 @@ func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unu
 }
 
 // KeyboardAction executes a keyboard action on the page.
-func (p *Page) KeyboardAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) KeyboardAction(act *Action, out map[string]string) error {
 	return p.page.Keyboard.Type([]input.Key(p.getActionArgWithDefaultValues(act, "keys"))...)
 }
 
 // RightClickElement executes right click actions for an element.
-func (p *Page) RightClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) RightClickElement(act *Action, out map[string]string) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -441,7 +441,7 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error {
 }
 
 // InputElement executes input element actions for an element.
-func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) InputElement(act *Action, out map[string]string) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -460,7 +460,7 @@ func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unu
 }
 
 // TimeInputElement executes time input on an element
-func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) TimeInputElement(act *Action, out map[string]string) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -483,7 +483,7 @@ func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review
 }
 
 // SelectInputElement executes select input statement action on a element
-func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) SelectInputElement(act *Action, out map[string]string) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -508,7 +508,7 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO revi
 }
 
 // WaitLoad waits for the page to load
-func (p *Page) WaitLoad(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) WaitLoad(act *Action, out map[string]string) error {
 	p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)()
 
 	// Wait for the window.onload event and also wait for the network requests
@@ -538,7 +538,7 @@ func (p *Page) GetResource(act *Action, out map[string]string) error {
 }
 
 // FilesInput acts with a file input element on page
-func (p *Page) FilesInput(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) FilesInput(act *Action, out map[string]string) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -589,7 +589,7 @@ func (p *Page) ExtractElement(act *Action, out map[string]string) error {
 }
 
 // WaitEvent waits for an event to happen on the page.
-func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused parameter*/) (func() error, error) {
+func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, error) {
 	event := p.getActionArgWithDefaultValues(act, "event")
 	if event == "" {
 		return nil, errors.New("event not recognized")
@@ -661,14 +661,14 @@ func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) {
 }
 
 // DebugAction enables debug action on a page.
-func (p *Page) DebugAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) DebugAction(act *Action, out map[string]string) error {
 	p.instance.browser.engine.SlowMotion(5 * time.Second)
 	p.instance.browser.engine.Trace(true)
 	return nil
 }
 
 // SleepAction sleeps on the page for a specified duration
-func (p *Page) SleepAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) SleepAction(act *Action, out map[string]string) error {
 	seconds := act.Data["duration"]
 	if seconds == "" {
 		seconds = "5"
diff --git a/pkg/protocols/http/httputils/spm.go b/pkg/protocols/http/httputils/spm.go
index ccaa9a85c9..bca6c2ee59 100644
--- a/pkg/protocols/http/httputils/spm.go
+++ b/pkg/protocols/http/httputils/spm.go
@@ -4,7 +4,7 @@ import (
 	"context"
 	"sync"
 
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // WorkPoolType is the type of work pool to use
@@ -26,7 +26,7 @@ type StopAtFirstMatchHandler[T any] struct {
 
 	// work pool and its type
 	poolType WorkPoolType
-	sgPool   sizedwaitgroup.SizedWaitGroup
+	sgPool   *syncutil.AdaptiveWaitGroup
 	wgPool   *sync.WaitGroup
 
 	// internal / unexported
@@ -40,10 +40,13 @@ type StopAtFirstMatchHandler[T any] struct {
 // NewBlockingSPMHandler creates a new stop at first match handler
 func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
+
+	awg, _ := syncutil.New(syncutil.WithSize(size))
+
 	s := &StopAtFirstMatchHandler[T]{
 		ResultChan:  make(chan T, 1),
 		poolType:    Blocking,
-		sgPool:      sizedwaitgroup.New(size),
+		sgPool:      awg,
 		internalWg:  &sync.WaitGroup{},
 		ctx:         ctx1,
 		cancel:      cancel,
diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go
index fc32486937..48a1be485d 100644
--- a/pkg/protocols/javascript/js.go
+++ b/pkg/protocols/javascript/js.go
@@ -32,8 +32,8 @@ import (
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	syncutil "github.com/projectdiscovery/utils/sync"
 	urlutil "github.com/projectdiscovery/utils/url"
-	"github.com/remeh/sizedwaitgroup"
 )
 
 // Request is a request for the javascript protocol
@@ -404,7 +404,8 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 	requestOptions := request.options
 	gotmatches := &atomic.Bool{}
 
-	sg := sizedwaitgroup.New(threads)
+	sg, _ := syncutil.New(syncutil.WithSize(threads))
+
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		for {
diff --git a/pkg/protocols/network/networkclientpool/clientpool.go b/pkg/protocols/network/networkclientpool/clientpool.go
index 1a933413e2..a67cee2967 100644
--- a/pkg/protocols/network/networkclientpool/clientpool.go
+++ b/pkg/protocols/network/networkclientpool/clientpool.go
@@ -11,7 +11,7 @@ var (
 )
 
 // Init initializes the clientpool implementation
-func Init(options *types.Options /*TODO review unused parameter*/) error {
+func Init(options *types.Options) error {
 	// Don't create clients if already created in the past.
 	if normalClient != nil {
 		return nil
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 01991c0459..ef0ff01c7f 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/multierr"
 	"golang.org/x/exp/maps"
 
@@ -34,6 +33,7 @@ import (
 	errorutil "github.com/projectdiscovery/utils/errors"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	"github.com/projectdiscovery/utils/reader"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var (
@@ -178,7 +178,10 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 		iterator := request.generator.NewIterator()
 		var multiErr error
 		m := &sync.Mutex{}
-		swg := sizedwaitgroup.New(request.Threads)
+		swg, err := syncutil.New(syncutil.WithSize(request.Threads))
+		if err != nil {
+			return err
+		}
 
 		for {
 			value, ok := iterator.Value()
diff --git a/pkg/protocols/offlinehttp/request.go b/pkg/protocols/offlinehttp/request.go
index 7c64859e96..4a440c167f 100644
--- a/pkg/protocols/offlinehttp/request.go
+++ b/pkg/protocols/offlinehttp/request.go
@@ -6,7 +6,6 @@ import (
 	"os"
 
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -17,6 +16,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/utils/conversion"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -29,10 +29,13 @@ func (request *Request) Type() templateTypes.ProtocolType {
 }
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
+func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+	wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
+	if err != nil {
+		return err
+	}
 
-	err := request.getInputPaths(input.MetaInput.Input, func(data string) {
+	err = request.getInputPaths(input.MetaInput.Input, func(data string) {
 		wg.Add()
 
 		go func(data string) {
diff --git a/pkg/tmplexec/flow/vm.go b/pkg/tmplexec/flow/vm.go
index 2e22bd8e12..f1f7dbb84e 100644
--- a/pkg/tmplexec/flow/vm.go
+++ b/pkg/tmplexec/flow/vm.go
@@ -1,6 +1,7 @@
 package flow
 
 import (
+	"context"
 	"reflect"
 	"sync"
 
@@ -12,45 +13,43 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/remeh/sizedwaitgroup"
+	"github.com/projectdiscovery/utils/sync/sizedpool"
 )
 
-type jsWaitGroup struct {
-	sync.Once
-	sg sizedwaitgroup.SizedWaitGroup
+var jsOnce sync.Once
+
+// js runtime pool using sync.Pool
+var gojapool = &sync.Pool{
+	New: func() interface{} {
+		runtime := protocolstate.NewJSRuntime()
+		registerBuiltins(runtime)
+		return runtime
+	},
 }
 
-var jsPool = &jsWaitGroup{}
+var sizedgojapool *sizedpool.SizedPool[*goja.Runtime]
 
 // GetJSRuntime returns a new JS runtime from pool
 func GetJSRuntime(opts *types.Options) *goja.Runtime {
-	jsPool.Do(func() {
+	jsOnce.Do(func() {
 		if opts.JsConcurrency < 100 {
 			opts.JsConcurrency = 100
 		}
-		jsPool.sg = sizedwaitgroup.New(opts.JsConcurrency)
+		sizedgojapool, _ = sizedpool.New[*goja.Runtime](
+			sizedpool.WithPool[*goja.Runtime](gojapool),
+			sizedpool.WithSize[*goja.Runtime](int64(opts.JsConcurrency)),
+		)
 	})
-	jsPool.sg.Add()
-	return gojapool.Get().(*goja.Runtime)
+	runtime, _ := sizedgojapool.Get(context.TODO())
+	return runtime
 }
 
 // PutJSRuntime returns a JS runtime to pool
 func PutJSRuntime(runtime *goja.Runtime) {
-	defer jsPool.sg.Done()
-	gojapool.Put(runtime)
-}
-
-// js runtime pool using sync.Pool
-var gojapool = &sync.Pool{
-	New: func() interface{} {
-		runtime := protocolstate.NewJSRuntime()
-		registerBuiltins(runtime)
-		return runtime
-	},
+	sizedgojapool.Put(runtime)
 }
 
 func registerBuiltins(runtime *goja.Runtime) {
-
 	_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
 		Name:        "log",
 		Description: "Logs a given object/message to stdout (only for debugging purposes)",

From 774db61655894d6b5e4f76e98ff09997ceece263 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 3 Apr 2024 18:50:46 +0200
Subject: [PATCH 02/16] lightweight adaptivity on workpool

---
 internal/runner/inputs.go   | 14 ++++++--------
 pkg/core/engine.go          | 13 +++++++++----
 pkg/core/execute_options.go |  4 +++-
 pkg/core/executors.go       |  6 ++++++
 pkg/core/workpool.go        | 25 +++++++++++++++++++++++++
 pkg/js/compiler/pool.go     |  5 +++++
 6 files changed, 54 insertions(+), 13 deletions(-)

diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go
index 75a869912b..60aa03199f 100644
--- a/internal/runner/inputs.go
+++ b/internal/runner/inputs.go
@@ -15,12 +15,11 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
-const probeBulkSize = 50
+var GlobalProbeBulkSize = 50
 
 // initializeTemplatesHTTPInput initializes the http form of input
 // for any loaded http templates if input is in non-standard format.
 func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
-
 	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create temporary input file")
@@ -31,11 +30,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	gologger.Info().Msgf("Running httpx on input host")
 
-	var bulkSize = probeBulkSize
-	if r.options.BulkSize > probeBulkSize {
-		bulkSize = r.options.BulkSize
-	}
-
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
@@ -45,7 +39,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 
 	// Probe the non-standard URLs and store them in cache
-	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
+	swg, err := syncutil.New(syncutil.WithSize(GlobalProbeBulkSize))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create adaptive group")
 	}
@@ -55,6 +49,10 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 			return true
 		}
 
+		if swg.Size != GlobalProbeBulkSize {
+			swg.Resize(GlobalProbeBulkSize)
+		}
+
 		swg.Add()
 		go func(input *contextargs.MetaInput) {
 			defer swg.Done()
diff --git a/pkg/core/engine.go b/pkg/core/engine.go
index 93915bc2c6..4dfb8e0b93 100644
--- a/pkg/core/engine.go
+++ b/pkg/core/engine.go
@@ -30,14 +30,19 @@ func New(options *types.Options) *Engine {
 	return engine
 }
 
-// GetWorkPool returns a workpool from options
-func (e *Engine) GetWorkPool() *WorkPool {
-	return NewWorkPool(WorkPoolConfig{
+func (e *Engine) GetWorkPoolConfig() WorkPoolConfig {
+	config := WorkPoolConfig{
 		InputConcurrency:         e.options.BulkSize,
 		TypeConcurrency:          e.options.TemplateThreads,
 		HeadlessInputConcurrency: e.options.HeadlessBulkSize,
 		HeadlessTypeConcurrency:  e.options.HeadlessTemplateThreads,
-	})
+	}
+	return config
+}
+
+// GetWorkPool returns a workpool from options
+func (e *Engine) GetWorkPool() *WorkPool {
+	return NewWorkPool(e.GetWorkPoolConfig())
 }
 
 // SetExecuterOptions sets the executer options for the engine. This is required
diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go
index 580b8b0a9d..93f197fc2b 100644
--- a/pkg/core/execute_options.go
+++ b/pkg/core/execute_options.go
@@ -108,8 +108,10 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 	wp := e.GetWorkPool()
 
 	for _, template := range templatesList {
-		templateType := template.Type()
+		// resize check point - nop if there are no changes
+		wp.RefreshWithConfig(e.GetWorkPoolConfig())
 
+		templateType := template.Type()
 		var wg *syncutil.AdaptiveWaitGroup
 		if templateType == types.HeadlessProtocol {
 			wg = wp.Headless
diff --git a/pkg/core/executors.go b/pkg/core/executors.go
index ace7acb208..a6421cd90c 100644
--- a/pkg/core/executors.go
+++ b/pkg/core/executors.go
@@ -158,6 +158,9 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	wp := e.GetWorkPool()
 
 	for _, tpl := range alltemplates {
+		// resize check point - nop if there are no changes
+		wp.RefreshWithConfig(e.GetWorkPoolConfig())
+
 		var sg *syncutil.AdaptiveWaitGroup
 		if tpl.Type() == types.HeadlessProtocol {
 			sg = wp.Headless
@@ -213,6 +216,9 @@ func (e *ChildExecuter) Close() *atomic.Bool {
 func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
 	templateType := template.Type()
 
+	// resize check point - nop if there are no changes
+	e.e.workPool.RefreshWithConfig(e.e.GetWorkPoolConfig())
+
 	var wg *syncutil.AdaptiveWaitGroup
 	if templateType == types.HeadlessProtocol {
 		wg = e.e.workPool.Headless
diff --git a/pkg/core/workpool.go b/pkg/core/workpool.go
index cd17a0b367..810f99392d 100644
--- a/pkg/core/workpool.go
+++ b/pkg/core/workpool.go
@@ -57,3 +57,28 @@ func (w *WorkPool) InputPool(templateType types.ProtocolType) *syncutil.Adaptive
 	swg, _ := syncutil.New(syncutil.WithSize(count))
 	return swg
 }
+
+func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) {
+	if w.config.TypeConcurrency != config.TypeConcurrency {
+		w.config.TypeConcurrency = config.TypeConcurrency
+	}
+	if w.config.HeadlessTypeConcurrency != config.HeadlessTypeConcurrency {
+		w.config.HeadlessTypeConcurrency = config.HeadlessTypeConcurrency
+	}
+	if w.config.InputConcurrency != config.InputConcurrency {
+		w.config.InputConcurrency = config.InputConcurrency
+	}
+	if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency {
+		w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency
+	}
+	w.Refresh()
+}
+
+func (w *WorkPool) Refresh() {
+	if w.Default.Size != w.config.TypeConcurrency {
+		w.Default.Resize(w.config.TypeConcurrency)
+	}
+	if w.Headless.Size != w.config.HeadlessTypeConcurrency {
+		w.Headless.Resize(w.config.HeadlessTypeConcurrency)
+	}
+}
diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go
index fc0b61639b..2b3bc2179d 100644
--- a/pkg/js/compiler/pool.go
+++ b/pkg/js/compiler/pool.go
@@ -100,6 +100,11 @@ func executeWithRuntime(runtime *goja.Runtime, p *goja.Program, args *ExecuteArg
 // ExecuteProgram executes a compiled program with the default options.
 // it deligates if a particular program should run in a pooled or non-pooled runtime
 func ExecuteProgram(p *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (result goja.Value, err error) {
+	// resize check point
+	if pooljsc.Size != PoolingJsVmConcurrency {
+		pooljsc.Resize(PoolingJsVmConcurrency)
+	}
+
 	if opts.Source == nil {
 		// not-recommended anymore
 		return executeWithoutPooling(p, args, opts)

From 3c62b56fd9e267a594f7548c1674999768a66caa Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 3 Apr 2024 19:02:30 +0200
Subject: [PATCH 03/16] panic at the pool

---
 pkg/js/compiler/pool.go | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go
index 2b3bc2179d..678b9afb0a 100644
--- a/pkg/js/compiler/pool.go
+++ b/pkg/js/compiler/pool.go
@@ -55,6 +55,12 @@ var (
 	lazySgInit = sync.OnceFunc(func() {
 		pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency))
 	})
+	sgResizeCheck = func() {
+		// resize check point
+		if pooljsc.Size != PoolingJsVmConcurrency {
+			pooljsc.Resize(PoolingJsVmConcurrency)
+		}
+	}
 )
 
 var gojapool = &sync.Pool{
@@ -100,11 +106,6 @@ func executeWithRuntime(runtime *goja.Runtime, p *goja.Program, args *ExecuteArg
 // ExecuteProgram executes a compiled program with the default options.
 // it deligates if a particular program should run in a pooled or non-pooled runtime
 func ExecuteProgram(p *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (result goja.Value, err error) {
-	// resize check point
-	if pooljsc.Size != PoolingJsVmConcurrency {
-		pooljsc.Resize(PoolingJsVmConcurrency)
-	}
-
 	if opts.Source == nil {
 		// not-recommended anymore
 		return executeWithoutPooling(p, args, opts)
@@ -121,6 +122,8 @@ func executeWithPoolingProgram(p *goja.Program, args *ExecuteArgs, opts *Execute
 	// its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static
 	// unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones
 	lazySgInit()
+	sgResizeCheck()
+
 	pooljsc.Add()
 	defer pooljsc.Done()
 	runtime := gojapool.Get().(*goja.Runtime)

From 620287f76b451ca7ce8978d2eaad0d8fe7c1753c Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 3 Apr 2024 19:28:39 +0200
Subject: [PATCH 04/16] deprecating rlm

---
 cmd/nuclei/main.go         |  8 +++++---
 internal/runner/runner.go  | 14 ++++++++++----
 lib/multi.go               | 13 +++++++++----
 lib/sdk_private.go         | 13 +++++++++----
 pkg/testutils/testutils.go |  1 +
 pkg/types/types.go         |  4 ++++
 6 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index 3caec21cbd..b6ecd1e452 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -15,6 +15,7 @@ import (
 	"github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/env"
 	_ "github.com/projectdiscovery/utils/pprof"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
@@ -329,7 +330,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	flagSet.CreateGroup("rate-limit", "Rate-Limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
-		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
+		flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "maximum number of requests to send per second"),
+		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute (DEPRECATED)"),
 		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"),
 		flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
@@ -597,10 +599,10 @@ Note: Make sure you have backup of your custom nuclei-templates before proceedin
 			gologger.Fatal().Msgf("could not read response: %s", err)
 		}
 		resp = strings.TrimSpace(resp)
-		if strings.EqualFold(resp, "y") || strings.EqualFold(resp, "yes") {
+		if stringsutil.EqualFoldAny(resp, "y", "yes") {
 			break
 		}
-		if strings.EqualFold(resp, "n") || strings.EqualFold(resp, "no") || resp == "" {
+		if stringsutil.EqualFoldAny(resp, "n", "no", "") {
 			fmt.Println("Exiting...")
 			os.Exit(0)
 		}
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index 3c6aaf0b53..23443647ea 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -314,11 +314,17 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	if options.RateLimitMinute > 0 {
-		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimitMinute), time.Minute)
-	} else if options.RateLimit > 0 {
-		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), time.Second)
-	} else {
+		gologger.Warning().Msgf("rate limit per minute is deprecated - use rate-limit-duration")
+		options.RateLimit = options.RateLimitMinute
+		options.RateLimitDuration = time.Minute
+	}
+	if options.RateLimit > 0 && options.RateLimitDuration == 0 {
+		options.RateLimitDuration = time.Second
+	}
+	if options.RateLimit == 0 && options.RateLimitDuration == 0 {
 		runner.rateLimiter = ratelimit.NewUnlimited(context.Background())
+	} else {
+		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), options.RateLimitDuration)
 	}
 
 	if tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*"); err == nil {
diff --git a/lib/multi.go b/lib/multi.go
index 3a573b16a9..6fa791a2f0 100644
--- a/lib/multi.go
+++ b/lib/multi.go
@@ -42,11 +42,16 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt
 		Parser:          base.parser,
 	}
 	if opts.RateLimitMinute > 0 {
-		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimitMinute), time.Minute)
-	} else if opts.RateLimit > 0 {
-		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), time.Second)
-	} else {
+		opts.RateLimit = opts.RateLimitMinute
+		opts.RateLimitDuration = time.Minute
+	}
+	if opts.RateLimit > 0 && opts.RateLimitDuration == 0 {
+		opts.RateLimitDuration = time.Second
+	}
+	if opts.RateLimit == 0 && opts.RateLimitDuration == 0 {
 		u.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+	} else {
+		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), opts.RateLimitDuration)
 	}
 	u.engine = core.New(opts)
 	u.engine.SetExecuterOptions(u.executerOpts)
diff --git a/lib/sdk_private.go b/lib/sdk_private.go
index bc66d53fd7..c76970c91d 100644
--- a/lib/sdk_private.go
+++ b/lib/sdk_private.go
@@ -192,11 +192,16 @@ func (e *NucleiEngine) init() error {
 
 	if e.executerOpts.RateLimiter == nil {
 		if e.opts.RateLimitMinute > 0 {
-			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimitMinute), time.Minute)
-		} else if e.opts.RateLimit > 0 {
-			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), time.Second)
-		} else {
+			e.opts.RateLimit = e.opts.RateLimitMinute
+			e.opts.RateLimitDuration = time.Minute
+		}
+		if e.opts.RateLimit > 0 && e.opts.RateLimitDuration == 0 {
+			e.opts.RateLimitDuration = time.Second
+		}
+		if e.opts.RateLimit == 0 && e.opts.RateLimitDuration == 0 {
 			e.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+		} else {
+			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration)
 		}
 	}
 
diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go
index 96d68e1d04..e59aa015e2 100644
--- a/pkg/testutils/testutils.go
+++ b/pkg/testutils/testutils.go
@@ -54,6 +54,7 @@ var DefaultOptions = &types.Options{
 	Timeout:                    5,
 	Retries:                    1,
 	RateLimit:                  150,
+	RateLimitDuration:          time.Second,
 	ProjectPath:                "",
 	Severities:                 severity.Severities{},
 	Targets:                    []string{},
diff --git a/pkg/types/types.go b/pkg/types/types.go
index ce78380901..34f3c3be10 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -132,7 +132,10 @@ type Options struct {
 	Retries int
 	// Rate-Limit is the maximum number of requests per specified target
 	RateLimit int
+	// Rate Limit Duration interval between burst resets
+	RateLimitDuration time.Duration
 	// Rate-Limit is the maximum number of requests per minute for specified target
+	// Deprecated: Use RateLimitDuration - automatically set Rate Limit Duration to 60 seconds
 	RateLimitMinute int
 	// PageTimeout is the maximum time to wait for a page in seconds
 	PageTimeout int
@@ -410,6 +413,7 @@ func (options *Options) HasClientCertificates() bool {
 func DefaultOptions() *Options {
 	return &Options{
 		RateLimit:               150,
+		RateLimitDuration:       time.Second,
 		BulkSize:                25,
 		TemplateThreads:         25,
 		HeadlessBulkSize:        10,

From a140a4194e2a521ecac5bffb03573e6a120c4ad6 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 3 Apr 2024 19:40:09 +0200
Subject: [PATCH 05/16] boh - placing resize in wrapped method

---
 pkg/protocols/dns/request.go       |  2 +-
 pkg/protocols/http/request.go      |  4 ++--
 pkg/protocols/http/request_fuzz.go |  2 +-
 pkg/protocols/protocols.go         | 11 +++++++++++
 4 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index ba9dd66621..8aa8c4a589 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -143,7 +143,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
 		}
 	}
 
-	request.options.RateLimiter.Take()
+	request.options.RateLimitTake()
 
 	// Send the request to the target servers
 	response, err := dnsClient.Do(compiledRequest)
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 2763c5c628..1b27aa7bf3 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -222,7 +222,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 				return
 			case spmHandler.ResultChan <- func() error {
 				// putting ratelimiter here prevents any unnecessary waiting if any
-				request.options.RateLimiter.Take()
+				request.options.RateLimitTake()
 				previous := make(map[string]interface{})
 				return request.executeRequest(input, httpRequest, previous, false, wrappedCallback, 0)
 			}():
@@ -366,7 +366,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 		executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) {
 			hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
 
-			request.options.RateLimiter.Take()
+			request.options.RateLimitTake()
 
 			ctx := request.newContext(input)
 			ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second)
diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go
index d6a29c181c..aed17f54e0 100644
--- a/pkg/protocols/http/request_fuzz.go
+++ b/pkg/protocols/http/request_fuzz.go
@@ -145,7 +145,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
 	if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {
 		return false
 	}
-	request.options.RateLimiter.Take()
+	request.options.RateLimitTake()
 	req := &generatedRequest{
 		request:        gr.Request,
 		dynamicValues:  gr.DynamicValues,
diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
index f9d9064200..5ac0a2f0b7 100644
--- a/pkg/protocols/protocols.go
+++ b/pkg/protocols/protocols.go
@@ -128,6 +128,17 @@ type ExecutorOptions struct {
 	ExportReqURLPattern bool
 }
 
+// todo: centralizing components is not feasible with current clogged architecture
+// a possible approach could be an internal event bus with pub-subs? This would be less invasive than
+// reworking dep injection from scratch
+func (eo *ExecutorOptions) RateLimitTake() {
+	if eo.RateLimiter.GetLimit() != uint(eo.Options.RateLimit) {
+		eo.RateLimiter.SetLimit(uint(eo.Options.RateLimit))
+		eo.RateLimiter.SetDuration(eo.Options.RateLimitDuration)
+	}
+	eo.RateLimiter.Take()
+}
+
 // GetThreadsForPayloadRequests returns the number of threads to use as default for
 // given max-request of payloads
 func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, currentThreads int) int {

From af7450737acfbf2ffd918d8a566596bcc9519250 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Wed, 3 Apr 2024 23:06:08 +0200
Subject: [PATCH 06/16] making payload concurrency dynamic via direct int
 change

---
 pkg/protocols/dns/request.go        |  9 +++++++++
 pkg/protocols/http/httputils/spm.go | 10 ++++++++++
 pkg/protocols/http/request.go       |  9 +++++++++
 pkg/protocols/javascript/js.go      |  9 +++++++++
 pkg/protocols/network/request.go    |  9 +++++++++
 pkg/protocols/protocols.go          | 14 ++------------
 6 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index 8aa8c4a589..d4e70e13e9 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -62,6 +62,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 	variablesMap := request.options.Variables.Evaluate(vars)
 	vars = generators.MergeMaps(vars, variablesMap, request.options.Constants)
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
+
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		swg, err := syncutil.New(syncutil.WithSize(request.Threads))
@@ -76,6 +79,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 			if !ok {
 				break
 			}
+
+			// resize check point - nop if there are no changes
+			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
+				swg.Resize(request.options.Options.PayloadConcurrency)
+			}
+
 			value = generators.MergeMaps(vars, value)
 			swg.Add()
 			go func(newVars map[string]interface{}) {
diff --git a/pkg/protocols/http/httputils/spm.go b/pkg/protocols/http/httputils/spm.go
index bca6c2ee59..52d13f06ff 100644
--- a/pkg/protocols/http/httputils/spm.go
+++ b/pkg/protocols/http/httputils/spm.go
@@ -143,6 +143,16 @@ func (h *StopAtFirstMatchHandler[T]) Release() {
 	}
 }
 
+func (h *StopAtFirstMatchHandler[T]) Resize(size int) {
+	if h.sgPool.Size != size {
+		h.sgPool.Resize(size)
+	}
+}
+
+func (h *StopAtFirstMatchHandler[T]) Size() int {
+	return h.sgPool.Size
+}
+
 // Wait waits for all work to be done
 func (h *StopAtFirstMatchHandler[T]) Wait() {
 	switch h.poolType {
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 1b27aa7bf3..5d64216f6a 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -165,6 +165,9 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 	// Workers that keeps enqueuing new requests
 	maxWorkers := request.Threads
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := maxWorkers == request.options.Options.PayloadConcurrency
+
 	if protocolstate.IsLowOnMemory() {
 		maxWorkers = protocolstate.GuardThreadsOrDefault(request.Threads)
 	}
@@ -198,6 +201,12 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		if !ok {
 			break
 		}
+
+		// resize check point - nop if there are no changes
+		if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
+			spmHandler.Resize(request.options.Options.PayloadConcurrency)
+		}
+
 		ctx := request.newContext(input)
 		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
 		if err != nil {
diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go
index 48a1be485d..efabd503c4 100644
--- a/pkg/protocols/javascript/js.go
+++ b/pkg/protocols/javascript/js.go
@@ -404,6 +404,9 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 	requestOptions := request.options
 	gotmatches := &atomic.Bool{}
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := threads == request.options.Options.PayloadConcurrency
+
 	sg, _ := syncutil.New(syncutil.WithSize(threads))
 
 	if request.generator != nil {
@@ -413,6 +416,12 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 			if !ok {
 				break
 			}
+
+			// resize check point - nop if there are no changes
+			if shouldFollowGlobal && sg.Size != request.options.Options.PayloadConcurrency {
+				sg.Resize(request.options.Options.PayloadConcurrency)
+			}
+
 			sg.Add()
 			go func() {
 				defer sg.Done()
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index ef0ff01c7f..146c657072 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -174,6 +174,9 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 		return err
 	}
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
+
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		var multiErr error
@@ -188,6 +191,12 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 			if !ok {
 				break
 			}
+
+			// resize check point - nop if there are no changes
+			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
+				swg.Resize(request.options.Options.PayloadConcurrency)
+			}
+
 			value = generators.MergeMaps(value, payloads)
 			swg.Add()
 			go func(vars map[string]interface{}) {
diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
index 5ac0a2f0b7..6328bc36a3 100644
--- a/pkg/protocols/protocols.go
+++ b/pkg/protocols/protocols.go
@@ -34,9 +34,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 )
 
-// Optional Callback to update Thread count in payloads across all requests
-type PayloadThreadSetterCallback func(opts *ExecutorOptions, totalRequests, currentThreads int) int
-
 var (
 	MaxTemplateFileSizeForEncoding = 1024 * 1024
 )
@@ -114,10 +111,6 @@ type ExecutorOptions struct {
 	// JsCompiler is abstracted javascript compiler which adds node modules and provides execution
 	// environment for javascript templates
 	JsCompiler *compiler.Compiler
-	// Optional Callback function to update Thread count in payloads across all protocols
-	// based on given logic. by default nuclei reverts to using value of `-c` when threads count
-	// is not specified or is 0 in template
-	OverrideThreadsCount PayloadThreadSetterCallback
 	// AuthProvider is a provider for auth strategies
 	AuthProvider authprovider.AuthProvider
 	//TemporaryDirectory is the directory to store temporary files
@@ -142,14 +135,11 @@ func (eo *ExecutorOptions) RateLimitTake() {
 // GetThreadsForPayloadRequests returns the number of threads to use as default for
 // given max-request of payloads
 func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, currentThreads int) int {
-	if e.OverrideThreadsCount != nil {
-		return e.OverrideThreadsCount(e, totalRequests, currentThreads)
-	}
 	if currentThreads > 0 {
 		return currentThreads
-	} else {
-		return e.Options.PayloadConcurrency
 	}
+
+	return e.Options.PayloadConcurrency
 }
 
 // CreateTemplateCtxStore creates template context store (which contains templateCtx for every scan)

From 99a018d12ad0e479abed4a1c663239db90dcfc0a Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 4 Apr 2024 20:19:46 +0200
Subject: [PATCH 07/16] adding speed change example

---
 examples/with_withspeed_control/main.go | 99 +++++++++++++++++++++++++
 go.mod                                  |  4 +-
 go.sum                                  |  8 +-
 lib/config.go                           | 42 +++++++++--
 lib/sdk.go                              |  8 ++
 5 files changed, 148 insertions(+), 13 deletions(-)
 create mode 100644 examples/with_withspeed_control/main.go

diff --git a/examples/with_withspeed_control/main.go b/examples/with_withspeed_control/main.go
new file mode 100644
index 0000000000..a61e820f26
--- /dev/null
+++ b/examples/with_withspeed_control/main.go
@@ -0,0 +1,99 @@
+package main
+
+import (
+	"sync"
+	"time"
+
+	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+)
+
+func main() {
+	ne, err := nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"header-command-injection"}}),
+		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
+		nuclei.WithGlobalRateLimit(1, time.Second),
+		nuclei.WithConcurrency(nuclei.Concurrency{
+			TemplateConcurrency:           1,
+			HostConcurrency:               1,
+			HeadlessHostConcurrency:       1,
+			HeadlessTemplateConcurrency:   1,
+			JavascriptTemplateConcurrency: 1,
+			TemplatePayloadConcurrency:    1,
+		}),
+	)
+	if err != nil {
+		panic(err)
+	}
+	// load targets and optionally probe non http/https targets
+	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
+
+	var wgtest sync.WaitGroup
+
+	// speed tests
+	// increase rate limit
+	wgtest.Add(1)
+	go func() {
+		defer wgtest.Done()
+		initialRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
+		if initialRate != 1 {
+			panic("wrong initial rate limit")
+		}
+		time.Sleep(10 * time.Second)
+		ne.Options().RateLimit = 5
+		time.Sleep(10 * time.Second)
+		finalRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
+		if finalRate != 5 {
+			panic("wrong final rate limit")
+		}
+	}()
+
+	// increase threads and bulk size
+	wgtest.Add(1)
+	go func() {
+		defer wgtest.Done()
+		initialTemplateThreads := ne.Options().TemplateThreads
+		initialBulkSize := ne.Options().BulkSize
+		if initialTemplateThreads != 1 || initialBulkSize != 1 {
+			panic("wrong initial standard concurrency")
+		}
+		time.Sleep(10 * time.Second)
+		ne.Options().TemplateThreads = 5
+		ne.Options().BulkSize = 25
+		time.Sleep(10 * time.Second)
+		// check new values via workpool
+		finalTemplateThreads := ne.Engine().WorkPool().Default.Size
+		finalBulkSize := ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size
+		if finalTemplateThreads != 5 && finalBulkSize != 25 {
+			panic("wrong final concurreny")
+		}
+	}()
+
+	// increase payload concurrency
+	wgtest.Add(1)
+	go func() {
+		defer wgtest.Done()
+		initialpayloadConcurrency := ne.Options().PayloadConcurrency
+		if initialpayloadConcurrency != 1 {
+			panic("wrong initial payload concurrency")
+		}
+		time.Sleep(10 * time.Second)
+		ne.Options().PayloadConcurrency = 5
+		time.Sleep(10 * time.Second)
+
+		// the ongoing and next payload iterations will retrieve parallelism from this function
+		// it should have the new set value, that will be cascade applied to all running adaptive wait groups
+		finalPayloadConcurrency := ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0)
+		if finalPayloadConcurrency != 5 {
+			panic("wrong initial payload concurrency")
+		}
+	}()
+
+	err = ne.ExecuteWithCallback(nil)
+	if err != nil {
+		panic(err)
+	}
+	defer ne.Close()
+
+	wgtest.Wait()
+}
diff --git a/go.mod b/go.mod
index 6a0e8bfccf..981d174836 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.64
+	github.com/projectdiscovery/fastdialer v0.0.65
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
 	github.com/projectdiscovery/rawhttp v0.1.41
@@ -94,7 +94,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.40
-	github.com/projectdiscovery/utils v0.0.87
+	github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0
 	github.com/projectdiscovery/wappalyzergo v0.0.116
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index 83551eb1c2..8557e5d49e 100644
--- a/go.sum
+++ b/go.sum
@@ -834,8 +834,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
 github.com/projectdiscovery/dsl v0.0.50 h1:4SuAwTS9l6o1tqlIC/79+EcUwTM6CjaU7MpY/nDlFaM=
 github.com/projectdiscovery/dsl v0.0.50/go.mod h1:6g740l4tH4d2j9UYtIchtxudb0Dphkq4o+VatpR4M6g=
-github.com/projectdiscovery/fastdialer v0.0.64 h1:xivkA4g14nwQElOVsxPkGMWsdcYPcp7DPhVjvI6yQkw=
-github.com/projectdiscovery/fastdialer v0.0.64/go.mod h1:S/7PAQRmVDYRaU7u4xXD0qA5a48NAZq2JcpcVoEVrlo=
+github.com/projectdiscovery/fastdialer v0.0.65 h1:msvKVJyILtP04CXSgSEWv4rUVsk0CCd3xhauo+H82IU=
+github.com/projectdiscovery/fastdialer v0.0.65/go.mod h1:wIE10NL7oa/zBCJfr1xAduv3q73aeuGbhfZ1Z8o4NUo=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -886,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
 github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.87 h1:9+RiTEhpUB/vk6XJUVpysNWJ2aCTD7WuyoyAcNnbIzk=
-github.com/projectdiscovery/utils v0.0.87/go.mod h1:jGK450sL9AVDTjaPwEs9za8NVeEC9xE97IWNoK138kI=
+github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0 h1:2ZR0yiN0cUm/qYEMq79MfcbgM374lJSdftheYhMFxNo=
+github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
 github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
diff --git a/lib/config.go b/lib/config.go
index 52a8e2328a..fd4e9c9a97 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -2,6 +2,7 @@ package nuclei
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"github.com/projectdiscovery/goflags"
@@ -120,12 +121,37 @@ type Concurrency struct {
 // WithConcurrency sets concurrency options
 func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
-		e.opts.TemplateThreads = opts.TemplateConcurrency
-		e.opts.BulkSize = opts.HostConcurrency
-		e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
-		e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
-		e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
-		e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
+		// minimum required is 1
+		if opts.TemplateConcurrency <= 0 {
+			return errors.New("template threads must be at least 1")
+		} else {
+			e.opts.TemplateThreads = opts.TemplateConcurrency
+		}
+		if opts.HostConcurrency <= 0 {
+			return errors.New("host concurrency must be at least 1")
+		} else {
+			e.opts.BulkSize = opts.HostConcurrency
+		}
+		if opts.HeadlessHostConcurrency <= 0 {
+			return errors.New("headless host concurrency must be at least 1")
+		} else {
+			e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
+		}
+		if opts.HeadlessTemplateConcurrency <= 0 {
+			return errors.New("headless template threads must be at least 1")
+		} else {
+			e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
+		}
+		if opts.JavascriptTemplateConcurrency <= 0 {
+			return errors.New("js must be at least 1")
+		} else {
+			e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
+		}
+		if opts.TemplatePayloadConcurrency <= 0 {
+			return errors.New("payload concurrency must be at least 1")
+		} else {
+			e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
+		}
 		return nil
 	}
 }
@@ -133,7 +159,9 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 // WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
 func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
-		e.rateLimiter = ratelimit.New(context.Background(), uint(maxTokens), duration)
+		e.opts.RateLimit = maxTokens
+		e.opts.RateLimitDuration = duration
+		e.rateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration)
 		return nil
 	}
 }
diff --git a/lib/sdk.go b/lib/sdk.go
index da0abae495..4bee921c31 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -215,6 +215,14 @@ func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.Result
 	return nil
 }
 
+func (e *NucleiEngine) Options() *types.Options {
+	return e.opts
+}
+
+func (e *NucleiEngine) Engine() *core.Engine {
+	return e.engine
+}
+
 // NewNucleiEngine creates a new nuclei engine instance
 func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) {
 	// default options

From 96d7d02701363821cd7805601ff18dc824e15abf Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 4 Apr 2024 20:22:28 +0200
Subject: [PATCH 08/16] adding speed example

---
 .github/workflows/build-test.yml                              | 4 ++++
 .../{with_withspeed_control => with_speed_control}/main.go    | 0
 2 files changed, 4 insertions(+)
 rename examples/{with_withspeed_control => with_speed_control}/main.go (100%)

diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 7ebdaee1c0..aa0d8aecfe 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -66,3 +66,7 @@ jobs:
       - name: Example SDK Advanced
         run: go run .
         working-directory: examples/advanced/
+
+      - name: Example SDK with speed control
+        run: go run .
+        working-directory: examples/with_speed_control/
diff --git a/examples/with_withspeed_control/main.go b/examples/with_speed_control/main.go
similarity index 100%
rename from examples/with_withspeed_control/main.go
rename to examples/with_speed_control/main.go

From c4aa7dc1a20efa3ff02a036ce2acb0ae2274e8a7 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 4 Apr 2024 20:43:00 +0200
Subject: [PATCH 09/16] f tag

---
 examples/with_speed_control/main.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go
index a61e820f26..ed3592bade 100644
--- a/examples/with_speed_control/main.go
+++ b/examples/with_speed_control/main.go
@@ -10,7 +10,10 @@ import (
 
 func main() {
 	ne, err := nuclei.NewNucleiEngine(
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"header-command-injection"}}),
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{
+			IDs:         []string{"header-command-injection"},
+			IncludeTags: []string{"fuzz"},
+		}),
 		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
 		nuclei.WithGlobalRateLimit(1, time.Second),
 		nuclei.WithConcurrency(nuclei.Concurrency{

From aa06c9ef1798448da2670671b6c299b13e55064d Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 4 Apr 2024 21:21:06 +0200
Subject: [PATCH 10/16] speed up

---
 examples/with_speed_control/main.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go
index ed3592bade..8637209a6f 100644
--- a/examples/with_speed_control/main.go
+++ b/examples/with_speed_control/main.go
@@ -43,10 +43,10 @@ func main() {
 			panic("wrong initial rate limit")
 		}
 		time.Sleep(10 * time.Second)
-		ne.Options().RateLimit = 5
+		ne.Options().RateLimit = 1000
 		time.Sleep(10 * time.Second)
 		finalRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
-		if finalRate != 5 {
+		if finalRate != 1000 {
 			panic("wrong final rate limit")
 		}
 	}()
@@ -81,13 +81,13 @@ func main() {
 			panic("wrong initial payload concurrency")
 		}
 		time.Sleep(10 * time.Second)
-		ne.Options().PayloadConcurrency = 5
+		ne.Options().PayloadConcurrency = 100
 		time.Sleep(10 * time.Second)
 
 		// the ongoing and next payload iterations will retrieve parallelism from this function
 		// it should have the new set value, that will be cascade applied to all running adaptive wait groups
 		finalPayloadConcurrency := ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0)
-		if finalPayloadConcurrency != 5 {
+		if finalPayloadConcurrency != 100 {
 			panic("wrong initial payload concurrency")
 		}
 	}()

From d0a0c6d0c3ab9a1c170716a48ade3c6c649190aa Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Fri, 5 Apr 2024 15:45:21 +0200
Subject: [PATCH 11/16] internal sync fix + speed up

---
 examples/with_speed_control/main.go | 145 ++++++++++++++--------------
 go.mod                              |   8 +-
 go.sum                              |  15 +--
 pkg/core/engine.go                  |   2 +
 pkg/core/executors.go               |   2 +-
 5 files changed, 88 insertions(+), 84 deletions(-)

diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go
index 8637209a6f..36a26cdd1c 100644
--- a/examples/with_speed_control/main.go
+++ b/examples/with_speed_control/main.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"log"
 	"sync"
 	"time"
 
@@ -9,11 +10,32 @@ import (
 )
 
 func main() {
-	ne, err := nuclei.NewNucleiEngine(
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{
-			IDs:         []string{"header-command-injection"},
-			IncludeTags: []string{"fuzz"},
-		}),
+	ne, err := initializeNucleiEngine()
+	if err != nil {
+		panic(err)
+	}
+	defer ne.Close()
+
+	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+
+	go testRateLimit(&wg, ne)
+	go testThreadsAndBulkSize(&wg, ne)
+	go testPayloadConcurrency(&wg, ne)
+
+	err = ne.ExecuteWithCallback(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	wg.Wait()
+}
+
+func initializeNucleiEngine() (*nuclei.NucleiEngine, error) {
+	return nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "http"}),
 		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
 		nuclei.WithGlobalRateLimit(1, time.Second),
 		nuclei.WithConcurrency(nuclei.Concurrency{
@@ -25,78 +47,57 @@ func main() {
 			TemplatePayloadConcurrency:    1,
 		}),
 	)
-	if err != nil {
-		panic(err)
-	}
-	// load targets and optionally probe non http/https targets
-	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
-
-	var wgtest sync.WaitGroup
+}
 
-	// speed tests
-	// increase rate limit
-	wgtest.Add(1)
-	go func() {
-		defer wgtest.Done()
-		initialRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
-		if initialRate != 1 {
-			panic("wrong initial rate limit")
-		}
-		time.Sleep(10 * time.Second)
-		ne.Options().RateLimit = 1000
-		time.Sleep(10 * time.Second)
-		finalRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
-		if finalRate != 1000 {
-			panic("wrong final rate limit")
-		}
-	}()
+func testRateLimit(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	verifyRateLimit(ne, 1, 5000)
+}
 
-	// increase threads and bulk size
-	wgtest.Add(1)
-	go func() {
-		defer wgtest.Done()
-		initialTemplateThreads := ne.Options().TemplateThreads
-		initialBulkSize := ne.Options().BulkSize
-		if initialTemplateThreads != 1 || initialBulkSize != 1 {
-			panic("wrong initial standard concurrency")
-		}
-		time.Sleep(10 * time.Second)
-		ne.Options().TemplateThreads = 5
-		ne.Options().BulkSize = 25
-		time.Sleep(10 * time.Second)
-		// check new values via workpool
-		finalTemplateThreads := ne.Engine().WorkPool().Default.Size
-		finalBulkSize := ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size
-		if finalTemplateThreads != 5 && finalBulkSize != 25 {
-			panic("wrong final concurreny")
-		}
-	}()
+func testThreadsAndBulkSize(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	initialTemplateThreads, initialBulkSize := 1, 1
+	verifyThreadsAndBulkSize(ne, initialTemplateThreads, initialBulkSize, 25, 25)
+}
 
-	// increase payload concurrency
-	wgtest.Add(1)
-	go func() {
-		defer wgtest.Done()
-		initialpayloadConcurrency := ne.Options().PayloadConcurrency
-		if initialpayloadConcurrency != 1 {
-			panic("wrong initial payload concurrency")
-		}
-		time.Sleep(10 * time.Second)
-		ne.Options().PayloadConcurrency = 100
-		time.Sleep(10 * time.Second)
+func testPayloadConcurrency(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	verifyPayloadConcurrency(ne, 1, 500)
+}
 
-		// the ongoing and next payload iterations will retrieve parallelism from this function
-		// it should have the new set value, that will be cascade applied to all running adaptive wait groups
-		finalPayloadConcurrency := ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0)
-		if finalPayloadConcurrency != 100 {
-			panic("wrong initial payload concurrency")
-		}
-	}()
+func verifyRateLimit(ne *nuclei.NucleiEngine, initialRate, finalRate int) {
+	if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(initialRate) {
+		panic("wrong initial rate limit")
+	}
+	time.Sleep(5 * time.Second)
+	ne.Options().RateLimit = finalRate
+	time.Sleep(20 * time.Second)
+	if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(finalRate) {
+		panic("wrong final rate limit")
+	}
+}
 
-	err = ne.ExecuteWithCallback(nil)
-	if err != nil {
-		panic(err)
+func verifyThreadsAndBulkSize(ne *nuclei.NucleiEngine, initialThreads, initialBulk, finalThreads, finalBulk int) {
+	if ne.Options().TemplateThreads != initialThreads || ne.Options().BulkSize != initialBulk {
+		panic("wrong initial standard concurrency")
 	}
-	defer ne.Close()
+	time.Sleep(5 * time.Second)
+	ne.Options().TemplateThreads = finalThreads
+	ne.Options().BulkSize = finalBulk
+	time.Sleep(20 * time.Second)
+	if ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size != finalBulk || ne.Engine().WorkPool().Default.Size != finalThreads {
+		log.Fatal("wrong final concurrency", ne.Engine().WorkPool().Default.Size, finalThreads, ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size, finalBulk)
+	}
+}
 
-	wgtest.Wait()
+func verifyPayloadConcurrency(ne *nuclei.NucleiEngine, initialPayloadConcurrency, finalPayloadConcurrency int) {
+	if ne.Options().PayloadConcurrency != initialPayloadConcurrency {
+		panic("wrong initial payload concurrency")
+	}
+	time.Sleep(5 * time.Second)
+	ne.Options().PayloadConcurrency = finalPayloadConcurrency
+	time.Sleep(20 * time.Second)
+	if ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0) != finalPayloadConcurrency {
+		panic("wrong final payload concurrency")
+	}
 }
diff --git a/go.mod b/go.mod
index 981d174836..a116f9611a 100644
--- a/go.mod
+++ b/go.mod
@@ -38,7 +38,7 @@ require (
 	github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
 	github.com/xanzy/go-gitlab v0.84.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/net v0.21.0
+	golang.org/x/net v0.24.0
 	golang.org/x/oauth2 v0.11.0
 	golang.org/x/text v0.14.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -100,7 +100,7 @@ require (
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
-	golang.org/x/term v0.17.0
+	golang.org/x/term v0.19.0
 	gopkg.in/yaml.v3 v3.0.1
 	moul.io/http2curl v1.0.0
 )
@@ -302,10 +302,10 @@ require (
 	go.etcd.io/bbolt v1.3.8 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	goftp.io/server/v2 v2.0.1 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.17.0
 	google.golang.org/appengine v1.6.7 // indirect
diff --git a/go.sum b/go.sum
index 8557e5d49e..a179db9b28 100644
--- a/go.sum
+++ b/go.sum
@@ -1186,8 +1186,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1280,8 +1280,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1382,8 +1382,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1395,8 +1396,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/pkg/core/engine.go b/pkg/core/engine.go
index 4dfb8e0b93..1b4155bbb5 100644
--- a/pkg/core/engine.go
+++ b/pkg/core/engine.go
@@ -58,5 +58,7 @@ func (e *Engine) ExecuterOptions() protocols.ExecutorOptions {
 
 // WorkPool returns the worker pool for the engine
 func (e *Engine) WorkPool() *WorkPool {
+	// resize check point - nop if there are no changes
+	e.workPool.RefreshWithConfig(e.GetWorkPoolConfig())
 	return e.workPool
 }
diff --git a/pkg/core/executors.go b/pkg/core/executors.go
index a6421cd90c..14fb75c631 100644
--- a/pkg/core/executors.go
+++ b/pkg/core/executors.go
@@ -217,7 +217,7 @@ func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs
 	templateType := template.Type()
 
 	// resize check point - nop if there are no changes
-	e.e.workPool.RefreshWithConfig(e.e.GetWorkPoolConfig())
+	e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig())
 
 	var wg *syncutil.AdaptiveWaitGroup
 	if templateType == types.HeadlessProtocol {

From f886fb2cf4a4e5e4500b3d336cac499e423e9994 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Fri, 5 Apr 2024 16:40:37 +0200
Subject: [PATCH 12/16] less templates in test

---
 examples/with_speed_control/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go
index 36a26cdd1c..88f7782cf1 100644
--- a/examples/with_speed_control/main.go
+++ b/examples/with_speed_control/main.go
@@ -35,7 +35,7 @@ func main() {
 
 func initializeNucleiEngine() (*nuclei.NucleiEngine, error) {
 	return nuclei.NewNucleiEngine(
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "http"}),
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}),
 		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
 		nuclei.WithGlobalRateLimit(1, time.Second),
 		nuclei.WithConcurrency(nuclei.Concurrency{

From 582a85d9c060ed74e308750f198ccb2f12864629 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 9 Apr 2024 18:31:22 +0200
Subject: [PATCH 13/16] mimic follow behavior

---
 cmd/nuclei/main.go        |  1 +
 internal/runner/inputs.go | 11 +++++++++--
 lib/config.go             |  6 ++++++
 pkg/types/types.go        |  2 ++
 4 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index b6ecd1e452..a90c4ed52e 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -338,6 +338,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"),
 		flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"),
 		flagSet.IntVarP(&options.PayloadConcurrency, "payload-concurrency", "pc", 25, "max payload concurrency for each template"),
+		flagSet.IntVarP(&options.ProbeConcurrency, "probe-concurrency", "prc", 50, "http probe concurrency with httpx"),
 	)
 	flagSet.CreateGroup("optimization", "Optimizations",
 		flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"),
diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go
index 60aa03199f..e7a7c29fae 100644
--- a/internal/runner/inputs.go
+++ b/internal/runner/inputs.go
@@ -30,6 +30,11 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	gologger.Info().Msgf("Running httpx on input host")
 
+	var bulkSize = GlobalProbeBulkSize
+	if r.options.BulkSize > GlobalProbeBulkSize {
+		bulkSize = r.options.BulkSize
+	}
+
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
@@ -38,8 +43,10 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 		return nil, errors.Wrap(err, "could not create httpx client")
 	}
 
+	shouldFollowGlobalProbeBulkSize := bulkSize == GlobalProbeBulkSize
+
 	// Probe the non-standard URLs and store them in cache
-	swg, err := syncutil.New(syncutil.WithSize(GlobalProbeBulkSize))
+	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create adaptive group")
 	}
@@ -49,7 +56,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 			return true
 		}
 
-		if swg.Size != GlobalProbeBulkSize {
+		if shouldFollowGlobalProbeBulkSize && swg.Size != GlobalProbeBulkSize {
 			swg.Resize(GlobalProbeBulkSize)
 		}
 
diff --git a/lib/config.go b/lib/config.go
index fd4e9c9a97..5b112f8d25 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -116,6 +116,7 @@ type Concurrency struct {
 	HeadlessTemplateConcurrency   int // number of templates to run concurrently for headless templates (per host in host-spray mode)
 	JavascriptTemplateConcurrency int // number of templates to run concurrently for javascript templates (per host in host-spray mode)
 	TemplatePayloadConcurrency    int // max concurrent payloads to run for a template (a good default is 25)
+	ProbeConcurrency              int // max concurrent http probes to run (a good default is 50)
 }
 
 // WithConcurrency sets concurrency options
@@ -152,6 +153,11 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 		} else {
 			e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
 		}
+		if opts.ProbeConcurrency <= 0 {
+			return errors.New("probe concurrency must be at least 1")
+		} else {
+			e.opts.ProbeConcurrency = opts.ProbeConcurrency
+		}
 		return nil
 	}
 }
diff --git a/pkg/types/types.go b/pkg/types/types.go
index 34f3c3be10..81a2731689 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -385,6 +385,8 @@ type Options struct {
 	SkipFormatValidation bool
 	// PayloadConcurrency is the number of concurrent payloads to run per template
 	PayloadConcurrency int
+	// ProbeConcurrency is the number of concurrent http probes to run with httpx
+	ProbeConcurrency int
 	// Dast only runs DAST templates
 	DAST bool
 }

From 70eb494a01ed17b665d6a4192e93174104365caa Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 9 Apr 2024 18:32:40 +0200
Subject: [PATCH 14/16] warning

---
 internal/runner/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index 23443647ea..76678bcb71 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -314,7 +314,7 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	if options.RateLimitMinute > 0 {
-		gologger.Warning().Msgf("rate limit per minute is deprecated - use rate-limit-duration")
+		gologger.Print().Msgf("[%v] %v", aurora.BrightYellow("WRN"), fmt.Sprintf("rate limit per minute is deprecated - use rate-limit-duration"))
 		options.RateLimit = options.RateLimitMinute
 		options.RateLimitDuration = time.Minute
 	}

From 6eeb98c71bddb4acfa39911fbc0844d475caeb47 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 9 Apr 2024 18:34:04 +0200
Subject: [PATCH 15/16] removing printf

---
 internal/runner/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index 76678bcb71..fa9b85bf05 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -314,7 +314,7 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	if options.RateLimitMinute > 0 {
-		gologger.Print().Msgf("[%v] %v", aurora.BrightYellow("WRN"), fmt.Sprintf("rate limit per minute is deprecated - use rate-limit-duration"))
+		gologger.Print().Msgf("[%v] %v", aurora.BrightYellow("WRN"), "rate limit per minute is deprecated - use rate-limit-duration")
 		options.RateLimit = options.RateLimitMinute
 		options.RateLimitDuration = time.Minute
 	}

From 5fc08cec48cf27029598ed11679b2f9b0bfabf13 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 9 Apr 2024 18:51:25 +0200
Subject: [PATCH 16/16] updating example

---
 examples/with_speed_control/main.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go
index 88f7782cf1..b56df967c5 100644
--- a/examples/with_speed_control/main.go
+++ b/examples/with_speed_control/main.go
@@ -45,6 +45,7 @@ func initializeNucleiEngine() (*nuclei.NucleiEngine, error) {
 			HeadlessTemplateConcurrency:   1,
 			JavascriptTemplateConcurrency: 1,
 			TemplatePayloadConcurrency:    1,
+			ProbeConcurrency:              1,
 		}),
 	)
 }