From a8d1393e961d5db72c45efa840a841854e96a2c1 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 3 Apr 2024 17:50:57 +0200
Subject: [PATCH 01/58] init- using resizable components

---
 examples/advanced/advanced.go                 |  7 +++-
 go.mod                                        |  2 +
 go.sum                                        |  3 ++
 internal/runner/inputs.go                     | 13 +++---
 pkg/core/execute_options.go                   |  7 ++--
 pkg/core/executors.go                         | 14 +++----
 pkg/core/workflow_execute.go                  |  9 ++--
 pkg/core/workpool.go                          | 26 +++++-------
 pkg/js/compiler/non-pool.go                   |  6 +--
 pkg/js/compiler/pool.go                       |  6 +--
 .../common/automaticscan/automaticscan.go     |  9 ++--
 pkg/protocols/dns/request.go                  |  7 +++-
 pkg/protocols/file/request.go                 |  9 ++--
 pkg/protocols/headless/engine/page_actions.go | 28 ++++++-------
 pkg/protocols/http/httputils/spm.go           |  9 ++--
 pkg/protocols/javascript/js.go                |  5 ++-
 .../network/networkclientpool/clientpool.go   |  2 +-
 pkg/protocols/network/request.go              |  7 +++-
 pkg/protocols/offlinehttp/request.go          | 11 +++--
 pkg/tmplexec/flow/vm.go                       | 41 +++++++++----------
 20 files changed, 121 insertions(+), 100 deletions(-)

diff --git a/examples/advanced/advanced.go b/examples/advanced/advanced.go
index 5ce579b3d6..110160f9a1 100644
--- a/examples/advanced/advanced.go
+++ b/examples/advanced/advanced.go
@@ -2,7 +2,7 @@ package main
 
 import (
 	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 func main() {
@@ -12,7 +12,10 @@ func main() {
 		panic(err)
 	}
 	// setup sizedWaitgroup to handle concurrency
-	sg := sizedwaitgroup.New(10)
+	sg, err := syncutil.New(syncutil.WithSize(10))
+	if err != nil {
+		panic(err)
+	}
 
 	// scan 1 = run dns templates on scanme.sh
 	sg.Add()
diff --git a/go.mod b/go.mod
index 7eeaf57386..6a0e8bfccf 100644
--- a/go.mod
+++ b/go.mod
@@ -142,6 +142,8 @@ require (
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/eapache/channels v1.1.0 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
diff --git a/go.sum b/go.sum
index cefd9ae445..83551eb1c2 100644
--- a/go.sum
+++ b/go.sum
@@ -296,8 +296,11 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
+github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go
index 8dc27a7a9e..75a869912b 100644
--- a/internal/runner/inputs.go
+++ b/internal/runner/inputs.go
@@ -12,7 +12,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	stringsutil "github.com/projectdiscovery/utils/strings"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 const probeBulkSize = 50
@@ -45,8 +45,11 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 
 	// Probe the non-standard URLs and store them in cache
-	swg := sizedwaitgroup.New(bulkSize)
-	count := int32(0)
+	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create adaptive group")
+	}
+	var count atomic.Int32
 	r.inputProvider.Iterate(func(value *contextargs.MetaInput) bool {
 		if stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
 			return true
@@ -57,7 +60,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 			defer swg.Done()
 
 			if result := utils.ProbeURL(input.Input, httpxClient); result != "" {
-				atomic.AddInt32(&count, 1)
+				count.Add(1)
 				_ = hm.Set(input.Input, []byte(result))
 			}
 		}(value)
@@ -65,6 +68,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	})
 	swg.Wait()
 
-	gologger.Info().Msgf("Found %d URL from httpx", atomic.LoadInt32(&count))
+	gologger.Info().Msgf("Found %d URL from httpx", count.Load())
 	return hm, nil
 }
diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go
index fd1fadae3d..580b8b0a9d 100644
--- a/pkg/core/execute_options.go
+++ b/pkg/core/execute_options.go
@@ -4,8 +4,6 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/remeh/sizedwaitgroup"
-
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -14,6 +12,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // Execute takes a list of templates/workflows that have been compiled
@@ -111,7 +110,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 	for _, template := range templatesList {
 		templateType := template.Type()
 
-		var wg *sizedwaitgroup.SizedWaitGroup
+		var wg *syncutil.AdaptiveWaitGroup
 		if templateType == types.HeadlessProtocol {
 			wg = wp.Headless
 		} else {
@@ -134,7 +133,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 // executeHostSpray executes scan using host spray strategy where templates are iterated over each target
 func (e *Engine) executeHostSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
 	results := &atomic.Bool{}
-	wp := sizedwaitgroup.New(e.options.BulkSize + e.options.HeadlessBulkSize)
+	wp, _ := syncutil.New(syncutil.WithSize(e.options.BulkSize + e.options.HeadlessBulkSize))
 
 	target.Iterate(func(value *contextargs.MetaInput) bool {
 		wp.Add()
diff --git a/pkg/core/executors.go b/pkg/core/executors.go
index b491bd8e0b..ace7acb208 100644
--- a/pkg/core/executors.go
+++ b/pkg/core/executors.go
@@ -11,7 +11,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	generalTypes "github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // Executors are low level executors that deals with template execution on a target
@@ -104,9 +104,9 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 			return true
 		}
 
-		wg.WaitGroup.Add()
+		wg.Add()
 		go func(index uint32, skip bool, value *contextargs.MetaInput) {
-			defer wg.WaitGroup.Done()
+			defer wg.Done()
 			defer cleanupInFlight(index)
 			if skip {
 				return
@@ -140,7 +140,7 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 		index++
 		return true
 	})
-	wg.WaitGroup.Wait()
+	wg.Wait()
 
 	// on completion marks the template as completed
 	currentInfo.Lock()
@@ -158,14 +158,14 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	wp := e.GetWorkPool()
 
 	for _, tpl := range alltemplates {
-		var sg *sizedwaitgroup.SizedWaitGroup
+		var sg *syncutil.AdaptiveWaitGroup
 		if tpl.Type() == types.HeadlessProtocol {
 			sg = wp.Headless
 		} else {
 			sg = wp.Default
 		}
 		sg.Add()
-		go func(template *templates.Template, value *contextargs.MetaInput, wg *sizedwaitgroup.SizedWaitGroup) {
+		go func(template *templates.Template, value *contextargs.MetaInput, wg *syncutil.AdaptiveWaitGroup) {
 			defer wg.Done()
 
 			var match bool
@@ -213,7 +213,7 @@ func (e *ChildExecuter) Close() *atomic.Bool {
 func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
 	templateType := template.Type()
 
-	var wg *sizedwaitgroup.SizedWaitGroup
+	var wg *syncutil.AdaptiveWaitGroup
 	if templateType == types.HeadlessProtocol {
 		wg = e.e.workPool.Headless
 	} else {
diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go
index cb877cc60e..19d6f0d69d 100644
--- a/pkg/core/workflow_execute.go
+++ b/pkg/core/workflow_execute.go
@@ -5,13 +5,12 @@ import (
 	"net/http/cookiejar"
 	"sync/atomic"
 
-	"github.com/remeh/sizedwaitgroup"
-
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/workflows"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n"
@@ -32,7 +31,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 	if templateThreads == 1 {
 		templateThreads++
 	}
-	swg := sizedwaitgroup.New(templateThreads)
+	swg, _ := syncutil.New(syncutil.WithSize(templateThreads))
 
 	for _, template := range w.Workflows {
 		swg.Add()
@@ -40,7 +39,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 		func(template *workflows.WorkflowTemplate) {
 			defer swg.Done()
 
-			if err := e.runWorkflowStep(template, ctx, results, &swg, w); err != nil {
+			if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil {
 				gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
 			}
 		}(template)
@@ -51,7 +50,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 
 // runWorkflowStep runs a workflow step for the workflow. It executes the workflow
 // in a recursive manner running all subtemplates and matchers.
-func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error {
+func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan.ScanContext, results *atomic.Bool, swg *syncutil.AdaptiveWaitGroup, w *workflows.Workflow) error {
 	var firstMatched bool
 	var err error
 	var mainErr error
diff --git a/pkg/core/workpool.go b/pkg/core/workpool.go
index 0711759582..cd17a0b367 100644
--- a/pkg/core/workpool.go
+++ b/pkg/core/workpool.go
@@ -1,9 +1,8 @@
 package core
 
 import (
-	"github.com/remeh/sizedwaitgroup"
-
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // WorkPool implements an execution pool for executing different
@@ -12,8 +11,8 @@ import (
 // It also allows Configuration of such requirements. This is used
 // for per-module like separate headless concurrency etc.
 type WorkPool struct {
-	Headless *sizedwaitgroup.SizedWaitGroup
-	Default  *sizedwaitgroup.SizedWaitGroup
+	Headless *syncutil.AdaptiveWaitGroup
+	Default  *syncutil.AdaptiveWaitGroup
 	config   WorkPoolConfig
 }
 
@@ -31,13 +30,13 @@ type WorkPoolConfig struct {
 
 // NewWorkPool returns a new WorkPool instance
 func NewWorkPool(config WorkPoolConfig) *WorkPool {
-	headlessWg := sizedwaitgroup.New(config.HeadlessTypeConcurrency)
-	defaultWg := sizedwaitgroup.New(config.TypeConcurrency)
+	headlessWg, _ := syncutil.New(syncutil.WithSize(config.HeadlessTypeConcurrency))
+	defaultWg, _ := syncutil.New(syncutil.WithSize(config.TypeConcurrency))
 
 	return &WorkPool{
 		config:   config,
-		Headless: &headlessWg,
-		Default:  &defaultWg,
+		Headless: headlessWg,
+		Default:  defaultWg,
 	}
 }
 
@@ -47,19 +46,14 @@ func (w *WorkPool) Wait() {
 	w.Headless.Wait()
 }
 
-// InputWorkPool is a work pool per-input
-type InputWorkPool struct {
-	WaitGroup *sizedwaitgroup.SizedWaitGroup
-}
-
 // InputPool returns a work pool for an input type
-func (w *WorkPool) InputPool(templateType types.ProtocolType) *InputWorkPool {
+func (w *WorkPool) InputPool(templateType types.ProtocolType) *syncutil.AdaptiveWaitGroup {
 	var count int
 	if templateType == types.HeadlessProtocol {
 		count = w.config.HeadlessInputConcurrency
 	} else {
 		count = w.config.InputConcurrency
 	}
-	swg := sizedwaitgroup.New(count)
-	return &InputWorkPool{WaitGroup: &swg}
+	swg, _ := syncutil.New(syncutil.WithSize(count))
+	return swg
 }
diff --git a/pkg/js/compiler/non-pool.go b/pkg/js/compiler/non-pool.go
index 8057c49608..218b89b82a 100644
--- a/pkg/js/compiler/non-pool.go
+++ b/pkg/js/compiler/non-pool.go
@@ -4,13 +4,13 @@ import (
 	"sync"
 
 	"github.com/dop251/goja"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var (
-	ephemeraljsc    = sizedwaitgroup.New(NonPoolingVMConcurrency)
+	ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
 	lazyFixedSgInit = sync.OnceFunc(func() {
-		ephemeraljsc = sizedwaitgroup.New(NonPoolingVMConcurrency)
+		ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
 	})
 )
 
diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go
index 6dba600f70..fc0b61639b 100644
--- a/pkg/js/compiler/pool.go
+++ b/pkg/js/compiler/pool.go
@@ -36,7 +36,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/goconsole"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	stringsutil "github.com/projectdiscovery/utils/strings"
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 const (
@@ -51,9 +51,9 @@ var (
 		// autoregister console node module with default printer it uses gologger backend
 		require.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(goconsole.NewGoConsolePrinter()))
 	})
-	pooljsc    sizedwaitgroup.SizedWaitGroup
+	pooljsc    *syncutil.AdaptiveWaitGroup
 	lazySgInit = sync.OnceFunc(func() {
-		pooljsc = sizedwaitgroup.New(PoolingJsVmConcurrency)
+		pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency))
 	})
 )
 
diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go
index 8193947190..7119377b01 100644
--- a/pkg/protocols/common/automaticscan/automaticscan.go
+++ b/pkg/protocols/common/automaticscan/automaticscan.go
@@ -30,8 +30,8 @@ import (
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	syncutil "github.com/projectdiscovery/utils/sync"
 	wappalyzer "github.com/projectdiscovery/wappalyzergo"
-	"github.com/remeh/sizedwaitgroup"
 	"gopkg.in/yaml.v2"
 )
 
@@ -128,7 +128,10 @@ func (s *Service) Close() bool {
 func (s *Service) Execute() error {
 	gologger.Info().Msgf("Executing Automatic scan on %d target[s]", s.target.Count())
 	// setup host concurrency
-	sg := sizedwaitgroup.New(s.opts.Options.BulkSize)
+	sg, err := syncutil.New(syncutil.WithSize(s.opts.Options.BulkSize))
+	if err != nil {
+		return err
+	}
 	s.target.Iterate(func(value *contextargs.MetaInput) bool {
 		sg.Add()
 		go func(input *contextargs.MetaInput) {
@@ -246,7 +249,7 @@ func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) (
 	// execute tech detection templates on target
 	tags := map[string]struct{}{}
 	m := &sync.Mutex{}
-	sg := sizedwaitgroup.New(s.opts.Options.TemplateThreads)
+	sg, _ := syncutil.New(syncutil.WithSize(s.opts.Options.TemplateThreads))
 	counter := atomic.Uint32{}
 
 	for _, t := range s.techTemplates {
diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index 280e8161fe..ba9dd66621 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -9,7 +9,6 @@ import (
 
 	"github.com/miekg/dns"
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/multierr"
 	"golang.org/x/exp/maps"
 
@@ -27,6 +26,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/projectdiscovery/retryabledns"
 	iputil "github.com/projectdiscovery/utils/ip"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -64,7 +64,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
-		swg := sizedwaitgroup.New(request.Threads)
+		swg, err := syncutil.New(syncutil.WithSize(request.Threads))
+		if err != nil {
+			return err
+		}
 		var multiErr error
 		m := &sync.Mutex{}
 
diff --git a/pkg/protocols/file/request.go b/pkg/protocols/file/request.go
index eb73544f9a..f13f08d1ab 100644
--- a/pkg/protocols/file/request.go
+++ b/pkg/protocols/file/request.go
@@ -11,7 +11,6 @@ import (
 	"github.com/docker/go-units"
 	"github.com/mholt/archiver"
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
@@ -24,6 +23,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	sliceutil "github.com/projectdiscovery/utils/slice"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -47,8 +47,11 @@ var errEmptyResult = errors.New("Empty result")
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
-	err := request.getInputPaths(input.MetaInput.Input, func(filePath string) {
+	wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
+	if err != nil {
+		return err
+	}
+	err = request.getInputPaths(input.MetaInput.Input, func(filePath string) {
 		wg.Add()
 		func(filePath string) {
 			defer wg.Done()
diff --git a/pkg/protocols/headless/engine/page_actions.go b/pkg/protocols/headless/engine/page_actions.go
index 7338db44ab..348cab0aec 100644
--- a/pkg/protocols/headless/engine/page_actions.go
+++ b/pkg/protocols/headless/engine/page_actions.go
@@ -214,7 +214,7 @@ func geTimeParameter(p *Page, act *Action, parameterName string, defaultValue ti
 }
 
 // ActionAddHeader executes a AddHeader action.
-func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ActionAddHeader(act *Action, out map[string]string) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -225,7 +225,7 @@ func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review
 }
 
 // ActionSetHeader executes a SetHeader action.
-func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ActionSetHeader(act *Action, out map[string]string) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -236,7 +236,7 @@ func (p *Page) ActionSetHeader(act *Action, out map[string]string /*TODO review
 }
 
 // ActionDeleteHeader executes a DeleteHeader action.
-func (p *Page) ActionDeleteHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error {
 	in := p.getActionArgWithDefaultValues(act, "part")
 
 	args := make(map[string]string)
@@ -343,7 +343,7 @@ func (p *Page) RunScript(action *Action, out map[string]string) error {
 }
 
 // ClickElement executes click actions for an element.
-func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) ClickElement(act *Action, out map[string]string) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -358,12 +358,12 @@ func (p *Page) ClickElement(act *Action, out map[string]string /*TODO review unu
 }
 
 // KeyboardAction executes a keyboard action on the page.
-func (p *Page) KeyboardAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) KeyboardAction(act *Action, out map[string]string) error {
 	return p.page.Keyboard.Type([]input.Key(p.getActionArgWithDefaultValues(act, "keys"))...)
 }
 
 // RightClickElement executes right click actions for an element.
-func (p *Page) RightClickElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) RightClickElement(act *Action, out map[string]string) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -441,7 +441,7 @@ func (p *Page) Screenshot(act *Action, out map[string]string) error {
 }
 
 // InputElement executes input element actions for an element.
-func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) InputElement(act *Action, out map[string]string) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -460,7 +460,7 @@ func (p *Page) InputElement(act *Action, out map[string]string /*TODO review unu
 }
 
 // TimeInputElement executes time input on an element
-func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) TimeInputElement(act *Action, out map[string]string) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -483,7 +483,7 @@ func (p *Page) TimeInputElement(act *Action, out map[string]string /*TODO review
 }
 
 // SelectInputElement executes select input statement action on a element
-func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) SelectInputElement(act *Action, out map[string]string) error {
 	value := p.getActionArgWithDefaultValues(act, "value")
 	if value == "" {
 		return errinvalidArguments
@@ -508,7 +508,7 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO revi
 }
 
 // WaitLoad waits for the page to load
-func (p *Page) WaitLoad(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) WaitLoad(act *Action, out map[string]string) error {
 	p.page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)()
 
 	// Wait for the window.onload event and also wait for the network requests
@@ -538,7 +538,7 @@ func (p *Page) GetResource(act *Action, out map[string]string) error {
 }
 
 // FilesInput acts with a file input element on page
-func (p *Page) FilesInput(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) FilesInput(act *Action, out map[string]string) error {
 	element, err := p.pageElementBy(act.Data)
 	if err != nil {
 		return errors.Wrap(err, errCouldNotGetElement)
@@ -589,7 +589,7 @@ func (p *Page) ExtractElement(act *Action, out map[string]string) error {
 }
 
 // WaitEvent waits for an event to happen on the page.
-func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused parameter*/) (func() error, error) {
+func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, error) {
 	event := p.getActionArgWithDefaultValues(act, "event")
 	if event == "" {
 		return nil, errors.New("event not recognized")
@@ -661,14 +661,14 @@ func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) {
 }
 
 // DebugAction enables debug action on a page.
-func (p *Page) DebugAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) DebugAction(act *Action, out map[string]string) error {
 	p.instance.browser.engine.SlowMotion(5 * time.Second)
 	p.instance.browser.engine.Trace(true)
 	return nil
 }
 
 // SleepAction sleeps on the page for a specified duration
-func (p *Page) SleepAction(act *Action, out map[string]string /*TODO review unused parameter*/) error {
+func (p *Page) SleepAction(act *Action, out map[string]string) error {
 	seconds := act.Data["duration"]
 	if seconds == "" {
 		seconds = "5"
diff --git a/pkg/protocols/http/httputils/spm.go b/pkg/protocols/http/httputils/spm.go
index ccaa9a85c9..bca6c2ee59 100644
--- a/pkg/protocols/http/httputils/spm.go
+++ b/pkg/protocols/http/httputils/spm.go
@@ -4,7 +4,7 @@ import (
 	"context"
 	"sync"
 
-	"github.com/remeh/sizedwaitgroup"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 // WorkPoolType is the type of work pool to use
@@ -26,7 +26,7 @@ type StopAtFirstMatchHandler[T any] struct {
 
 	// work pool and its type
 	poolType WorkPoolType
-	sgPool   sizedwaitgroup.SizedWaitGroup
+	sgPool   *syncutil.AdaptiveWaitGroup
 	wgPool   *sync.WaitGroup
 
 	// internal / unexported
@@ -40,10 +40,13 @@ type StopAtFirstMatchHandler[T any] struct {
 // NewBlockingSPMHandler creates a new stop at first match handler
 func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
+
+	awg, _ := syncutil.New(syncutil.WithSize(size))
+
 	s := &StopAtFirstMatchHandler[T]{
 		ResultChan:  make(chan T, 1),
 		poolType:    Blocking,
-		sgPool:      sizedwaitgroup.New(size),
+		sgPool:      awg,
 		internalWg:  &sync.WaitGroup{},
 		ctx:         ctx1,
 		cancel:      cancel,
diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go
index fc32486937..48a1be485d 100644
--- a/pkg/protocols/javascript/js.go
+++ b/pkg/protocols/javascript/js.go
@@ -32,8 +32,8 @@ import (
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	syncutil "github.com/projectdiscovery/utils/sync"
 	urlutil "github.com/projectdiscovery/utils/url"
-	"github.com/remeh/sizedwaitgroup"
 )
 
 // Request is a request for the javascript protocol
@@ -404,7 +404,8 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 	requestOptions := request.options
 	gotmatches := &atomic.Bool{}
 
-	sg := sizedwaitgroup.New(threads)
+	sg, _ := syncutil.New(syncutil.WithSize(threads))
+
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		for {
diff --git a/pkg/protocols/network/networkclientpool/clientpool.go b/pkg/protocols/network/networkclientpool/clientpool.go
index 1a933413e2..a67cee2967 100644
--- a/pkg/protocols/network/networkclientpool/clientpool.go
+++ b/pkg/protocols/network/networkclientpool/clientpool.go
@@ -11,7 +11,7 @@ var (
 )
 
 // Init initializes the clientpool implementation
-func Init(options *types.Options /*TODO review unused parameter*/) error {
+func Init(options *types.Options) error {
 	// Don't create clients if already created in the past.
 	if normalClient != nil {
 		return nil
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 01991c0459..ef0ff01c7f 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/multierr"
 	"golang.org/x/exp/maps"
 
@@ -34,6 +33,7 @@ import (
 	errorutil "github.com/projectdiscovery/utils/errors"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	"github.com/projectdiscovery/utils/reader"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var (
@@ -178,7 +178,10 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 		iterator := request.generator.NewIterator()
 		var multiErr error
 		m := &sync.Mutex{}
-		swg := sizedwaitgroup.New(request.Threads)
+		swg, err := syncutil.New(syncutil.WithSize(request.Threads))
+		if err != nil {
+			return err
+		}
 
 		for {
 			value, ok := iterator.Value()
diff --git a/pkg/protocols/offlinehttp/request.go b/pkg/protocols/offlinehttp/request.go
index 7c64859e96..4a440c167f 100644
--- a/pkg/protocols/offlinehttp/request.go
+++ b/pkg/protocols/offlinehttp/request.go
@@ -6,7 +6,6 @@ import (
 	"os"
 
 	"github.com/pkg/errors"
-	"github.com/remeh/sizedwaitgroup"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -17,6 +16,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/utils/conversion"
+	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 var _ protocols.Request = &Request{}
@@ -29,10 +29,13 @@ func (request *Request) Type() templateTypes.ProtocolType {
 }
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
-func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
+func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+	wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))
+	if err != nil {
+		return err
+	}
 
-	err := request.getInputPaths(input.MetaInput.Input, func(data string) {
+	err = request.getInputPaths(input.MetaInput.Input, func(data string) {
 		wg.Add()
 
 		go func(data string) {
diff --git a/pkg/tmplexec/flow/vm.go b/pkg/tmplexec/flow/vm.go
index 2e22bd8e12..f1f7dbb84e 100644
--- a/pkg/tmplexec/flow/vm.go
+++ b/pkg/tmplexec/flow/vm.go
@@ -1,6 +1,7 @@
 package flow
 
 import (
+	"context"
 	"reflect"
 	"sync"
 
@@ -12,45 +13,43 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/remeh/sizedwaitgroup"
+	"github.com/projectdiscovery/utils/sync/sizedpool"
 )
 
-type jsWaitGroup struct {
-	sync.Once
-	sg sizedwaitgroup.SizedWaitGroup
+var jsOnce sync.Once
+
+// js runtime pool using sync.Pool
+var gojapool = &sync.Pool{
+	New: func() interface{} {
+		runtime := protocolstate.NewJSRuntime()
+		registerBuiltins(runtime)
+		return runtime
+	},
 }
 
-var jsPool = &jsWaitGroup{}
+var sizedgojapool *sizedpool.SizedPool[*goja.Runtime]
 
 // GetJSRuntime returns a new JS runtime from pool
 func GetJSRuntime(opts *types.Options) *goja.Runtime {
-	jsPool.Do(func() {
+	jsOnce.Do(func() {
 		if opts.JsConcurrency < 100 {
 			opts.JsConcurrency = 100
 		}
-		jsPool.sg = sizedwaitgroup.New(opts.JsConcurrency)
+		sizedgojapool, _ = sizedpool.New[*goja.Runtime](
+			sizedpool.WithPool[*goja.Runtime](gojapool),
+			sizedpool.WithSize[*goja.Runtime](int64(opts.JsConcurrency)),
+		)
 	})
-	jsPool.sg.Add()
-	return gojapool.Get().(*goja.Runtime)
+	runtime, _ := sizedgojapool.Get(context.TODO())
+	return runtime
 }
 
 // PutJSRuntime returns a JS runtime to pool
 func PutJSRuntime(runtime *goja.Runtime) {
-	defer jsPool.sg.Done()
-	gojapool.Put(runtime)
-}
-
-// js runtime pool using sync.Pool
-var gojapool = &sync.Pool{
-	New: func() interface{} {
-		runtime := protocolstate.NewJSRuntime()
-		registerBuiltins(runtime)
-		return runtime
-	},
+	sizedgojapool.Put(runtime)
 }
 
 func registerBuiltins(runtime *goja.Runtime) {
-
 	_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
 		Name:        "log",
 		Description: "Logs a given object/message to stdout (only for debugging purposes)",

From 774db61655894d6b5e4f76e98ff09997ceece263 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 3 Apr 2024 18:50:46 +0200
Subject: [PATCH 02/58] lightweight adaptivity on workpool

---
 internal/runner/inputs.go   | 14 ++++++--------
 pkg/core/engine.go          | 13 +++++++++----
 pkg/core/execute_options.go |  4 +++-
 pkg/core/executors.go       |  6 ++++++
 pkg/core/workpool.go        | 25 +++++++++++++++++++++++++
 pkg/js/compiler/pool.go     |  5 +++++
 6 files changed, 54 insertions(+), 13 deletions(-)

diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go
index 75a869912b..60aa03199f 100644
--- a/internal/runner/inputs.go
+++ b/internal/runner/inputs.go
@@ -15,12 +15,11 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
-const probeBulkSize = 50
+var GlobalProbeBulkSize = 50
 
 // initializeTemplatesHTTPInput initializes the http form of input
 // for any loaded http templates if input is in non-standard format.
 func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
-
 	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create temporary input file")
@@ -31,11 +30,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	gologger.Info().Msgf("Running httpx on input host")
 
-	var bulkSize = probeBulkSize
-	if r.options.BulkSize > probeBulkSize {
-		bulkSize = r.options.BulkSize
-	}
-
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
@@ -45,7 +39,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 
 	// Probe the non-standard URLs and store them in cache
-	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
+	swg, err := syncutil.New(syncutil.WithSize(GlobalProbeBulkSize))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create adaptive group")
 	}
@@ -55,6 +49,10 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 			return true
 		}
 
+		if swg.Size != GlobalProbeBulkSize {
+			swg.Resize(GlobalProbeBulkSize)
+		}
+
 		swg.Add()
 		go func(input *contextargs.MetaInput) {
 			defer swg.Done()
diff --git a/pkg/core/engine.go b/pkg/core/engine.go
index 93915bc2c6..4dfb8e0b93 100644
--- a/pkg/core/engine.go
+++ b/pkg/core/engine.go
@@ -30,14 +30,19 @@ func New(options *types.Options) *Engine {
 	return engine
 }
 
-// GetWorkPool returns a workpool from options
-func (e *Engine) GetWorkPool() *WorkPool {
-	return NewWorkPool(WorkPoolConfig{
+func (e *Engine) GetWorkPoolConfig() WorkPoolConfig {
+	config := WorkPoolConfig{
 		InputConcurrency:         e.options.BulkSize,
 		TypeConcurrency:          e.options.TemplateThreads,
 		HeadlessInputConcurrency: e.options.HeadlessBulkSize,
 		HeadlessTypeConcurrency:  e.options.HeadlessTemplateThreads,
-	})
+	}
+	return config
+}
+
+// GetWorkPool returns a workpool from options
+func (e *Engine) GetWorkPool() *WorkPool {
+	return NewWorkPool(e.GetWorkPoolConfig())
 }
 
 // SetExecuterOptions sets the executer options for the engine. This is required
diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go
index 580b8b0a9d..93f197fc2b 100644
--- a/pkg/core/execute_options.go
+++ b/pkg/core/execute_options.go
@@ -108,8 +108,10 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 	wp := e.GetWorkPool()
 
 	for _, template := range templatesList {
-		templateType := template.Type()
+		// resize check point - nop if there are no changes
+		wp.RefreshWithConfig(e.GetWorkPoolConfig())
 
+		templateType := template.Type()
 		var wg *syncutil.AdaptiveWaitGroup
 		if templateType == types.HeadlessProtocol {
 			wg = wp.Headless
diff --git a/pkg/core/executors.go b/pkg/core/executors.go
index ace7acb208..a6421cd90c 100644
--- a/pkg/core/executors.go
+++ b/pkg/core/executors.go
@@ -158,6 +158,9 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	wp := e.GetWorkPool()
 
 	for _, tpl := range alltemplates {
+		// resize check point - nop if there are no changes
+		wp.RefreshWithConfig(e.GetWorkPoolConfig())
+
 		var sg *syncutil.AdaptiveWaitGroup
 		if tpl.Type() == types.HeadlessProtocol {
 			sg = wp.Headless
@@ -213,6 +216,9 @@ func (e *ChildExecuter) Close() *atomic.Bool {
 func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
 	templateType := template.Type()
 
+	// resize check point - nop if there are no changes
+	e.e.workPool.RefreshWithConfig(e.e.GetWorkPoolConfig())
+
 	var wg *syncutil.AdaptiveWaitGroup
 	if templateType == types.HeadlessProtocol {
 		wg = e.e.workPool.Headless
diff --git a/pkg/core/workpool.go b/pkg/core/workpool.go
index cd17a0b367..810f99392d 100644
--- a/pkg/core/workpool.go
+++ b/pkg/core/workpool.go
@@ -57,3 +57,28 @@ func (w *WorkPool) InputPool(templateType types.ProtocolType) *syncutil.Adaptive
 	swg, _ := syncutil.New(syncutil.WithSize(count))
 	return swg
 }
+
+func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) {
+	if w.config.TypeConcurrency != config.TypeConcurrency {
+		w.config.TypeConcurrency = config.TypeConcurrency
+	}
+	if w.config.HeadlessTypeConcurrency != config.HeadlessTypeConcurrency {
+		w.config.HeadlessTypeConcurrency = config.HeadlessTypeConcurrency
+	}
+	if w.config.InputConcurrency != config.InputConcurrency {
+		w.config.InputConcurrency = config.InputConcurrency
+	}
+	if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency {
+		w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency
+	}
+	w.Refresh()
+}
+
+func (w *WorkPool) Refresh() {
+	if w.Default.Size != w.config.TypeConcurrency {
+		w.Default.Resize(w.config.TypeConcurrency)
+	}
+	if w.Headless.Size != w.config.HeadlessTypeConcurrency {
+		w.Headless.Resize(w.config.HeadlessTypeConcurrency)
+	}
+}
diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go
index fc0b61639b..2b3bc2179d 100644
--- a/pkg/js/compiler/pool.go
+++ b/pkg/js/compiler/pool.go
@@ -100,6 +100,11 @@ func executeWithRuntime(runtime *goja.Runtime, p *goja.Program, args *ExecuteArg
 // ExecuteProgram executes a compiled program with the default options.
 // it deligates if a particular program should run in a pooled or non-pooled runtime
 func ExecuteProgram(p *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (result goja.Value, err error) {
+	// resize check point
+	if pooljsc.Size != PoolingJsVmConcurrency {
+		pooljsc.Resize(PoolingJsVmConcurrency)
+	}
+
 	if opts.Source == nil {
 		// not-recommended anymore
 		return executeWithoutPooling(p, args, opts)

From 3c62b56fd9e267a594f7548c1674999768a66caa Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 3 Apr 2024 19:02:30 +0200
Subject: [PATCH 03/58] panic at the pool

---
 pkg/js/compiler/pool.go | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go
index 2b3bc2179d..678b9afb0a 100644
--- a/pkg/js/compiler/pool.go
+++ b/pkg/js/compiler/pool.go
@@ -55,6 +55,12 @@ var (
 	lazySgInit = sync.OnceFunc(func() {
 		pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency))
 	})
+	sgResizeCheck = func() {
+		// resize check point
+		if pooljsc.Size != PoolingJsVmConcurrency {
+			pooljsc.Resize(PoolingJsVmConcurrency)
+		}
+	}
 )
 
 var gojapool = &sync.Pool{
@@ -100,11 +106,6 @@ func executeWithRuntime(runtime *goja.Runtime, p *goja.Program, args *ExecuteArg
 // ExecuteProgram executes a compiled program with the default options.
 // it deligates if a particular program should run in a pooled or non-pooled runtime
 func ExecuteProgram(p *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (result goja.Value, err error) {
-	// resize check point
-	if pooljsc.Size != PoolingJsVmConcurrency {
-		pooljsc.Resize(PoolingJsVmConcurrency)
-	}
-
 	if opts.Source == nil {
 		// not-recommended anymore
 		return executeWithoutPooling(p, args, opts)
@@ -121,6 +122,8 @@ func executeWithPoolingProgram(p *goja.Program, args *ExecuteArgs, opts *Execute
 	// its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static
 	// unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones
 	lazySgInit()
+	sgResizeCheck()
+
 	pooljsc.Add()
 	defer pooljsc.Done()
 	runtime := gojapool.Get().(*goja.Runtime)

From 620287f76b451ca7ce8978d2eaad0d8fe7c1753c Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 3 Apr 2024 19:28:39 +0200
Subject: [PATCH 04/58] deprecating rlm

---
 cmd/nuclei/main.go         |  8 +++++---
 internal/runner/runner.go  | 14 ++++++++++----
 lib/multi.go               | 13 +++++++++----
 lib/sdk_private.go         | 13 +++++++++----
 pkg/testutils/testutils.go |  1 +
 pkg/types/types.go         |  4 ++++
 6 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index 3caec21cbd..b6ecd1e452 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -15,6 +15,7 @@ import (
 	"github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/env"
 	_ "github.com/projectdiscovery/utils/pprof"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
@@ -329,7 +330,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	flagSet.CreateGroup("rate-limit", "Rate-Limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
-		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
+		flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "maximum number of requests to send per second"),
+		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute (DEPRECATED)"),
 		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
 		flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"),
 		flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
@@ -597,10 +599,10 @@ Note: Make sure you have backup of your custom nuclei-templates before proceedin
 			gologger.Fatal().Msgf("could not read response: %s", err)
 		}
 		resp = strings.TrimSpace(resp)
-		if strings.EqualFold(resp, "y") || strings.EqualFold(resp, "yes") {
+		if stringsutil.EqualFoldAny(resp, "y", "yes") {
 			break
 		}
-		if strings.EqualFold(resp, "n") || strings.EqualFold(resp, "no") || resp == "" {
+		if stringsutil.EqualFoldAny(resp, "n", "no", "") {
 			fmt.Println("Exiting...")
 			os.Exit(0)
 		}
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index 3c6aaf0b53..23443647ea 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -314,11 +314,17 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	if options.RateLimitMinute > 0 {
-		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimitMinute), time.Minute)
-	} else if options.RateLimit > 0 {
-		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), time.Second)
-	} else {
+		gologger.Warning().Msgf("rate limit per minute is deprecated - use rate-limit-duration")
+		options.RateLimit = options.RateLimitMinute
+		options.RateLimitDuration = time.Minute
+	}
+	if options.RateLimit > 0 && options.RateLimitDuration == 0 {
+		options.RateLimitDuration = time.Second
+	}
+	if options.RateLimit == 0 && options.RateLimitDuration == 0 {
 		runner.rateLimiter = ratelimit.NewUnlimited(context.Background())
+	} else {
+		runner.rateLimiter = ratelimit.New(context.Background(), uint(options.RateLimit), options.RateLimitDuration)
 	}
 
 	if tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*"); err == nil {
diff --git a/lib/multi.go b/lib/multi.go
index 3a573b16a9..6fa791a2f0 100644
--- a/lib/multi.go
+++ b/lib/multi.go
@@ -42,11 +42,16 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt
 		Parser:          base.parser,
 	}
 	if opts.RateLimitMinute > 0 {
-		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimitMinute), time.Minute)
-	} else if opts.RateLimit > 0 {
-		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), time.Second)
-	} else {
+		opts.RateLimit = opts.RateLimitMinute
+		opts.RateLimitDuration = time.Minute
+	}
+	if opts.RateLimit > 0 && opts.RateLimitDuration == 0 {
+		opts.RateLimitDuration = time.Second
+	}
+	if opts.RateLimit == 0 && opts.RateLimitDuration == 0 {
 		u.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+	} else {
+		u.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(opts.RateLimit), opts.RateLimitDuration)
 	}
 	u.engine = core.New(opts)
 	u.engine.SetExecuterOptions(u.executerOpts)
diff --git a/lib/sdk_private.go b/lib/sdk_private.go
index bc66d53fd7..c76970c91d 100644
--- a/lib/sdk_private.go
+++ b/lib/sdk_private.go
@@ -192,11 +192,16 @@ func (e *NucleiEngine) init() error {
 
 	if e.executerOpts.RateLimiter == nil {
 		if e.opts.RateLimitMinute > 0 {
-			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimitMinute), time.Minute)
-		} else if e.opts.RateLimit > 0 {
-			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), time.Second)
-		} else {
+			e.opts.RateLimit = e.opts.RateLimitMinute
+			e.opts.RateLimitDuration = time.Minute
+		}
+		if e.opts.RateLimit > 0 && e.opts.RateLimitDuration == 0 {
+			e.opts.RateLimitDuration = time.Second
+		}
+		if e.opts.RateLimit == 0 && e.opts.RateLimitDuration == 0 {
 			e.executerOpts.RateLimiter = ratelimit.NewUnlimited(context.Background())
+		} else {
+			e.executerOpts.RateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration)
 		}
 	}
 
diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go
index 96d68e1d04..e59aa015e2 100644
--- a/pkg/testutils/testutils.go
+++ b/pkg/testutils/testutils.go
@@ -54,6 +54,7 @@ var DefaultOptions = &types.Options{
 	Timeout:                    5,
 	Retries:                    1,
 	RateLimit:                  150,
+	RateLimitDuration:          time.Second,
 	ProjectPath:                "",
 	Severities:                 severity.Severities{},
 	Targets:                    []string{},
diff --git a/pkg/types/types.go b/pkg/types/types.go
index ce78380901..34f3c3be10 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -132,7 +132,10 @@ type Options struct {
 	Retries int
 	// Rate-Limit is the maximum number of requests per specified target
 	RateLimit int
+	// Rate Limit Duration interval between burst resets
+	RateLimitDuration time.Duration
 	// Rate-Limit is the maximum number of requests per minute for specified target
+	// Deprecated: Use RateLimitDuration - automatically set Rate Limit Duration to 60 seconds
 	RateLimitMinute int
 	// PageTimeout is the maximum time to wait for a page in seconds
 	PageTimeout int
@@ -410,6 +413,7 @@ func (options *Options) HasClientCertificates() bool {
 func DefaultOptions() *Options {
 	return &Options{
 		RateLimit:               150,
+		RateLimitDuration:       time.Second,
 		BulkSize:                25,
 		TemplateThreads:         25,
 		HeadlessBulkSize:        10,

From a140a4194e2a521ecac5bffb03573e6a120c4ad6 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 3 Apr 2024 19:40:09 +0200
Subject: [PATCH 05/58] boh - placing resize in wrapped method

---
 pkg/protocols/dns/request.go       |  2 +-
 pkg/protocols/http/request.go      |  4 ++--
 pkg/protocols/http/request_fuzz.go |  2 +-
 pkg/protocols/protocols.go         | 11 +++++++++++
 4 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index ba9dd66621..8aa8c4a589 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -143,7 +143,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
 		}
 	}
 
-	request.options.RateLimiter.Take()
+	request.options.RateLimitTake()
 
 	// Send the request to the target servers
 	response, err := dnsClient.Do(compiledRequest)
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 2763c5c628..1b27aa7bf3 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -222,7 +222,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 				return
 			case spmHandler.ResultChan <- func() error {
 				// putting ratelimiter here prevents any unnecessary waiting if any
-				request.options.RateLimiter.Take()
+				request.options.RateLimitTake()
 				previous := make(map[string]interface{})
 				return request.executeRequest(input, httpRequest, previous, false, wrappedCallback, 0)
 			}():
@@ -366,7 +366,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 		executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) {
 			hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
 
-			request.options.RateLimiter.Take()
+			request.options.RateLimitTake()
 
 			ctx := request.newContext(input)
 			ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second)
diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go
index d6a29c181c..aed17f54e0 100644
--- a/pkg/protocols/http/request_fuzz.go
+++ b/pkg/protocols/http/request_fuzz.go
@@ -145,7 +145,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
 	if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {
 		return false
 	}
-	request.options.RateLimiter.Take()
+	request.options.RateLimitTake()
 	req := &generatedRequest{
 		request:        gr.Request,
 		dynamicValues:  gr.DynamicValues,
diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
index f9d9064200..5ac0a2f0b7 100644
--- a/pkg/protocols/protocols.go
+++ b/pkg/protocols/protocols.go
@@ -128,6 +128,17 @@ type ExecutorOptions struct {
 	ExportReqURLPattern bool
 }
 
+// todo: centralizing components is not feasible with current clogged architecture
+// a possible approach could be an internal event bus with pub-subs? This would be less invasive than
+// reworking dep injection from scratch
+func (eo *ExecutorOptions) RateLimitTake() {
+	if eo.RateLimiter.GetLimit() != uint(eo.Options.RateLimit) {
+		eo.RateLimiter.SetLimit(uint(eo.Options.RateLimit))
+		eo.RateLimiter.SetDuration(eo.Options.RateLimitDuration)
+	}
+	eo.RateLimiter.Take()
+}
+
 // GetThreadsForPayloadRequests returns the number of threads to use as default for
 // given max-request of payloads
 func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, currentThreads int) int {

From af7450737acfbf2ffd918d8a566596bcc9519250 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Wed, 3 Apr 2024 23:06:08 +0200
Subject: [PATCH 06/58] making payload concurrency dynamic via direct int
 change

---
 pkg/protocols/dns/request.go        |  9 +++++++++
 pkg/protocols/http/httputils/spm.go | 10 ++++++++++
 pkg/protocols/http/request.go       |  9 +++++++++
 pkg/protocols/javascript/js.go      |  9 +++++++++
 pkg/protocols/network/request.go    |  9 +++++++++
 pkg/protocols/protocols.go          | 14 ++------------
 6 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index 8aa8c4a589..d4e70e13e9 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -62,6 +62,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 	variablesMap := request.options.Variables.Evaluate(vars)
 	vars = generators.MergeMaps(vars, variablesMap, request.options.Constants)
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
+
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		swg, err := syncutil.New(syncutil.WithSize(request.Threads))
@@ -76,6 +79,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 			if !ok {
 				break
 			}
+
+			// resize check point - nop if there are no changes
+			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
+				swg.Resize(request.options.Options.PayloadConcurrency)
+			}
+
 			value = generators.MergeMaps(vars, value)
 			swg.Add()
 			go func(newVars map[string]interface{}) {
diff --git a/pkg/protocols/http/httputils/spm.go b/pkg/protocols/http/httputils/spm.go
index bca6c2ee59..52d13f06ff 100644
--- a/pkg/protocols/http/httputils/spm.go
+++ b/pkg/protocols/http/httputils/spm.go
@@ -143,6 +143,16 @@ func (h *StopAtFirstMatchHandler[T]) Release() {
 	}
 }
 
+func (h *StopAtFirstMatchHandler[T]) Resize(size int) {
+	if h.sgPool.Size != size {
+		h.sgPool.Resize(size)
+	}
+}
+
+func (h *StopAtFirstMatchHandler[T]) Size() int {
+	return h.sgPool.Size
+}
+
 // Wait waits for all work to be done
 func (h *StopAtFirstMatchHandler[T]) Wait() {
 	switch h.poolType {
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 1b27aa7bf3..5d64216f6a 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -165,6 +165,9 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 	// Workers that keeps enqueuing new requests
 	maxWorkers := request.Threads
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := maxWorkers == request.options.Options.PayloadConcurrency
+
 	if protocolstate.IsLowOnMemory() {
 		maxWorkers = protocolstate.GuardThreadsOrDefault(request.Threads)
 	}
@@ -198,6 +201,12 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		if !ok {
 			break
 		}
+
+		// resize check point - nop if there are no changes
+		if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
+			spmHandler.Resize(request.options.Options.PayloadConcurrency)
+		}
+
 		ctx := request.newContext(input)
 		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
 		if err != nil {
diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go
index 48a1be485d..efabd503c4 100644
--- a/pkg/protocols/javascript/js.go
+++ b/pkg/protocols/javascript/js.go
@@ -404,6 +404,9 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 	requestOptions := request.options
 	gotmatches := &atomic.Bool{}
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := threads == request.options.Options.PayloadConcurrency
+
 	sg, _ := syncutil.New(syncutil.WithSize(threads))
 
 	if request.generator != nil {
@@ -413,6 +416,12 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 			if !ok {
 				break
 			}
+
+			// resize check point - nop if there are no changes
+			if shouldFollowGlobal && sg.Size != request.options.Options.PayloadConcurrency {
+				sg.Resize(request.options.Options.PayloadConcurrency)
+			}
+
 			sg.Add()
 			go func() {
 				defer sg.Done()
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index ef0ff01c7f..146c657072 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -174,6 +174,9 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 		return err
 	}
 
+	// if request threads matches global payload concurrency we follow it
+	shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
+
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		var multiErr error
@@ -188,6 +191,12 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 			if !ok {
 				break
 			}
+
+			// resize check point - nop if there are no changes
+			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
+				swg.Resize(request.options.Options.PayloadConcurrency)
+			}
+
 			value = generators.MergeMaps(value, payloads)
 			swg.Add()
 			go func(vars map[string]interface{}) {
diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
index 5ac0a2f0b7..6328bc36a3 100644
--- a/pkg/protocols/protocols.go
+++ b/pkg/protocols/protocols.go
@@ -34,9 +34,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 )
 
-// Optional Callback to update Thread count in payloads across all requests
-type PayloadThreadSetterCallback func(opts *ExecutorOptions, totalRequests, currentThreads int) int
-
 var (
 	MaxTemplateFileSizeForEncoding = 1024 * 1024
 )
@@ -114,10 +111,6 @@ type ExecutorOptions struct {
 	// JsCompiler is abstracted javascript compiler which adds node modules and provides execution
 	// environment for javascript templates
 	JsCompiler *compiler.Compiler
-	// Optional Callback function to update Thread count in payloads across all protocols
-	// based on given logic. by default nuclei reverts to using value of `-c` when threads count
-	// is not specified or is 0 in template
-	OverrideThreadsCount PayloadThreadSetterCallback
 	// AuthProvider is a provider for auth strategies
 	AuthProvider authprovider.AuthProvider
 	//TemporaryDirectory is the directory to store temporary files
@@ -142,14 +135,11 @@ func (eo *ExecutorOptions) RateLimitTake() {
 // GetThreadsForPayloadRequests returns the number of threads to use as default for
 // given max-request of payloads
 func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, currentThreads int) int {
-	if e.OverrideThreadsCount != nil {
-		return e.OverrideThreadsCount(e, totalRequests, currentThreads)
-	}
 	if currentThreads > 0 {
 		return currentThreads
-	} else {
-		return e.Options.PayloadConcurrency
 	}
+
+	return e.Options.PayloadConcurrency
 }
 
 // CreateTemplateCtxStore creates template context store (which contains templateCtx for every scan)

From 99a018d12ad0e479abed4a1c663239db90dcfc0a Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 4 Apr 2024 20:19:46 +0200
Subject: [PATCH 07/58] adding speed change example

---
 examples/with_withspeed_control/main.go | 99 +++++++++++++++++++++++++
 go.mod                                  |  4 +-
 go.sum                                  |  8 +-
 lib/config.go                           | 42 +++++++++--
 lib/sdk.go                              |  8 ++
 5 files changed, 148 insertions(+), 13 deletions(-)
 create mode 100644 examples/with_withspeed_control/main.go

diff --git a/examples/with_withspeed_control/main.go b/examples/with_withspeed_control/main.go
new file mode 100644
index 0000000000..a61e820f26
--- /dev/null
+++ b/examples/with_withspeed_control/main.go
@@ -0,0 +1,99 @@
+package main
+
+import (
+	"sync"
+	"time"
+
+	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+)
+
+func main() {
+	ne, err := nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"header-command-injection"}}),
+		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
+		nuclei.WithGlobalRateLimit(1, time.Second),
+		nuclei.WithConcurrency(nuclei.Concurrency{
+			TemplateConcurrency:           1,
+			HostConcurrency:               1,
+			HeadlessHostConcurrency:       1,
+			HeadlessTemplateConcurrency:   1,
+			JavascriptTemplateConcurrency: 1,
+			TemplatePayloadConcurrency:    1,
+		}),
+	)
+	if err != nil {
+		panic(err)
+	}
+	// load targets and optionally probe non http/https targets
+	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
+
+	var wgtest sync.WaitGroup
+
+	// speed tests
+	// increase rate limit
+	wgtest.Add(1)
+	go func() {
+		defer wgtest.Done()
+		initialRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
+		if initialRate != 1 {
+			panic("wrong initial rate limit")
+		}
+		time.Sleep(10 * time.Second)
+		ne.Options().RateLimit = 5
+		time.Sleep(10 * time.Second)
+		finalRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
+		if finalRate != 5 {
+			panic("wrong final rate limit")
+		}
+	}()
+
+	// increase threads and bulk size
+	wgtest.Add(1)
+	go func() {
+		defer wgtest.Done()
+		initialTemplateThreads := ne.Options().TemplateThreads
+		initialBulkSize := ne.Options().BulkSize
+		if initialTemplateThreads != 1 || initialBulkSize != 1 {
+			panic("wrong initial standard concurrency")
+		}
+		time.Sleep(10 * time.Second)
+		ne.Options().TemplateThreads = 5
+		ne.Options().BulkSize = 25
+		time.Sleep(10 * time.Second)
+		// check new values via workpool
+		finalTemplateThreads := ne.Engine().WorkPool().Default.Size
+		finalBulkSize := ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size
+		if finalTemplateThreads != 5 && finalBulkSize != 25 {
+			panic("wrong final concurreny")
+		}
+	}()
+
+	// increase payload concurrency
+	wgtest.Add(1)
+	go func() {
+		defer wgtest.Done()
+		initialpayloadConcurrency := ne.Options().PayloadConcurrency
+		if initialpayloadConcurrency != 1 {
+			panic("wrong initial payload concurrency")
+		}
+		time.Sleep(10 * time.Second)
+		ne.Options().PayloadConcurrency = 5
+		time.Sleep(10 * time.Second)
+
+		// the ongoing and next payload iterations will retrieve parallelism from this function
+		// it should have the new set value, that will be cascade applied to all running adaptive wait groups
+		finalPayloadConcurrency := ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0)
+		if finalPayloadConcurrency != 5 {
+			panic("wrong initial payload concurrency")
+		}
+	}()
+
+	err = ne.ExecuteWithCallback(nil)
+	if err != nil {
+		panic(err)
+	}
+	defer ne.Close()
+
+	wgtest.Wait()
+}
diff --git a/go.mod b/go.mod
index 6a0e8bfccf..981d174836 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.64
+	github.com/projectdiscovery/fastdialer v0.0.65
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
 	github.com/projectdiscovery/rawhttp v0.1.41
@@ -94,7 +94,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.40
-	github.com/projectdiscovery/utils v0.0.87
+	github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0
 	github.com/projectdiscovery/wappalyzergo v0.0.116
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index 83551eb1c2..8557e5d49e 100644
--- a/go.sum
+++ b/go.sum
@@ -834,8 +834,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
 github.com/projectdiscovery/dsl v0.0.50 h1:4SuAwTS9l6o1tqlIC/79+EcUwTM6CjaU7MpY/nDlFaM=
 github.com/projectdiscovery/dsl v0.0.50/go.mod h1:6g740l4tH4d2j9UYtIchtxudb0Dphkq4o+VatpR4M6g=
-github.com/projectdiscovery/fastdialer v0.0.64 h1:xivkA4g14nwQElOVsxPkGMWsdcYPcp7DPhVjvI6yQkw=
-github.com/projectdiscovery/fastdialer v0.0.64/go.mod h1:S/7PAQRmVDYRaU7u4xXD0qA5a48NAZq2JcpcVoEVrlo=
+github.com/projectdiscovery/fastdialer v0.0.65 h1:msvKVJyILtP04CXSgSEWv4rUVsk0CCd3xhauo+H82IU=
+github.com/projectdiscovery/fastdialer v0.0.65/go.mod h1:wIE10NL7oa/zBCJfr1xAduv3q73aeuGbhfZ1Z8o4NUo=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -886,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
 github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.87 h1:9+RiTEhpUB/vk6XJUVpysNWJ2aCTD7WuyoyAcNnbIzk=
-github.com/projectdiscovery/utils v0.0.87/go.mod h1:jGK450sL9AVDTjaPwEs9za8NVeEC9xE97IWNoK138kI=
+github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0 h1:2ZR0yiN0cUm/qYEMq79MfcbgM374lJSdftheYhMFxNo=
+github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
 github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
diff --git a/lib/config.go b/lib/config.go
index 52a8e2328a..fd4e9c9a97 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -2,6 +2,7 @@ package nuclei
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"github.com/projectdiscovery/goflags"
@@ -120,12 +121,37 @@ type Concurrency struct {
 // WithConcurrency sets concurrency options
 func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
-		e.opts.TemplateThreads = opts.TemplateConcurrency
-		e.opts.BulkSize = opts.HostConcurrency
-		e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
-		e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
-		e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
-		e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
+		// minimum required is 1
+		if opts.TemplateConcurrency <= 0 {
+			return errors.New("template threads must be at least 1")
+		} else {
+			e.opts.TemplateThreads = opts.TemplateConcurrency
+		}
+		if opts.HostConcurrency <= 0 {
+			return errors.New("host concurrency must be at least 1")
+		} else {
+			e.opts.BulkSize = opts.HostConcurrency
+		}
+		if opts.HeadlessHostConcurrency <= 0 {
+			return errors.New("headless host concurrency must be at least 1")
+		} else {
+			e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
+		}
+		if opts.HeadlessTemplateConcurrency <= 0 {
+			return errors.New("headless template threads must be at least 1")
+		} else {
+			e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
+		}
+		if opts.JavascriptTemplateConcurrency <= 0 {
+			return errors.New("js must be at least 1")
+		} else {
+			e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
+		}
+		if opts.TemplatePayloadConcurrency <= 0 {
+			return errors.New("payload concurrency must be at least 1")
+		} else {
+			e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
+		}
 		return nil
 	}
 }
@@ -133,7 +159,9 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 // WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
 func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
-		e.rateLimiter = ratelimit.New(context.Background(), uint(maxTokens), duration)
+		e.opts.RateLimit = maxTokens
+		e.opts.RateLimitDuration = duration
+		e.rateLimiter = ratelimit.New(context.Background(), uint(e.opts.RateLimit), e.opts.RateLimitDuration)
 		return nil
 	}
 }
diff --git a/lib/sdk.go b/lib/sdk.go
index da0abae495..4bee921c31 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -215,6 +215,14 @@ func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.Result
 	return nil
 }
 
+func (e *NucleiEngine) Options() *types.Options {
+	return e.opts
+}
+
+func (e *NucleiEngine) Engine() *core.Engine {
+	return e.engine
+}
+
 // NewNucleiEngine creates a new nuclei engine instance
 func NewNucleiEngine(options ...NucleiSDKOptions) (*NucleiEngine, error) {
 	// default options

From 96d7d02701363821cd7805601ff18dc824e15abf Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 4 Apr 2024 20:22:28 +0200
Subject: [PATCH 08/58] adding speed example

---
 .github/workflows/build-test.yml                              | 4 ++++
 .../{with_withspeed_control => with_speed_control}/main.go    | 0
 2 files changed, 4 insertions(+)
 rename examples/{with_withspeed_control => with_speed_control}/main.go (100%)

diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index 7ebdaee1c0..aa0d8aecfe 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -66,3 +66,7 @@ jobs:
       - name: Example SDK Advanced
         run: go run .
         working-directory: examples/advanced/
+
+      - name: Example SDK with speed control
+        run: go run .
+        working-directory: examples/with_speed_control/
diff --git a/examples/with_withspeed_control/main.go b/examples/with_speed_control/main.go
similarity index 100%
rename from examples/with_withspeed_control/main.go
rename to examples/with_speed_control/main.go

From c4aa7dc1a20efa3ff02a036ce2acb0ae2274e8a7 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 4 Apr 2024 20:43:00 +0200
Subject: [PATCH 09/58] f tag

---
 examples/with_speed_control/main.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go
index a61e820f26..ed3592bade 100644
--- a/examples/with_speed_control/main.go
+++ b/examples/with_speed_control/main.go
@@ -10,7 +10,10 @@ import (
 
 func main() {
 	ne, err := nuclei.NewNucleiEngine(
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{IDs: []string{"header-command-injection"}}),
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{
+			IDs:         []string{"header-command-injection"},
+			IncludeTags: []string{"fuzz"},
+		}),
 		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
 		nuclei.WithGlobalRateLimit(1, time.Second),
 		nuclei.WithConcurrency(nuclei.Concurrency{

From aa06c9ef1798448da2670671b6c299b13e55064d Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 4 Apr 2024 21:21:06 +0200
Subject: [PATCH 10/58] speed up

---
 examples/with_speed_control/main.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go
index ed3592bade..8637209a6f 100644
--- a/examples/with_speed_control/main.go
+++ b/examples/with_speed_control/main.go
@@ -43,10 +43,10 @@ func main() {
 			panic("wrong initial rate limit")
 		}
 		time.Sleep(10 * time.Second)
-		ne.Options().RateLimit = 5
+		ne.Options().RateLimit = 1000
 		time.Sleep(10 * time.Second)
 		finalRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
-		if finalRate != 5 {
+		if finalRate != 1000 {
 			panic("wrong final rate limit")
 		}
 	}()
@@ -81,13 +81,13 @@ func main() {
 			panic("wrong initial payload concurrency")
 		}
 		time.Sleep(10 * time.Second)
-		ne.Options().PayloadConcurrency = 5
+		ne.Options().PayloadConcurrency = 100
 		time.Sleep(10 * time.Second)
 
 		// the ongoing and next payload iterations will retrieve parallelism from this function
 		// it should have the new set value, that will be cascade applied to all running adaptive wait groups
 		finalPayloadConcurrency := ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0)
-		if finalPayloadConcurrency != 5 {
+		if finalPayloadConcurrency != 100 {
 			panic("wrong initial payload concurrency")
 		}
 	}()

From d0a0c6d0c3ab9a1c170716a48ade3c6c649190aa Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Fri, 5 Apr 2024 15:45:21 +0200
Subject: [PATCH 11/58] internal sync fix + speed up

---
 examples/with_speed_control/main.go | 145 ++++++++++++++--------------
 go.mod                              |   8 +-
 go.sum                              |  15 +--
 pkg/core/engine.go                  |   2 +
 pkg/core/executors.go               |   2 +-
 5 files changed, 88 insertions(+), 84 deletions(-)

diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go
index 8637209a6f..36a26cdd1c 100644
--- a/examples/with_speed_control/main.go
+++ b/examples/with_speed_control/main.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"log"
 	"sync"
 	"time"
 
@@ -9,11 +10,32 @@ import (
 )
 
 func main() {
-	ne, err := nuclei.NewNucleiEngine(
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{
-			IDs:         []string{"header-command-injection"},
-			IncludeTags: []string{"fuzz"},
-		}),
+	ne, err := initializeNucleiEngine()
+	if err != nil {
+		panic(err)
+	}
+	defer ne.Close()
+
+	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+
+	go testRateLimit(&wg, ne)
+	go testThreadsAndBulkSize(&wg, ne)
+	go testPayloadConcurrency(&wg, ne)
+
+	err = ne.ExecuteWithCallback(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	wg.Wait()
+}
+
+func initializeNucleiEngine() (*nuclei.NucleiEngine, error) {
+	return nuclei.NewNucleiEngine(
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "http"}),
 		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
 		nuclei.WithGlobalRateLimit(1, time.Second),
 		nuclei.WithConcurrency(nuclei.Concurrency{
@@ -25,78 +47,57 @@ func main() {
 			TemplatePayloadConcurrency:    1,
 		}),
 	)
-	if err != nil {
-		panic(err)
-	}
-	// load targets and optionally probe non http/https targets
-	ne.LoadTargets([]string{"http://honey.scanme.sh"}, false)
-
-	var wgtest sync.WaitGroup
+}
 
-	// speed tests
-	// increase rate limit
-	wgtest.Add(1)
-	go func() {
-		defer wgtest.Done()
-		initialRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
-		if initialRate != 1 {
-			panic("wrong initial rate limit")
-		}
-		time.Sleep(10 * time.Second)
-		ne.Options().RateLimit = 1000
-		time.Sleep(10 * time.Second)
-		finalRate := ne.GetExecuterOptions().RateLimiter.GetLimit()
-		if finalRate != 1000 {
-			panic("wrong final rate limit")
-		}
-	}()
+func testRateLimit(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	verifyRateLimit(ne, 1, 5000)
+}
 
-	// increase threads and bulk size
-	wgtest.Add(1)
-	go func() {
-		defer wgtest.Done()
-		initialTemplateThreads := ne.Options().TemplateThreads
-		initialBulkSize := ne.Options().BulkSize
-		if initialTemplateThreads != 1 || initialBulkSize != 1 {
-			panic("wrong initial standard concurrency")
-		}
-		time.Sleep(10 * time.Second)
-		ne.Options().TemplateThreads = 5
-		ne.Options().BulkSize = 25
-		time.Sleep(10 * time.Second)
-		// check new values via workpool
-		finalTemplateThreads := ne.Engine().WorkPool().Default.Size
-		finalBulkSize := ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size
-		if finalTemplateThreads != 5 && finalBulkSize != 25 {
-			panic("wrong final concurreny")
-		}
-	}()
+func testThreadsAndBulkSize(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	initialTemplateThreads, initialBulkSize := 1, 1
+	verifyThreadsAndBulkSize(ne, initialTemplateThreads, initialBulkSize, 25, 25)
+}
 
-	// increase payload concurrency
-	wgtest.Add(1)
-	go func() {
-		defer wgtest.Done()
-		initialpayloadConcurrency := ne.Options().PayloadConcurrency
-		if initialpayloadConcurrency != 1 {
-			panic("wrong initial payload concurrency")
-		}
-		time.Sleep(10 * time.Second)
-		ne.Options().PayloadConcurrency = 100
-		time.Sleep(10 * time.Second)
+func testPayloadConcurrency(wg *sync.WaitGroup, ne *nuclei.NucleiEngine) {
+	defer wg.Done()
+	verifyPayloadConcurrency(ne, 1, 500)
+}
 
-		// the ongoing and next payload iterations will retrieve parallelism from this function
-		// it should have the new set value, that will be cascade applied to all running adaptive wait groups
-		finalPayloadConcurrency := ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0)
-		if finalPayloadConcurrency != 100 {
-			panic("wrong initial payload concurrency")
-		}
-	}()
+func verifyRateLimit(ne *nuclei.NucleiEngine, initialRate, finalRate int) {
+	if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(initialRate) {
+		panic("wrong initial rate limit")
+	}
+	time.Sleep(5 * time.Second)
+	ne.Options().RateLimit = finalRate
+	time.Sleep(20 * time.Second)
+	if ne.GetExecuterOptions().RateLimiter.GetLimit() != uint(finalRate) {
+		panic("wrong final rate limit")
+	}
+}
 
-	err = ne.ExecuteWithCallback(nil)
-	if err != nil {
-		panic(err)
+func verifyThreadsAndBulkSize(ne *nuclei.NucleiEngine, initialThreads, initialBulk, finalThreads, finalBulk int) {
+	if ne.Options().TemplateThreads != initialThreads || ne.Options().BulkSize != initialBulk {
+		panic("wrong initial standard concurrency")
 	}
-	defer ne.Close()
+	time.Sleep(5 * time.Second)
+	ne.Options().TemplateThreads = finalThreads
+	ne.Options().BulkSize = finalBulk
+	time.Sleep(20 * time.Second)
+	if ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size != finalBulk || ne.Engine().WorkPool().Default.Size != finalThreads {
+		log.Fatal("wrong final concurrency", ne.Engine().WorkPool().Default.Size, finalThreads, ne.Engine().GetWorkPool().InputPool(types.HTTPProtocol).Size, finalBulk)
+	}
+}
 
-	wgtest.Wait()
+func verifyPayloadConcurrency(ne *nuclei.NucleiEngine, initialPayloadConcurrency, finalPayloadConcurrency int) {
+	if ne.Options().PayloadConcurrency != initialPayloadConcurrency {
+		panic("wrong initial payload concurrency")
+	}
+	time.Sleep(5 * time.Second)
+	ne.Options().PayloadConcurrency = finalPayloadConcurrency
+	time.Sleep(20 * time.Second)
+	if ne.GetExecuterOptions().GetThreadsForNPayloadRequests(100, 0) != finalPayloadConcurrency {
+		panic("wrong final payload concurrency")
+	}
 }
diff --git a/go.mod b/go.mod
index 981d174836..a116f9611a 100644
--- a/go.mod
+++ b/go.mod
@@ -38,7 +38,7 @@ require (
 	github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
 	github.com/xanzy/go-gitlab v0.84.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/net v0.21.0
+	golang.org/x/net v0.24.0
 	golang.org/x/oauth2 v0.11.0
 	golang.org/x/text v0.14.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -100,7 +100,7 @@ require (
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
-	golang.org/x/term v0.17.0
+	golang.org/x/term v0.19.0
 	gopkg.in/yaml.v3 v3.0.1
 	moul.io/http2curl v1.0.0
 )
@@ -302,10 +302,10 @@ require (
 	go.etcd.io/bbolt v1.3.8 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	goftp.io/server/v2 v2.0.1 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.17.0
 	google.golang.org/appengine v1.6.7 // indirect
diff --git a/go.sum b/go.sum
index 8557e5d49e..a179db9b28 100644
--- a/go.sum
+++ b/go.sum
@@ -1186,8 +1186,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1280,8 +1280,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1382,8 +1382,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1395,8 +1396,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/pkg/core/engine.go b/pkg/core/engine.go
index 4dfb8e0b93..1b4155bbb5 100644
--- a/pkg/core/engine.go
+++ b/pkg/core/engine.go
@@ -58,5 +58,7 @@ func (e *Engine) ExecuterOptions() protocols.ExecutorOptions {
 
 // WorkPool returns the worker pool for the engine
 func (e *Engine) WorkPool() *WorkPool {
+	// resize check point - nop if there are no changes
+	e.workPool.RefreshWithConfig(e.GetWorkPoolConfig())
 	return e.workPool
 }
diff --git a/pkg/core/executors.go b/pkg/core/executors.go
index a6421cd90c..14fb75c631 100644
--- a/pkg/core/executors.go
+++ b/pkg/core/executors.go
@@ -217,7 +217,7 @@ func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs
 	templateType := template.Type()
 
 	// resize check point - nop if there are no changes
-	e.e.workPool.RefreshWithConfig(e.e.GetWorkPoolConfig())
+	e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig())
 
 	var wg *syncutil.AdaptiveWaitGroup
 	if templateType == types.HeadlessProtocol {

From f886fb2cf4a4e5e4500b3d336cac499e423e9994 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Fri, 5 Apr 2024 16:40:37 +0200
Subject: [PATCH 12/58] less templates in test

---
 examples/with_speed_control/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go
index 36a26cdd1c..88f7782cf1 100644
--- a/examples/with_speed_control/main.go
+++ b/examples/with_speed_control/main.go
@@ -35,7 +35,7 @@ func main() {
 
 func initializeNucleiEngine() (*nuclei.NucleiEngine, error) {
 	return nuclei.NewNucleiEngine(
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "http"}),
+		nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}),
 		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}),
 		nuclei.WithGlobalRateLimit(1, time.Second),
 		nuclei.WithConcurrency(nuclei.Concurrency{

From 582a85d9c060ed74e308750f198ccb2f12864629 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 9 Apr 2024 18:31:22 +0200
Subject: [PATCH 13/58] mimic follow behavior

---
 cmd/nuclei/main.go        |  1 +
 internal/runner/inputs.go | 11 +++++++++--
 lib/config.go             |  6 ++++++
 pkg/types/types.go        |  2 ++
 4 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index b6ecd1e452..a90c4ed52e 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -338,6 +338,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"),
 		flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"),
 		flagSet.IntVarP(&options.PayloadConcurrency, "payload-concurrency", "pc", 25, "max payload concurrency for each template"),
+		flagSet.IntVarP(&options.ProbeConcurrency, "probe-concurrency", "prc", 50, "http probe concurrency with httpx"),
 	)
 	flagSet.CreateGroup("optimization", "Optimizations",
 		flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"),
diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go
index 60aa03199f..e7a7c29fae 100644
--- a/internal/runner/inputs.go
+++ b/internal/runner/inputs.go
@@ -30,6 +30,11 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	gologger.Info().Msgf("Running httpx on input host")
 
+	var bulkSize = GlobalProbeBulkSize
+	if r.options.BulkSize > GlobalProbeBulkSize {
+		bulkSize = r.options.BulkSize
+	}
+
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
@@ -38,8 +43,10 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 		return nil, errors.Wrap(err, "could not create httpx client")
 	}
 
+	shouldFollowGlobalProbeBulkSize := bulkSize == GlobalProbeBulkSize
+
 	// Probe the non-standard URLs and store them in cache
-	swg, err := syncutil.New(syncutil.WithSize(GlobalProbeBulkSize))
+	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create adaptive group")
 	}
@@ -49,7 +56,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 			return true
 		}
 
-		if swg.Size != GlobalProbeBulkSize {
+		if shouldFollowGlobalProbeBulkSize && swg.Size != GlobalProbeBulkSize {
 			swg.Resize(GlobalProbeBulkSize)
 		}
 
diff --git a/lib/config.go b/lib/config.go
index fd4e9c9a97..5b112f8d25 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -116,6 +116,7 @@ type Concurrency struct {
 	HeadlessTemplateConcurrency   int // number of templates to run concurrently for headless templates (per host in host-spray mode)
 	JavascriptTemplateConcurrency int // number of templates to run concurrently for javascript templates (per host in host-spray mode)
 	TemplatePayloadConcurrency    int // max concurrent payloads to run for a template (a good default is 25)
+	ProbeConcurrency              int // max concurrent http probes to run (a good default is 50)
 }
 
 // WithConcurrency sets concurrency options
@@ -152,6 +153,11 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
 		} else {
 			e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
 		}
+		if opts.ProbeConcurrency <= 0 {
+			return errors.New("probe concurrency must be at least 1")
+		} else {
+			e.opts.ProbeConcurrency = opts.ProbeConcurrency
+		}
 		return nil
 	}
 }
diff --git a/pkg/types/types.go b/pkg/types/types.go
index 34f3c3be10..81a2731689 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -385,6 +385,8 @@ type Options struct {
 	SkipFormatValidation bool
 	// PayloadConcurrency is the number of concurrent payloads to run per template
 	PayloadConcurrency int
+	// ProbeConcurrency is the number of concurrent http probes to run with httpx
+	ProbeConcurrency int
 	// Dast only runs DAST templates
 	DAST bool
 }

From 70eb494a01ed17b665d6a4192e93174104365caa Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 9 Apr 2024 18:32:40 +0200
Subject: [PATCH 14/58] warning

---
 internal/runner/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index 23443647ea..76678bcb71 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -314,7 +314,7 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	if options.RateLimitMinute > 0 {
-		gologger.Warning().Msgf("rate limit per minute is deprecated - use rate-limit-duration")
+		gologger.Print().Msgf("[%v] %v", aurora.BrightYellow("WRN"), fmt.Sprintf("rate limit per minute is deprecated - use rate-limit-duration"))
 		options.RateLimit = options.RateLimitMinute
 		options.RateLimitDuration = time.Minute
 	}

From 6eeb98c71bddb4acfa39911fbc0844d475caeb47 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 9 Apr 2024 18:34:04 +0200
Subject: [PATCH 15/58] removing printf

---
 internal/runner/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index 76678bcb71..fa9b85bf05 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -314,7 +314,7 @@ func New(options *types.Options) (*Runner, error) {
 	}
 
 	if options.RateLimitMinute > 0 {
-		gologger.Print().Msgf("[%v] %v", aurora.BrightYellow("WRN"), fmt.Sprintf("rate limit per minute is deprecated - use rate-limit-duration"))
+		gologger.Print().Msgf("[%v] %v", aurora.BrightYellow("WRN"), "rate limit per minute is deprecated - use rate-limit-duration")
 		options.RateLimit = options.RateLimitMinute
 		options.RateLimitDuration = time.Minute
 	}

From 5fc08cec48cf27029598ed11679b2f9b0bfabf13 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 9 Apr 2024 18:51:25 +0200
Subject: [PATCH 16/58] updating example

---
 examples/with_speed_control/main.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/with_speed_control/main.go b/examples/with_speed_control/main.go
index 88f7782cf1..b56df967c5 100644
--- a/examples/with_speed_control/main.go
+++ b/examples/with_speed_control/main.go
@@ -45,6 +45,7 @@ func initializeNucleiEngine() (*nuclei.NucleiEngine, error) {
 			HeadlessTemplateConcurrency:   1,
 			JavascriptTemplateConcurrency: 1,
 			TemplatePayloadConcurrency:    1,
+			ProbeConcurrency:              1,
 		}),
 	)
 }

From 2ed33e472367c9ffe77062a1723d5b893020e0b8 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 11 Apr 2024 18:57:50 +0200
Subject: [PATCH 17/58] adding dns srv type

---
 pkg/protocols/dns/dns.go       | 2 ++
 pkg/protocols/dns/dns_types.go | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/pkg/protocols/dns/dns.go b/pkg/protocols/dns/dns.go
index 80c14d234f..0a1bbca6cb 100644
--- a/pkg/protocols/dns/dns.go
+++ b/pkg/protocols/dns/dns.go
@@ -275,6 +275,8 @@ func questionTypeToInt(questionType string) uint16 {
 		question = dns.TypeTLSA
 	case "ANY":
 		question = dns.TypeANY
+	case "SRV":
+		question = dns.TypeSRV
 	}
 	return question
 }
diff --git a/pkg/protocols/dns/dns_types.go b/pkg/protocols/dns/dns_types.go
index 034f69b326..8201df2229 100644
--- a/pkg/protocols/dns/dns_types.go
+++ b/pkg/protocols/dns/dns_types.go
@@ -37,6 +37,8 @@ const (
 	TLSA
 	// name:ANY
 	ANY
+	// name:SRV
+	SRV
 	limit
 )
 
@@ -54,6 +56,7 @@ var DNSRequestTypeMapping = map[DNSRequestType]string{
 	CAA:   "CAA",
 	TLSA:  "TLSA",
 	ANY:   "ANY",
+	SRV:   "SRV",
 }
 
 // GetSupportedDNSRequestTypes returns list of supported types

From 7b71886309f65880aaff351409409f678bea3caa Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 11 Apr 2024 19:10:31 +0200
Subject: [PATCH 18/58] Fixing internal resolver override

---
 pkg/protocols/common/protocolstate/state.go   | 2 +-
 pkg/protocols/dns/dnsclientpool/clientpool.go | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go
index e6bacb4915..08db87295b 100644
--- a/pkg/protocols/common/protocolstate/state.go
+++ b/pkg/protocols/common/protocolstate/state.go
@@ -124,7 +124,7 @@ func Init(options *types.Options) error {
 		opts.ResolversFile = true
 		opts.EnableFallback = true
 	}
-	if options.ResolversFile != "" {
+	if options.ResolversFile != "" || len(options.InternalResolversList) > 0 {
 		opts.BaseResolvers = options.InternalResolversList
 	}
 
diff --git a/pkg/protocols/dns/dnsclientpool/clientpool.go b/pkg/protocols/dns/dnsclientpool/clientpool.go
index fd632bd110..3f5126c824 100644
--- a/pkg/protocols/dns/dnsclientpool/clientpool.go
+++ b/pkg/protocols/dns/dnsclientpool/clientpool.go
@@ -34,7 +34,7 @@ func Init(options *types.Options) error {
 	clientPool = make(map[string]*retryabledns.Client)
 
 	resolvers := defaultResolvers
-	if options.ResolversFile != "" {
+	if options.ResolversFile != "" || len(options.InternalResolversList) > 0 {
 		resolvers = options.InternalResolversList
 	}
 	var err error
@@ -78,7 +78,7 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl
 	poolMutex.RUnlock()
 
 	resolvers := defaultResolvers
-	if options.ResolversFile != "" {
+	if options.ResolversFile != "" || len(options.InternalResolversList) > 0 {
 		resolvers = options.InternalResolversList
 	} else if len(configuration.Resolvers) > 0 {
 		resolvers = configuration.Resolvers

From f4394b8e1158ce4efae5ee05ee4bf1e2231e036f Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Thu, 11 Apr 2024 19:34:03 +0200
Subject: [PATCH 19/58] Adding networkpolicy to httpx probes

---
 internal/runner/inputs.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go
index e7a7c29fae..87fbb573e1 100644
--- a/internal/runner/inputs.go
+++ b/internal/runner/inputs.go
@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/httpx/common/httpx"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	syncutil "github.com/projectdiscovery/utils/sync"
@@ -38,6 +39,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
+	httpxOptions.NetworkPolicy = protocolstate.NetworkPolicy
 	httpxClient, err := httpx.New(&httpxOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create httpx client")

From 0807113e6c445171b62a0a18a53a0bd09f950af5 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Fri, 12 Apr 2024 00:02:43 +0200
Subject: [PATCH 20/58] adding more query types test

---
 cmd/integration-test/dns.go                    | 18 +++++++++++++++---
 .../protocols/dns/{basic.yaml => a.yaml}       |  6 +++---
 integration_tests/protocols/dns/aaaa.yaml      | 17 +++++++++++++++++
 3 files changed, 35 insertions(+), 6 deletions(-)
 rename integration_tests/protocols/dns/{basic.yaml => a.yaml} (69%)
 create mode 100644 integration_tests/protocols/dns/aaaa.yaml

diff --git a/cmd/integration-test/dns.go b/cmd/integration-test/dns.go
index 982aab98b0..9f1883716a 100644
--- a/cmd/integration-test/dns.go
+++ b/cmd/integration-test/dns.go
@@ -5,7 +5,8 @@ import (
 )
 
 var dnsTestCases = []TestCaseInfo{
-	{Path: "protocols/dns/basic.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/a.yaml", TestCase: &dnsA{}},
+	{Path: "protocols/dns/aaaa.yaml", TestCase: &dnsAAAA{}},
 	{Path: "protocols/dns/ptr.yaml", TestCase: &dnsPtr{}},
 	{Path: "protocols/dns/caa.yaml", TestCase: &dnsCAA{}},
 	{Path: "protocols/dns/tlsa.yaml", TestCase: &dnsTLSA{}},
@@ -14,10 +15,21 @@ var dnsTestCases = []TestCaseInfo{
 	{Path: "protocols/dns/dsl-matcher-variable.yaml", TestCase: &dnsDSLMatcherVariable{}},
 }
 
-type dnsBasic struct{}
+type dnsA struct{}
 
 // Execute executes a test case and returns an error if occurred
-func (h *dnsBasic) Execute(filePath string) error {
+func (h *dnsA) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}
+
+type dnsAAAA struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *dnsAAAA) Execute(filePath string) error {
 	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug)
 	if err != nil {
 		return err
diff --git a/integration_tests/protocols/dns/basic.yaml b/integration_tests/protocols/dns/a.yaml
similarity index 69%
rename from integration_tests/protocols/dns/basic.yaml
rename to integration_tests/protocols/dns/a.yaml
index b0dbdf3b02..5389746262 100644
--- a/integration_tests/protocols/dns/basic.yaml
+++ b/integration_tests/protocols/dns/a.yaml
@@ -1,7 +1,7 @@
-id: basic-dns-example
+id: basic-dns-a-example
 
 info:
-  name: Test DNS Template
+  name: Test DNS A Query Template
   author: pdteam
   severity: info
 
@@ -14,4 +14,4 @@ dns:
     matchers:
       - type: word
         words:
-          - "1.1.1.1"
\ No newline at end of file
+          - "1.1.1.1"
diff --git a/integration_tests/protocols/dns/aaaa.yaml b/integration_tests/protocols/dns/aaaa.yaml
new file mode 100644
index 0000000000..3df3293a9b
--- /dev/null
+++ b/integration_tests/protocols/dns/aaaa.yaml
@@ -0,0 +1,17 @@
+id: basic-dns-aaaa-example
+
+info:
+  name: Test DNS AAAA Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: AAAA
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        words:
+          - "2606:4700:4700::1001"

From 8f084eedd63b3892f90d8d58b9d1ba73074d9fda Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Fri, 12 Apr 2024 00:22:11 +0200
Subject: [PATCH 21/58] adding more tests

---
 cmd/integration-test/dns.go                   | 23 +++++++------------
 cmd/nuclei/srv.yaml                           | 18 +++++++++++++++
 .../protocols/code/pre-condition.yaml         |  2 +-
 .../protocols/code/py-env-var.yaml            |  2 +-
 integration_tests/protocols/code/py-file.yaml |  2 +-
 .../protocols/code/py-interactsh.yaml         |  2 +-
 .../protocols/code/py-snippet.yaml            |  2 +-
 integration_tests/protocols/dns/a.yaml        |  2 +-
 integration_tests/protocols/dns/aaaa.yaml     |  2 +-
 integration_tests/protocols/dns/cname.yaml    | 18 +++++++++++++++
 integration_tests/protocols/dns/ns.yaml       | 18 +++++++++++++++
 integration_tests/protocols/dns/srv.yaml      | 18 +++++++++++++++
 integration_tests/protocols/dns/txt.yaml      | 18 +++++++++++++++
 13 files changed, 105 insertions(+), 22 deletions(-)
 create mode 100644 cmd/nuclei/srv.yaml
 create mode 100644 integration_tests/protocols/dns/cname.yaml
 create mode 100644 integration_tests/protocols/dns/ns.yaml
 create mode 100644 integration_tests/protocols/dns/srv.yaml
 create mode 100644 integration_tests/protocols/dns/txt.yaml

diff --git a/cmd/integration-test/dns.go b/cmd/integration-test/dns.go
index 9f1883716a..82cf9c91c4 100644
--- a/cmd/integration-test/dns.go
+++ b/cmd/integration-test/dns.go
@@ -5,8 +5,12 @@ import (
 )
 
 var dnsTestCases = []TestCaseInfo{
-	{Path: "protocols/dns/a.yaml", TestCase: &dnsA{}},
-	{Path: "protocols/dns/aaaa.yaml", TestCase: &dnsAAAA{}},
+	{Path: "protocols/dns/a.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/aaaa.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/cname.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/srv.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/ns.yaml", TestCase: &dnsBasic{}},
+	{Path: "protocols/dns/txt.yaml", TestCase: &dnsBasic{}},
 	{Path: "protocols/dns/ptr.yaml", TestCase: &dnsPtr{}},
 	{Path: "protocols/dns/caa.yaml", TestCase: &dnsCAA{}},
 	{Path: "protocols/dns/tlsa.yaml", TestCase: &dnsTLSA{}},
@@ -15,21 +19,10 @@ var dnsTestCases = []TestCaseInfo{
 	{Path: "protocols/dns/dsl-matcher-variable.yaml", TestCase: &dnsDSLMatcherVariable{}},
 }
 
-type dnsA struct{}
+type dnsBasic struct{}
 
 // Execute executes a test case and returns an error if occurred
-func (h *dnsA) Execute(filePath string) error {
-	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug)
-	if err != nil {
-		return err
-	}
-	return expectResultsCount(results, 1)
-}
-
-type dnsAAAA struct{}
-
-// Execute executes a test case and returns an error if occurred
-func (h *dnsAAAA) Execute(filePath string) error {
+func (h *dnsBasic) Execute(filePath string) error {
 	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug)
 	if err != nil {
 		return err
diff --git a/cmd/nuclei/srv.yaml b/cmd/nuclei/srv.yaml
new file mode 100644
index 0000000000..198b397cea
--- /dev/null
+++ b/cmd/nuclei/srv.yaml
@@ -0,0 +1,18 @@
+id: basic-dns-a-example
+
+info:
+  name: Test DNS A Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: SRV
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        part: all
+        words:
+          - "SRV"
diff --git a/integration_tests/protocols/code/pre-condition.yaml b/integration_tests/protocols/code/pre-condition.yaml
index a61b4f90d8..1c44e9579a 100644
--- a/integration_tests/protocols/code/pre-condition.yaml
+++ b/integration_tests/protocols/code/pre-condition.yaml
@@ -23,4 +23,4 @@ code:
       - type: dsl
         dsl:
           - true
-# digest: 4a0a00473045022100c7215ce9f11e6a51c193bb54643a05cdd1cde18a3abb6c9983c5c7524d3ff03002203d93581c81d3ad5db463570cbbd2bdee529328d32a5b00e037610c211e448cef:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 490a004630440220192fb8f704b078c2885047b85ac1a0491be86485c033a976d201599683a35aab0220604b1c3781e9d97079d0e5c23c18e6a2d87493c8e2b930536e692ee7d06e9247:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml
index 4ccf3648bf..9ff947c208 100644
--- a/integration_tests/protocols/code/py-env-var.yaml
+++ b/integration_tests/protocols/code/py-env-var.yaml
@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input baz"
-# digest: 4a0a0047304502207e3a5eda5f3207c3c01c820562243281926c1215224a7c80ed7528559b9f52cb022100f6ef99bb45843f481705778630f2cfd8f4d1cc3acb96392ff016f22e06aa91af:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 4a0a00473045022033f72f1b9d5143f58a2dc79c2597000f34080251ac3702c36c3fad00917dfeeb0221009ba05c715c9e2e36dba471be6c0106a09ae3822d8a3e9e4bcf377e9f4a395a01:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml
index 9e0b041bcf..ad69371d05 100644
--- a/integration_tests/protocols/code/py-file.yaml
+++ b/integration_tests/protocols/code/py-file.yaml
@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4a0a004730450220069673af9bd6d6677f9529d06f5d8bd46d543089a4731ed18ee806761d75fd60022100913a3e27b0a5809baf710ba9585bf9fe729634c0e19e3e13eef70a6bd100df34:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 4a0a004730450220377128cb11d9f6f0fee1f4dbd841e46783de26e90a216fa55a7609ee2bc823c60221009166ee0f85e3a1811588ab19e73ea96ab3d582dc8180dbcbbad0ea9ab7e9025d:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml
index 24e4b06241..76d14efb2e 100644
--- a/integration_tests/protocols/code/py-interactsh.yaml
+++ b/integration_tests/protocols/code/py-interactsh.yaml
@@ -26,4 +26,4 @@ code:
         part: interactsh_protocol
         words:
           - "http"
-# digest: 490a00463044022003b8d069e3c84412729c43e33013a52ee04eabcf096d511979691d71d8e905f60220011f4475899abed4f86b4bd5e6c2423750759135206e4729826afe1ed8a44f4d:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 4b0a00483046022100d472d50bd83117d334f5217c7a40dcdf34138e90029eaace51697d902296bf37022100a393b49420a96f60d6d89b79b5135ee2233b2468d374851890eea114b08195d1:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml
index 287ca2c6eb..4837fa7e0e 100644
--- a/integration_tests/protocols/code/py-snippet.yaml
+++ b/integration_tests/protocols/code/py-snippet.yaml
@@ -21,4 +21,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4a0a00473045022100c291615cf2a8005450c17a6554e81a9cdab14743b299f0679c644247929198b502206fdacc8ab173bde2b4015340012637916bf2659f66f320fcc06b97ac639072a1:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
+# digest: 4b0a004830460221008886054bb5dd6345e434e30f31c8fddce3c484a4f33aa6321b5185675866029d022100d188a83d0fde029f8b586061c65ab72b43755c3fb10fdd59501bb9bbadbb1ff7:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/protocols/dns/a.yaml b/integration_tests/protocols/dns/a.yaml
index 5389746262..0e51245817 100644
--- a/integration_tests/protocols/dns/a.yaml
+++ b/integration_tests/protocols/dns/a.yaml
@@ -1,4 +1,4 @@
-id: basic-dns-a-example
+id: dns-a-query-example
 
 info:
   name: Test DNS A Query Template
diff --git a/integration_tests/protocols/dns/aaaa.yaml b/integration_tests/protocols/dns/aaaa.yaml
index 3df3293a9b..58a2e496ce 100644
--- a/integration_tests/protocols/dns/aaaa.yaml
+++ b/integration_tests/protocols/dns/aaaa.yaml
@@ -1,4 +1,4 @@
-id: basic-dns-aaaa-example
+id: dns-aaaa-query-example
 
 info:
   name: Test DNS AAAA Query Template
diff --git a/integration_tests/protocols/dns/cname.yaml b/integration_tests/protocols/dns/cname.yaml
new file mode 100644
index 0000000000..f4ddb8a2ed
--- /dev/null
+++ b/integration_tests/protocols/dns/cname.yaml
@@ -0,0 +1,18 @@
+id: dns-cname-query-example
+
+info:
+  name: Test DNS CNAME Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: CNAME
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        part: all
+        words:
+          - "CNAME"
diff --git a/integration_tests/protocols/dns/ns.yaml b/integration_tests/protocols/dns/ns.yaml
new file mode 100644
index 0000000000..9d40655743
--- /dev/null
+++ b/integration_tests/protocols/dns/ns.yaml
@@ -0,0 +1,18 @@
+id: dns-ns-query-example
+
+info:
+  name: Test DNS NS Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: NS
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        part: all
+        words:
+          - "NS"
diff --git a/integration_tests/protocols/dns/srv.yaml b/integration_tests/protocols/dns/srv.yaml
new file mode 100644
index 0000000000..2669333c5e
--- /dev/null
+++ b/integration_tests/protocols/dns/srv.yaml
@@ -0,0 +1,18 @@
+id: dns-a-query-example
+
+info:
+  name: Test DNS SRV Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: SRV
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        part: all
+        words:
+          - "SRV"
diff --git a/integration_tests/protocols/dns/txt.yaml b/integration_tests/protocols/dns/txt.yaml
new file mode 100644
index 0000000000..273a53ab7d
--- /dev/null
+++ b/integration_tests/protocols/dns/txt.yaml
@@ -0,0 +1,18 @@
+id: dns-txt-query-example
+
+info:
+  name: Test DNS TXT Query Template
+  author: pdteam
+  severity: info
+
+dns:
+  - name: "{{FQDN}}"
+    type: TXT
+    class: inet
+    recursion: true
+    retries: 3
+    matchers:
+      - type: word
+        part: all
+        words:
+          - "TXT"

From 38e185c410c7ed85bb44ccc467780808eafe97db Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Fri, 12 Apr 2024 00:32:06 +0200
Subject: [PATCH 22/58] simpler logic

---
 pkg/protocols/common/protocolstate/state.go   | 2 +-
 pkg/protocols/dns/dnsclientpool/clientpool.go | 4 ++--
 pkg/types/types.go                            | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go
index 08db87295b..3298c91366 100644
--- a/pkg/protocols/common/protocolstate/state.go
+++ b/pkg/protocols/common/protocolstate/state.go
@@ -124,7 +124,7 @@ func Init(options *types.Options) error {
 		opts.ResolversFile = true
 		opts.EnableFallback = true
 	}
-	if options.ResolversFile != "" || len(options.InternalResolversList) > 0 {
+	if len(options.InternalResolversList) > 0 {
 		opts.BaseResolvers = options.InternalResolversList
 	}
 
diff --git a/pkg/protocols/dns/dnsclientpool/clientpool.go b/pkg/protocols/dns/dnsclientpool/clientpool.go
index 3f5126c824..4f019808fe 100644
--- a/pkg/protocols/dns/dnsclientpool/clientpool.go
+++ b/pkg/protocols/dns/dnsclientpool/clientpool.go
@@ -34,7 +34,7 @@ func Init(options *types.Options) error {
 	clientPool = make(map[string]*retryabledns.Client)
 
 	resolvers := defaultResolvers
-	if options.ResolversFile != "" || len(options.InternalResolversList) > 0 {
+	if len(options.InternalResolversList) > 0 {
 		resolvers = options.InternalResolversList
 	}
 	var err error
@@ -78,7 +78,7 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl
 	poolMutex.RUnlock()
 
 	resolvers := defaultResolvers
-	if options.ResolversFile != "" || len(options.InternalResolversList) > 0 {
+	if len(options.InternalResolversList) > 0 {
 		resolvers = options.InternalResolversList
 	} else if len(configuration.Resolvers) > 0 {
 		resolvers = configuration.Resolvers
diff --git a/pkg/types/types.go b/pkg/types/types.go
index 81a2731689..c251dc0c85 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -66,8 +66,8 @@ type Options struct {
 	IncludeIds goflags.StringSlice
 	// ExcludeIds contains templates ids to not be executed
 	ExcludeIds goflags.StringSlice
-
-	InternalResolversList []string // normalized from resolvers flag as well as file provided.
+	// InternalResolversList is the list of internal resolvers to use
+	InternalResolversList []string
 	// ProjectPath allows nuclei to use a user defined project folder
 	ProjectPath string
 	// InteractshURL is the URL for the interactsh server.

From 0d5e26d7bdac2631c3f208765cd203e62097ce43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Levente=20Kov=C3=A1ts?=
 <22732484+tovask@users.noreply.github.com>
Date: Fri, 12 Apr 2024 00:50:11 +0200
Subject: [PATCH 23/58] run workflow subtemplates with new ScanContext (#5031)

fix projectdiscovery/nuclei#4933
---
 cmd/integration-test/workflow.go              | 26 +++++++++++++++++++
 .../workflow/complex-conditions.yaml          | 23 ++++++++++++++++
 integration_tests/workflow/match-3.yaml       | 16 ++++++++++++
 pkg/core/workflow_execute.go                  |  8 ++++--
 4 files changed, 71 insertions(+), 2 deletions(-)
 create mode 100644 integration_tests/workflow/complex-conditions.yaml
 create mode 100644 integration_tests/workflow/match-3.yaml

diff --git a/cmd/integration-test/workflow.go b/cmd/integration-test/workflow.go
index 3ffd28871e..8672283b6c 100644
--- a/cmd/integration-test/workflow.go
+++ b/cmd/integration-test/workflow.go
@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 
 	"github.com/julienschmidt/httprouter"
 
@@ -16,6 +17,7 @@ var workflowTestcases = []TestCaseInfo{
 	{Path: "workflow/condition-matched.yaml", TestCase: &workflowConditionMatched{}},
 	{Path: "workflow/condition-unmatched.yaml", TestCase: &workflowConditionUnmatch{}},
 	{Path: "workflow/matcher-name.yaml", TestCase: &workflowMatcherName{}},
+	{Path: "workflow/complex-conditions.yaml", TestCase: &workflowComplexConditions{}},
 	{Path: "workflow/http-value-share-workflow.yaml", TestCase: &workflowHttpKeyValueShare{}},
 	{Path: "workflow/dns-value-share-workflow.yaml", TestCase: &workflowDnsKeyValueShare{}},
 	{Path: "workflow/shared-cookie.yaml", TestCase: &workflowSharedCookies{}},
@@ -97,6 +99,30 @@ func (h *workflowMatcherName) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type workflowComplexConditions struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *workflowComplexConditions) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is test matcher text")
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	for _, result := range results {
+		if !strings.Contains(result, "test-matcher-3") {
+			return fmt.Errorf("incorrect result: the \"basic-get-third:test-matcher-3\" and only that should be matched!\nResults:\n\t%s", strings.Join(results, "\n\t"))
+		}
+	}
+	return expectResultsCount(results, 2)
+}
+
 type workflowHttpKeyValueShare struct{}
 
 // Execute executes a test case and returns an error if occurred
diff --git a/integration_tests/workflow/complex-conditions.yaml b/integration_tests/workflow/complex-conditions.yaml
new file mode 100644
index 0000000000..bd1e66be5d
--- /dev/null
+++ b/integration_tests/workflow/complex-conditions.yaml
@@ -0,0 +1,23 @@
+id: complex-conditions-workflow
+
+info:
+  name: Complex Conditions Workflow
+  author: tovask
+  severity: info
+  description: Workflow to test a complex scenario, e.g. race conditions when evaluating the results of the templates
+
+workflows:
+  - template: workflow/match-1.yaml
+    subtemplates:
+      - template: workflow/nomatch-1.yaml
+        subtemplates:
+          - template: workflow/match-2.yaml
+  - template: workflow/match-3.yaml
+  - template: workflow/match-2.yaml
+    matchers: 
+      - name: test-matcher
+        subtemplates:
+          - template: workflow/nomatch-1.yaml
+            subtemplates:
+              - template: workflow/match-1.yaml
+  - template: workflow/match-3.yaml
diff --git a/integration_tests/workflow/match-3.yaml b/integration_tests/workflow/match-3.yaml
new file mode 100644
index 0000000000..b2e23fb993
--- /dev/null
+++ b/integration_tests/workflow/match-3.yaml
@@ -0,0 +1,16 @@
+id: basic-get-third
+
+info:
+  name: Basic 3rd GET Request
+  author: tovask
+  severity: info
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        name: test-matcher-3
+        words:
+          - "This is test matcher text"
diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go
index 19d6f0d69d..c17a1af8d5 100644
--- a/pkg/core/workflow_execute.go
+++ b/pkg/core/workflow_execute.go
@@ -138,7 +138,9 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 						go func(subtemplate *workflows.WorkflowTemplate) {
 							defer swg.Done()
 
-							if err := e.runWorkflowStep(subtemplate, ctx, results, swg, w); err != nil {
+							// create a new context with the same input but with unset callbacks
+							subCtx := scan.NewScanContext(ctx.Input)
+							if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, w); err != nil {
 								gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err)
 							}
 						}(subtemplate)
@@ -162,7 +164,9 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 			swg.Add()
 
 			go func(template *workflows.WorkflowTemplate) {
-				if err := e.runWorkflowStep(template, ctx, results, swg, w); err != nil {
+				// create a new context with the same input but with unset callbacks
+				subCtx := scan.NewScanContext(ctx.Input)
+				if err := e.runWorkflowStep(template, subCtx, results, swg, w); err != nil {
 					gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
 				}
 				swg.Done()

From c145adb3bf724218d283a35d6ffe1820b367bfcf Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Apr 2024 05:35:43 +0000
Subject: [PATCH 24/58] chore(deps): bump github.com/projectdiscovery/goflags

Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.46 to 0.1.48.
- [Release notes](https://github.com/projectdiscovery/goflags/releases)
- [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.46...v0.1.48)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/goflags
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 2f239bc148..f7fb5beabf 100644
--- a/go.mod
+++ b/go.mod
@@ -81,7 +81,7 @@ require (
 	github.com/projectdiscovery/dsl v0.0.51
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
-	github.com/projectdiscovery/goflags v0.1.46
+	github.com/projectdiscovery/goflags v0.1.48
 	github.com/projectdiscovery/gologger v1.1.12
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.2
@@ -94,7 +94,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.40
-	github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0
+	github.com/projectdiscovery/utils v0.0.88
 	github.com/projectdiscovery/wappalyzergo v0.0.116
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index bf4e00c357..607f14df2f 100644
--- a/go.sum
+++ b/go.sum
@@ -842,8 +842,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm
 github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.46 h1:JlYvFxJcimKJGWYbygiFBN052MWrbls/kKiwOKpLzEE=
-github.com/projectdiscovery/goflags v0.1.46/go.mod h1:X7A6ELNgczyOyEy2gyNC/tJTuhtwQk6ZLyzsnDVlZkw=
+github.com/projectdiscovery/goflags v0.1.48 h1:iJgq+YuQOEm0V7BqlV/0wx7GaI2rmGiFzPkvL/4sCj0=
+github.com/projectdiscovery/goflags v0.1.48/go.mod h1:C9Qqyivehk5iYCf3VAC3+85y60Qbde1ZBMqe5wHhcVw=
 github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
 github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
@@ -886,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
 github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0 h1:2ZR0yiN0cUm/qYEMq79MfcbgM374lJSdftheYhMFxNo=
-github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
+github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs=
+github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
 github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

From c8824ac71a51414977cf98592008d033fccb1893 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Apr 2024 05:35:54 +0000
Subject: [PATCH 25/58] chore(deps): bump
 github.com/projectdiscovery/fastdialer

Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.66 to 0.0.67.
- [Release notes](https://github.com/projectdiscovery/fastdialer/releases)
- [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.66...v0.0.67)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/fastdialer
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 2f239bc148..9c3ea9e3c8 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.66
+	github.com/projectdiscovery/fastdialer v0.0.67
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
 	github.com/projectdiscovery/rawhttp v0.1.44
@@ -94,7 +94,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.40
-	github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0
+	github.com/projectdiscovery/utils v0.0.88
 	github.com/projectdiscovery/wappalyzergo v0.0.116
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index bf4e00c357..22e511931e 100644
--- a/go.sum
+++ b/go.sum
@@ -834,8 +834,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
 github.com/projectdiscovery/dsl v0.0.51 h1:7OQPumOrrUCFnCA7Y0nchhPvRo3IJGMIJ2Oy4DVTQsc=
 github.com/projectdiscovery/dsl v0.0.51/go.mod h1:GYhusn+T9EL7t+iJ8zN/GXlp8ohLGU+Yv/nevAPlJZg=
-github.com/projectdiscovery/fastdialer v0.0.66 h1:DRpmok9TArLyQKaSjRWSzikt2N2Qyzx/z0BmTmDyJvI=
-github.com/projectdiscovery/fastdialer v0.0.66/go.mod h1:7uPrwFsIBhtUBkXd72K4VSo9lvcwqOzOGXIZ9UZXFYw=
+github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E=
+github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -886,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
 github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0 h1:2ZR0yiN0cUm/qYEMq79MfcbgM374lJSdftheYhMFxNo=
-github.com/projectdiscovery/utils v0.0.88-0.20240404181359-663cfe2196d0/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
+github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs=
+github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
 github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

From 3b51a77bc7c2107efaf9fbc5f0c42406389afd58 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Apr 2024 06:00:23 +0000
Subject: [PATCH 26/58] chore(deps): bump github.com/projectdiscovery/rawhttp

Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.44 to 0.1.45.
- [Release notes](https://github.com/projectdiscovery/rawhttp/releases)
- [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.44...v0.1.45)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/rawhttp
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 9c3ea9e3c8..fd3fa51094 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/projectdiscovery/fastdialer v0.0.67
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
-	github.com/projectdiscovery/rawhttp v0.1.44
+	github.com/projectdiscovery/rawhttp v0.1.45
 	github.com/projectdiscovery/retryabledns v1.0.58
 	github.com/projectdiscovery/retryablehttp-go v1.0.55
 	github.com/projectdiscovery/yamldoc-go v1.0.4
diff --git a/go.sum b/go.sum
index 22e511931e..23c56081d5 100644
--- a/go.sum
+++ b/go.sum
@@ -868,8 +868,8 @@ github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7
 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
 github.com/projectdiscovery/ratelimit v0.0.35 h1:epEzFATOcXZ4tssV4Hax5Op9lrbUnQMEGMV5PoUpTKc=
 github.com/projectdiscovery/ratelimit v0.0.35/go.mod h1:mPqa8UpV5I7eAN5/ZcsjLiXMhjtVvZRrHtpBRsTPuyA=
-github.com/projectdiscovery/rawhttp v0.1.44 h1:mkXTTUR65TTNisQGpLo5y5PRYRgNwZLW15KZNhNpsO8=
-github.com/projectdiscovery/rawhttp v0.1.44/go.mod h1:jaldbYYP0QihgQKk6Ar9ym9NPLAz5QkXp5TPET0sjYM=
+github.com/projectdiscovery/rawhttp v0.1.45 h1:5jssUybhRz2ljI3bW4kUO9MTvZ37ArG8pISYtgqzwBg=
+github.com/projectdiscovery/rawhttp v0.1.45/go.mod h1:oLWuZkyKi4rddPoLEN3TeAqyLvFNfr3dxmzV2kwpPvs=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=

From f876e3290b371d1124a6f5e39392bcc3fdf798aa Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Apr 2024 06:09:00 +0000
Subject: [PATCH 27/58] chore(deps): bump github.com/projectdiscovery/dsl from
 0.0.51 to 0.0.52

Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.51 to 0.0.52.
- [Release notes](https://github.com/projectdiscovery/dsl/releases)
- [Commits](https://github.com/projectdiscovery/dsl/compare/v0.0.51...v0.0.52)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/dsl
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 4533ce936f..700e11e33c 100644
--- a/go.mod
+++ b/go.mod
@@ -78,7 +78,7 @@ require (
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/praetorian-inc/fingerprintx v1.1.9
-	github.com/projectdiscovery/dsl v0.0.51
+	github.com/projectdiscovery/dsl v0.0.52
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
 	github.com/projectdiscovery/goflags v0.1.48
diff --git a/go.sum b/go.sum
index 3912760c97..053952fe1f 100644
--- a/go.sum
+++ b/go.sum
@@ -832,8 +832,8 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
 github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
 github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.0.51 h1:7OQPumOrrUCFnCA7Y0nchhPvRo3IJGMIJ2Oy4DVTQsc=
-github.com/projectdiscovery/dsl v0.0.51/go.mod h1:GYhusn+T9EL7t+iJ8zN/GXlp8ohLGU+Yv/nevAPlJZg=
+github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
+github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
 github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E=
 github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=

From f2c0b4b443d7242525350c07fdc1ac91a72b56b4 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 15 Apr 2024 14:44:52 +0000
Subject: [PATCH 28/58] Auto Generate Syntax Docs + JSONSchema [Mon Apr 15
 14:44:52 UTC 2024] :robot:

---
 SYNTAX-REFERENCE.md            | 2 ++
 nuclei-jsonschema.json         | 3 ++-
 pkg/templates/templates_doc.go | 1 +
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index 9222e34aad..7fa62c5f44 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -2750,6 +2750,8 @@ Enum Values:
   - <code>TLSA</code>
 
   - <code>ANY</code>
+
+  - <code>SRV</code>
 </div>
 
 <hr />
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 9e14e531e0..46e1ab986d 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -84,7 +84,8 @@
         "AAAA",
         "CAA",
         "TLSA",
-        "ANY"
+        "ANY",
+        "SRV"
       ],
       "title": "type of DNS request to make",
       "description": "Type is the type of DNS request to make"
diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go
index dbd865c4e7..42fa25b947 100644
--- a/pkg/templates/templates_doc.go
+++ b/pkg/templates/templates_doc.go
@@ -1166,6 +1166,7 @@ func init() {
 		"CAA",
 		"TLSA",
 		"ANY",
+		"SRV",
 	}
 
 	FILERequestDoc.Type = "file.Request"

From 431d3fa2d940f23ab3b4be86f008d3d81cb8291d Mon Sep 17 00:00:00 2001
From: guangwu <guoguangwu@magic-shield.com>
Date: Tue, 16 Apr 2024 17:23:49 +0800
Subject: [PATCH 29/58] fix: close res body (#5025)

---
 pkg/reporting/exporters/es/elasticsearch.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkg/reporting/exporters/es/elasticsearch.go b/pkg/reporting/exporters/es/elasticsearch.go
index 68fb64e7e5..7b627492c9 100644
--- a/pkg/reporting/exporters/es/elasticsearch.go
+++ b/pkg/reporting/exporters/es/elasticsearch.go
@@ -131,7 +131,8 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
 	if err != nil {
 		return err
 	}
-
+	defer res.Body.Close() 
+	
 	b, err = io.ReadAll(res.Body)
 	if err != nil {
 		return errors.New(err.Error() + "error thrown by elasticsearch " + string(b))

From bec7cb273a1375d64b401eeee05a7667fa4998d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Levente=20Kov=C3=A1ts?=
 <22732484+tovask@users.noreply.github.com>
Date: Tue, 16 Apr 2024 13:27:07 +0200
Subject: [PATCH 30/58] add context vars in code and multi (#5051)

make the extracted variables available in subsequence templates when executing in a workflow

fix projectdiscovery/nuclei#4797
---
 cmd/integration-test/workflow.go              | 70 +++++++++++++++++++
 .../workflow/code-template-1.yaml             | 22 ++++++
 .../workflow/code-template-2.yaml             | 21 ++++++
 .../workflow/code-value-share-workflow.yaml   | 12 ++++
 .../multiprotocol-value-share-template.yaml   | 22 ++++++
 .../multiprotocol-value-share-workflow.yaml   | 11 +++
 pkg/protocols/code/code.go                    |  2 +
 pkg/tmplexec/multiproto/multi.go              |  6 ++
 8 files changed, 166 insertions(+)
 create mode 100644 integration_tests/workflow/code-template-1.yaml
 create mode 100644 integration_tests/workflow/code-template-2.yaml
 create mode 100644 integration_tests/workflow/code-value-share-workflow.yaml
 create mode 100644 integration_tests/workflow/multiprotocol-value-share-template.yaml
 create mode 100644 integration_tests/workflow/multiprotocol-value-share-workflow.yaml

diff --git a/cmd/integration-test/workflow.go b/cmd/integration-test/workflow.go
index 8672283b6c..e11575f90e 100644
--- a/cmd/integration-test/workflow.go
+++ b/cmd/integration-test/workflow.go
@@ -3,12 +3,15 @@ package main
 import (
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 
 	"github.com/julienschmidt/httprouter"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 )
 
@@ -20,9 +23,37 @@ var workflowTestcases = []TestCaseInfo{
 	{Path: "workflow/complex-conditions.yaml", TestCase: &workflowComplexConditions{}},
 	{Path: "workflow/http-value-share-workflow.yaml", TestCase: &workflowHttpKeyValueShare{}},
 	{Path: "workflow/dns-value-share-workflow.yaml", TestCase: &workflowDnsKeyValueShare{}},
+	{Path: "workflow/code-value-share-workflow.yaml", TestCase: &workflowCodeKeyValueShare{}, DisableOn: isCodeDisabled}, // isCodeDisabled declared in code.go
+	{Path: "workflow/multiprotocol-value-share-workflow.yaml", TestCase: &workflowMultiProtocolKeyValueShare{}},
 	{Path: "workflow/shared-cookie.yaml", TestCase: &workflowSharedCookies{}},
 }
 
+func init() {
+	// sign code templates (unless they are disabled)
+	if !isCodeDisabled() {
+		// allow local file access to load content of file references in template
+		// in order to sign them for testing purposes
+		templates.TemplateSignerLFA()
+
+		// testCertFile and testKeyFile are declared in code.go
+		tsigner, err := signer.NewTemplateSignerFromFiles(testCertFile, testKeyFile)
+		if err != nil {
+			panic(err)
+		}
+
+		// only the code templates are necessary to be signed
+		var templatesToSign = []string{
+			"workflow/code-template-1.yaml",
+			"workflow/code-template-2.yaml",
+		}
+		for _, templatePath := range templatesToSign {
+			if err := templates.SignTemplate(tsigner, templatePath); err != nil {
+				log.Fatalf("Could not sign template %v got: %s\n", templatePath, err)
+			}
+		}
+	}
+}
+
 type workflowBasic struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -159,6 +190,45 @@ func (h *workflowDnsKeyValueShare) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type workflowCodeKeyValueShare struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *workflowCodeKeyValueShare) Execute(filePath string) error {
+	// provide the Certificate File that the code templates are signed with
+	certEnvVar := signer.CertEnvVarName + "=" + testCertFile
+
+	results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, []string{certEnvVar}, "-workflows", filePath, "-target", "input", "-code")
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
+type workflowMultiProtocolKeyValueShare struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *workflowMultiProtocolKeyValueShare) Execute(filePath string) error {
+	router := httprouter.New()
+	// the response of path1 contains a domain that will be extracted and shared with the second template
+	router.GET("/path1", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "href=\"blog.projectdiscovery.io\"")
+	})
+	// path2 responds with the value of the "extracted" query parameter, e.g.: /path2?extracted=blog.projectdiscovery.io => blog.projectdiscovery.io
+	router.GET("/path2", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "%s", r.URL.Query().Get("extracted"))
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 2)
+}
+
 type workflowSharedCookies struct{}
 
 // Execute executes a test case and returns an error if occurred
diff --git a/integration_tests/workflow/code-template-1.yaml b/integration_tests/workflow/code-template-1.yaml
new file mode 100644
index 0000000000..d41a1a6957
--- /dev/null
+++ b/integration_tests/workflow/code-template-1.yaml
@@ -0,0 +1,22 @@
+id: code-template-1
+
+info:
+  name: code-template-1
+  author: tovask
+  severity: info
+  tags: code
+
+code:
+  - engine:
+      - py
+      - python3
+      - python
+    source: |
+      print("hello from first")
+    extractors:
+      - type: regex
+        name: extracted
+        regex:
+          - 'hello from (.*)'
+        group: 1
+# digest: 490a0046304402202c63d47bb0acdd40b3b852d95490d492ff5741b84071b2a8a40371be7797c13602202b6b977e157edf2ef70a402a2e57d4eb5a67c5ca91f0a2f9a10a966e8485ebaf:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/workflow/code-template-2.yaml b/integration_tests/workflow/code-template-2.yaml
new file mode 100644
index 0000000000..ddd7d36a4e
--- /dev/null
+++ b/integration_tests/workflow/code-template-2.yaml
@@ -0,0 +1,21 @@
+id: code-template-2
+
+info:
+  name: code-template-2
+  author: tovask
+  severity: info
+  tags: code
+
+code:
+  - engine:
+      - py
+      - python3
+      - python
+    source: |
+      import os
+      print("hello from " + os.getenv("extracted"))
+    matchers:
+      - type: word
+        words:
+          - "hello from first"
+# digest: 490a00463044022025661eab353b7f359c0d428a86b6287545d7f759375e8025cc8c9c77b616ca6502200bc2c019059622df3c88e7caa6dd7d1fb9b956010aa0de2ee2b9f7dd0a3c4954:4a3eb6b4988d95847d4203be25ed1d46
\ No newline at end of file
diff --git a/integration_tests/workflow/code-value-share-workflow.yaml b/integration_tests/workflow/code-value-share-workflow.yaml
new file mode 100644
index 0000000000..a7c27b5eab
--- /dev/null
+++ b/integration_tests/workflow/code-value-share-workflow.yaml
@@ -0,0 +1,12 @@
+id: code-value-sharing-workflow
+
+info:
+  name: Code Value Sharing Workflow
+  author: tovask
+  severity: info
+  tags: code
+
+workflows:
+  - template: workflow/code-template-1.yaml
+    subtemplates:
+      - template: workflow/code-template-2.yaml
diff --git a/integration_tests/workflow/multiprotocol-value-share-template.yaml b/integration_tests/workflow/multiprotocol-value-share-template.yaml
new file mode 100644
index 0000000000..41a2469191
--- /dev/null
+++ b/integration_tests/workflow/multiprotocol-value-share-template.yaml
@@ -0,0 +1,22 @@
+id: multiprotocol-value-sharing-template
+
+info:
+  name: MultiProtocol Value Sharing Template
+  author: tovask
+  severity: info
+
+dns:
+  - name: "{{extracted}}"
+    type: PTR
+    matchers:
+      - type: word
+        words:
+          - "blog.projectdiscovery.io"
+
+http:
+  - path:
+      - "{{BaseURL}}/path2?extracted={{extracted}}"
+    matchers:
+      - type: word
+        words:
+          - "blog.projectdiscovery.io"
diff --git a/integration_tests/workflow/multiprotocol-value-share-workflow.yaml b/integration_tests/workflow/multiprotocol-value-share-workflow.yaml
new file mode 100644
index 0000000000..914ddbc4e7
--- /dev/null
+++ b/integration_tests/workflow/multiprotocol-value-share-workflow.yaml
@@ -0,0 +1,11 @@
+id: multiprotocol-value-sharing-workflow
+
+info:
+  name: MultiProtocol Value Sharing Workflow
+  author: tovask
+  severity: info
+
+workflows:
+  - template: workflow/http-value-share-template-1.yaml
+    subtemplates:
+      - template: workflow/multiprotocol-value-share-template.yaml
diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go
index 43c6721c59..1b59de3efd 100644
--- a/pkg/protocols/code/code.go
+++ b/pkg/protocols/code/code.go
@@ -160,6 +160,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	if request.options.HasTemplateCtx(input.MetaInput) {
 		allvars = generators.MergeMaps(allvars, request.options.GetTemplateCtx(input.MetaInput).GetAll())
 	}
+	// add dynamic and previous variables
+	allvars = generators.MergeMaps(allvars, dynamicValues, previous)
 	// optionvars are vars passed from CLI or env variables
 	optionVars := generators.BuildPayloadFromOptions(request.options.Options)
 	variablesMap := request.options.Variables.Evaluate(allvars)
diff --git a/pkg/tmplexec/multiproto/multi.go b/pkg/tmplexec/multiproto/multi.go
index d164c03aab..997e62243d 100644
--- a/pkg/tmplexec/multiproto/multi.go
+++ b/pkg/tmplexec/multiproto/multi.go
@@ -46,6 +46,12 @@ func (m *MultiProtocol) Compile() error {
 func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
 	// put all readonly args into template context
 	m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(m.readOnlyArgs)
+
+	// add all input args to template context
+	ctx.Input.ForEach(func(key string, value interface{}) {
+		m.options.GetTemplateCtx(ctx.Input.MetaInput).Set(key, value)
+	})
+
 	// callback to process results from all protocols
 	multiProtoCallback := func(event *output.InternalWrappedEvent) {
 		if event == nil {

From ea2e13a4aa83ac349e3fde29bdbaa0814a2540f9 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Tue, 16 Apr 2024 16:57:32 +0530
Subject: [PATCH 31/58] nuclei 'stats' build : scan events + chart utils
 (#5032)

* prototype new scan events

* scan-event: improvements + conditional build

* add scan charts server: make scan-charts

* scan-charts: bug fix
---
 .gitignore                     |   3 +
 Makefile                       |   9 +
 cmd/scan-charts/main.go        |  40 ++++
 go.mod                         |   3 +
 go.sum                         |   4 +
 internal/runner/runner.go      |  15 ++
 pkg/scan/charts/charts.go      |  87 ++++++++
 pkg/scan/charts/echarts.go     | 351 +++++++++++++++++++++++++++++++++
 pkg/scan/events/scan_noop.go   |  11 ++
 pkg/scan/events/stats_build.go |  80 ++++++++
 pkg/scan/events/utils.go       |  45 +++++
 pkg/scan/scan_context.go       |   1 +
 pkg/tmplexec/exec.go           |  41 ++++
 pkg/types/types.go             |   2 +
 14 files changed, 692 insertions(+)
 create mode 100644 cmd/scan-charts/main.go
 create mode 100644 pkg/scan/charts/charts.go
 create mode 100644 pkg/scan/charts/echarts.go
 create mode 100644 pkg/scan/events/scan_noop.go
 create mode 100644 pkg/scan/events/stats_build.go
 create mode 100644 pkg/scan/events/utils.go

diff --git a/.gitignore b/.gitignore
index 65815d8db6..9386137b54 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,5 +36,8 @@ pkg/protocols/headless/engine/.cache
 /fuzzplayground
 integration_tests/fuzzplayground
 /dsl.md
+/nuclei-stats
+/nuclei-stats-*
+/scan-charts
 
 
diff --git a/Makefile b/Makefile
index dadbe6205d..e916b980a6 100644
--- a/Makefile
+++ b/Makefile
@@ -11,9 +11,18 @@ ifneq ($(shell go env GOOS),darwin)
 LDFLAGS := -extldflags "-static"
 endif
     
+.PHONY: all build build-stats scan-charts docs test integration functional tidy devtools jsupdate ts fuzzplayground memogen dsl-docs
+
 all: build
 build:
+	rm -f nuclei 2>/dev/null
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "nuclei" cmd/nuclei/main.go
+build-stats:
+	rm -f nuclei-stats 2>/dev/null
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -tags=stats -o "nuclei-stats" cmd/nuclei/main.go
+scan-charts:
+	rm -f scan-charts 2>/dev/null
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scan-charts" cmd/scan-charts/main.go
 docs:
 	if ! which dstdocgen > /dev/null; then
 		echo -e "Command not found! Install? (y/n) \c"
diff --git a/cmd/scan-charts/main.go b/cmd/scan-charts/main.go
new file mode 100644
index 0000000000..644a1f004e
--- /dev/null
+++ b/cmd/scan-charts/main.go
@@ -0,0 +1,40 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/charts"
+)
+
+var (
+	dir     string
+	address string
+	output  string
+)
+
+func main() {
+	flag.StringVar(&dir, "dir", "", "directory to scan")
+	flag.StringVar(&address, "address", ":9000", "address to run the server on")
+	flag.StringVar(&output, "output", "", "output filename of generated html file")
+	flag.Parse()
+
+	if dir == "" {
+		flag.Usage()
+		return
+	}
+
+	server, err := charts.NewScanEventsCharts(dir)
+	if err != nil {
+		panic(err)
+	}
+	server.PrintInfo()
+
+	if output != "" {
+		if err = server.GenerateHTML(output); err != nil {
+			panic(err)
+		}
+		return
+	}
+
+	server.Start(address)
+}
diff --git a/go.mod b/go.mod
index a533740474..4cf94ec00c 100644
--- a/go.mod
+++ b/go.mod
@@ -329,6 +329,7 @@ require (
 	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/dop251/goja_nodejs v0.0.0-20230821135201-94e508132562
 	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-echarts/go-echarts/v2 v2.3.3
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
@@ -348,3 +349,5 @@ require (
 
 // https://go.dev/ref/mod#go-mod-file-retract
 retract v3.2.0 // retract due to broken js protocol issue
+
+replace github.com/go-echarts/go-echarts/v2 => github.com/tarunKoyalwar/go-echarts/v2 v2.1.1
diff --git a/go.sum b/go.sum
index 290fd9e482..888602a714 100644
--- a/go.sum
+++ b/go.sum
@@ -341,6 +341,8 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
 github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwSmwnCg4bxg=
+github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
 github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
 github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -1013,6 +1015,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 h1:5fsXGPmK+i18J8cDgxy7AJkiXWBARpVTb0Gbv+bAzPo=
+github.com/tarunKoyalwar/go-echarts/v2 v2.1.1/go.mod h1:VEeyPT5Odx/UHeuxtIAHGu2+87MWGA5OBaZ120NFi/w=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
 github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index fa9b85bf05..174c06f0f7 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -18,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
 	uncoverlib "github.com/projectdiscovery/uncover"
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/env"
@@ -553,6 +554,20 @@ func (r *Runner) RunEnumeration() error {
 		executorOpts.InputHelper.InputsHTTP = inputHelpers
 	}
 
+	// initialize stats worker ( this is no-op unless nuclei is built with stats build tag)
+	// during execution a directory with 2 files will be created in the current directory
+	// config.json - containing below info
+	// events.jsonl - containing all start and end times of all templates
+	events.InitWithConfig(&events.ScanConfig{
+		Name:                "nuclei-stats", // make this configurable
+		TargetCount:         int(r.inputProvider.Count()),
+		TemplatesCount:      len(store.Templates()) + len(store.Workflows()),
+		TemplateConcurrency: r.options.TemplateThreads,
+		PayloadConcurrency:  r.options.PayloadConcurrency,
+		JsConcurrency:       r.options.JsConcurrency,
+		Retries:             r.options.Retries,
+	}, "")
+
 	enumeration := false
 	var results *atomic.Bool
 	results, err = r.runStandardEnumeration(executorOpts, store, executorEngine)
diff --git a/pkg/scan/charts/charts.go b/pkg/scan/charts/charts.go
new file mode 100644
index 0000000000..03bfdcc6c2
--- /dev/null
+++ b/pkg/scan/charts/charts.go
@@ -0,0 +1,87 @@
+package charts
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/labstack/echo/v4"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
+	fileutil "github.com/projectdiscovery/utils/file"
+)
+
+// ScanEventsCharts is a struct for nuclei event charts
+type ScanEventsCharts struct {
+	eventsDir string
+	config    *events.ScanConfig
+	data      []events.ScanEvent
+}
+
+func (sc *ScanEventsCharts) PrintInfo() {
+	fmt.Printf("[+] Scan Info\n")
+	fmt.Printf("  - Name: %s\n", sc.config.Name)
+	fmt.Printf("  - Target Count: %d\n", sc.config.TargetCount)
+	fmt.Printf("  - Template Count: %d\n", sc.config.TemplatesCount)
+	fmt.Printf("  - Template Concurrency: %d\n", sc.config.TemplateConcurrency)
+	fmt.Printf("  - Payload Concurrency: %d\n", sc.config.PayloadConcurrency)
+	fmt.Printf("  - Retries: %v\n", sc.config.Retries)
+	fmt.Printf("  - Total Events: %d\n", len(sc.data))
+	fmt.Println()
+}
+
+// NewScanEventsCharts creates a new nuclei event charts
+func NewScanEventsCharts(eventsDir string) (*ScanEventsCharts, error) {
+	sc := &ScanEventsCharts{eventsDir: eventsDir}
+	if !fileutil.FolderExists(eventsDir) {
+		return nil, fmt.Errorf("events directory does not exist")
+	}
+	// open two files
+	// config.json
+	bin, err := os.ReadFile(filepath.Join(eventsDir, events.ConfigFile))
+	if err != nil {
+		return nil, err
+	}
+	var config events.ScanConfig
+	err = json.Unmarshal(bin, &config)
+	if err != nil {
+		return nil, err
+	}
+	sc.config = &config
+
+	// events.jsonl
+	f, err := os.Open(filepath.Join(eventsDir, events.EventsFile))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	data := []events.ScanEvent{}
+	dec := json.NewDecoder(f)
+	for {
+		var event events.ScanEvent
+		if err := dec.Decode(&event); err != nil {
+			break
+		}
+		data = append(data, event)
+	}
+	sc.data = data
+
+	if len(data) == 0 {
+		return nil, fmt.Errorf("no events found in the events file")
+	}
+
+	return sc, nil
+}
+
+// Start starts the nuclei event charts server
+func (sc *ScanEventsCharts) Start(addr string) {
+	e := echo.New()
+	e.HideBanner = true
+	e.GET("/concurrency", sc.ConcurrencyVsTime)
+	e.GET("/requests", sc.TotalRequestsOverTime)
+	e.GET("/slow", sc.TopSlowTemplates)
+	e.GET("/rps", sc.RequestsVSInterval)
+	e.GET("/", sc.AllCharts)
+	e.Logger.Fatal(e.Start(addr))
+}
diff --git a/pkg/scan/charts/echarts.go b/pkg/scan/charts/echarts.go
new file mode 100644
index 0000000000..bf70fee710
--- /dev/null
+++ b/pkg/scan/charts/echarts.go
@@ -0,0 +1,351 @@
+package charts
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"time"
+
+	"github.com/go-echarts/go-echarts/v2/charts"
+	"github.com/go-echarts/go-echarts/v2/components"
+	"github.com/go-echarts/go-echarts/v2/opts"
+	"github.com/labstack/echo/v4"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
+	sliceutil "github.com/projectdiscovery/utils/slice"
+)
+
+const (
+	TopK         = 50
+	SpacerHeight = "50px"
+)
+
+func (s *ScanEventsCharts) AllCharts(c echo.Context) error {
+	page := s.allCharts(c)
+	return page.Render(c.Response().Writer)
+}
+
+func (s *ScanEventsCharts) GenerateHTML(filePath string) error {
+	page := s.allCharts(nil)
+	output, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	return page.Render(output)
+}
+
+// AllCharts generates all the charts for the scan events and returns a page component
+func (s *ScanEventsCharts) allCharts(c echo.Context) *components.Page {
+	page := components.NewPage()
+	page.PageTitle = "Nuclei Charts"
+	line1 := s.totalRequestsOverTime(c)
+	line1.SetSpacerHeight(SpacerHeight)
+	kline := s.topSlowTemplates(c)
+	kline.SetSpacerHeight(SpacerHeight)
+	line2 := s.requestsVSInterval(c)
+	line2.SetSpacerHeight(SpacerHeight)
+	line3 := s.concurrencyVsTime(c)
+	line3.SetSpacerHeight(SpacerHeight)
+	page.AddCharts(line1, kline, line2, line3)
+	page.Validate()
+	page.SetLayout(components.PageCenterLayout)
+	return page
+}
+
+func (s *ScanEventsCharts) TotalRequestsOverTime(c echo.Context) error {
+	line := s.totalRequestsOverTime(c)
+	return line.Render(c.Response().Writer)
+}
+
+// totalRequestsOverTime generates a line chart showing total requests count over time
+func (s *ScanEventsCharts) totalRequestsOverTime(c echo.Context) *charts.Line {
+	line := charts.NewLine()
+	line.SetCaption("Chart Shows Total Requests Count Over Time (for each/all Protocols)")
+
+	var startTime time.Time = time.Now()
+	var endTime time.Time
+
+	for _, event := range s.data {
+		if event.Time.Before(startTime) {
+			startTime = event.Time
+		}
+		if event.Time.After(endTime) {
+			endTime = event.Time
+		}
+	}
+	data := getCategoryRequestCount(s.data)
+	max := 0
+	for _, v := range data {
+		if len(v) > max {
+			max = len(v)
+		}
+	}
+	line.SetXAxis(time.Now().Format(time.RFC3339))
+	for k, v := range data {
+		lineData := make([]opts.LineData, 0)
+		temp := 0
+		for _, scanEvent := range v {
+			temp += scanEvent.MaxRequests
+			val := scanEvent.Time.Sub(startTime)
+			lineData = append(lineData, opts.LineData{
+				Value: []interface{}{val.Milliseconds(), temp},
+				Name:  scanEvent.TemplateID,
+			})
+		}
+		line.AddSeries(k, lineData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
+	}
+
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: "Nuclei: total-req vs time"}),
+		charts.WithXAxisOpts(opts.XAxis{Name: "Time", Type: "time", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
+		charts.WithYAxisOpts(opts.YAxis{Name: "Requests Sent", Type: "value"}),
+		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	line.Validate()
+	return line
+}
+
+func (s *ScanEventsCharts) TopSlowTemplates(c echo.Context) error {
+	kline := s.topSlowTemplates(c)
+	return kline.Render(c.Response().Writer)
+}
+
+// topSlowTemplates generates a Kline chart showing the top slow templates by time taken
+func (s *ScanEventsCharts) topSlowTemplates(c echo.Context) *charts.Kline {
+	kline := charts.NewKLine()
+	kline.SetCaption(fmt.Sprintf("Chart Shows Top Slow Templates (by time taken) (Top %v)", TopK))
+
+	ids := map[string][]int64{}
+	var startTime time.Time = time.Now()
+	for _, event := range s.data {
+		if event.Time.Before(startTime) {
+			startTime = event.Time
+		}
+	}
+	for _, event := range s.data {
+		ids[event.TemplateID] = append(ids[event.TemplateID], event.Time.Sub(startTime).Milliseconds())
+	}
+
+	type entry struct {
+		ID        string
+		KlineData opts.KlineData
+		start     int64
+		end       int64
+	}
+	data := []entry{}
+
+	for a, b := range ids {
+		if len(b) < 2 {
+			continue // Prevents index out of range error
+		}
+		d := entry{
+			ID:        a,
+			KlineData: opts.KlineData{Value: []int64{b[0], b[len(b)-1], b[0], b[len(b)-1]}}, // Adjusted to prevent index out of range error
+			start:     b[0],
+			end:       b[len(b)-1],
+		}
+		data = append(data, d)
+	}
+
+	sort.Slice(data, func(i, j int) bool {
+		return data[i].end-data[i].start > data[j].end-data[j].start
+	})
+
+	x := make([]string, 0)
+	y := make([]opts.KlineData, 0)
+	for _, event := range data[:TopK] {
+		x = append(x, event.ID)
+		y = append(y, event.KlineData)
+	}
+
+	kline.SetXAxis(x).AddSeries("templates", y)
+	kline.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: fmt.Sprintf("Nuclei: Top %v Slow Templates", TopK)}),
+		charts.WithXAxisOpts(opts.XAxis{
+			Type:      "category",
+			Show:      true,
+			AxisLabel: &opts.AxisLabel{Rotate: 90, Show: true, ShowMinLabel: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (value) { return value; }`)},
+		}),
+		charts.WithYAxisOpts(opts.YAxis{
+			Scale:     true,
+			Type:      "value",
+			Show:      true,
+			AxisLabel: &opts.AxisLabel{Show: true, Formatter: opts.FuncOpts(`function (ms) {  return Math.floor(ms/60000) + 'm' + Math.floor((ms/60000 - Math.floor(ms/60000))*60) + 's'; }`)},
+		}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "40%", Top: "10%"}),
+		charts.WithTooltipOpts(opts.Tooltip{Show: true, Trigger: "events.ScanEvent", TriggerOn: "mousemove|click", Enterable: true, Formatter: opts.FuncOpts(`function (params) { return params.name ; }`)}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	return kline
+}
+
+func (s *ScanEventsCharts) RequestsVSInterval(c echo.Context) error {
+	line := s.requestsVSInterval(c)
+	return line.Render(c.Response().Writer)
+}
+
+// requestsVSInterval generates a line chart showing requests per second over time
+func (s *ScanEventsCharts) requestsVSInterval(c echo.Context) *charts.Line {
+	line := charts.NewLine()
+	line.SetCaption("Chart Shows RPS (Requests Per Second) Over Time")
+
+	sort.Slice(s.data, func(i, j int) bool {
+		return s.data[i].Time.Before(s.data[j].Time)
+	})
+
+	var interval time.Duration
+
+	if c != nil {
+		interval, _ = time.ParseDuration(c.QueryParam("interval"))
+	}
+	if interval <= 3 {
+		interval = 5 * time.Second
+	}
+
+	data := []opts.LineData{}
+	temp := 0
+	if len(s.data) > 0 {
+		orig := s.data[0].Time
+		startTime := orig
+		xaxisData := []int64{}
+		for _, v := range s.data {
+			if v.Time.Sub(startTime) > interval {
+				millisec := v.Time.Sub(orig).Milliseconds()
+				xaxisData = append(xaxisData, millisec)
+				data = append(data, opts.LineData{Value: temp, Name: v.Time.Sub(orig).String()})
+				temp = 0
+				startTime = v.Time
+			}
+			temp += 1
+		}
+		// Handle last interval if exists
+		if temp > 0 {
+			millisec := s.data[len(s.data)-1].Time.Sub(orig).Milliseconds()
+			xaxisData = append(xaxisData, millisec)
+			data = append(data, opts.LineData{Value: temp, Name: s.data[len(s.data)-1].Time.Sub(orig).String()})
+		}
+		line.SetXAxis(xaxisData)
+		line.AddSeries("RPS", data, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
+	}
+
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: "Nuclei: Template Execution", Subtitle: "Time Interval: " + interval.String()}),
+		charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
+		charts.WithYAxisOpts(opts.YAxis{Name: "RPS Value", Type: "value", Show: true}),
+		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	line.Validate()
+	return line
+}
+
+func (s *ScanEventsCharts) ConcurrencyVsTime(c echo.Context) error {
+	line := s.concurrencyVsTime(c)
+	return line.Render(c.Response().Writer)
+}
+
+// concurrencyVsTime generates a line chart showing concurrency (total workers) over time
+func (s *ScanEventsCharts) concurrencyVsTime(c echo.Context) *charts.Line {
+	line := charts.NewLine()
+	line.SetCaption("Chart Shows Concurrency (Total Workers) Over Time")
+
+	dataset := sliceutil.Clone(s.data)
+
+	sort.Slice(dataset, func(i, j int) bool {
+		return dataset[i].Time.Before(dataset[j].Time)
+	})
+
+	var interval time.Duration
+	if c != nil {
+		interval, _ = time.ParseDuration(c.QueryParam("interval"))
+	}
+	if interval <= 3 {
+		interval = 5 * time.Second
+	}
+
+	// create array with time interval as x-axis and worker count as y-axis
+	// entry is a struct with time and poolsize
+	type entry struct {
+		Time     time.Duration
+		poolsize int
+	}
+	allEntries := []entry{}
+
+	dataIndex := 0
+	maxIndex := len(dataset) - 1
+	currEntry := entry{}
+
+	lastTime := dataset[0].Time
+	for dataIndex <= maxIndex {
+		currTime := dataset[dataIndex].Time
+		if currTime.Sub(lastTime) > interval {
+			// next batch
+			currEntry.Time = interval
+			allEntries = append(allEntries, currEntry)
+			lastTime = dataset[dataIndex-1].Time
+		}
+		if dataset[dataIndex].EventType == events.ScanStarted {
+			currEntry.poolsize += 1
+		} else {
+			currEntry.poolsize -= 1
+		}
+		dataIndex += 1
+	}
+
+	plotData := []opts.LineData{}
+	xaxisData := []int64{}
+	tempTime := time.Duration(0)
+	for _, v := range allEntries {
+		tempTime += v.Time
+		plotData = append(plotData, opts.LineData{Value: v.poolsize, Name: tempTime.String()})
+		xaxisData = append(xaxisData, tempTime.Milliseconds())
+	}
+	line.SetXAxis(xaxisData)
+	line.AddSeries("Concurrency", plotData, charts.WithLineChartOpts(opts.LineChart{Smooth: false}), charts.WithLabelOpts(opts.Label{Show: true, Position: "top"}))
+
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{Title: "Nuclei: WorkerPool", Subtitle: "Time Interval: " + interval.String()}),
+		charts.WithXAxisOpts(opts.XAxis{Name: "Time Intervals", Type: "category", AxisLabel: &opts.AxisLabel{Show: true, ShowMaxLabel: true, Formatter: opts.FuncOpts(`function (date) { return (date/1000)+'s'; }`)}}),
+		charts.WithYAxisOpts(opts.YAxis{Name: "Total Workers", Type: "value", Show: true}),
+		charts.WithInitializationOpts(opts.Initialization{Theme: "dark"}),
+		charts.WithDataZoomOpts(opts.DataZoom{Type: "slider", Start: 0, End: 100}),
+		charts.WithGridOpts(opts.Grid{Left: "10%", Right: "10%", Bottom: "15%", Top: "20%"}),
+		charts.WithToolboxOpts(opts.Toolbox{Show: true, Feature: &opts.ToolBoxFeature{
+			SaveAsImage: &opts.ToolBoxFeatureSaveAsImage{Show: true, Name: "save", Title: "save"},
+			DataZoom:    &opts.ToolBoxFeatureDataZoom{Show: true, Title: map[string]string{"zoom": "zoom", "back": "back"}},
+			DataView:    &opts.ToolBoxFeatureDataView{Show: true, Title: "raw", Lang: []string{"raw", "exit", "refresh"}},
+		}}),
+	)
+
+	line.Validate()
+	return line
+}
+
+// getCategoryRequestCount returns a map of category and request count
+func getCategoryRequestCount(values []events.ScanEvent) map[string][]events.ScanEvent {
+	mx := make(map[string][]events.ScanEvent)
+	for _, event := range values {
+		mx[event.TemplateType] = append(mx[event.TemplateType], event)
+	}
+	return mx
+}
diff --git a/pkg/scan/events/scan_noop.go b/pkg/scan/events/scan_noop.go
new file mode 100644
index 0000000000..a284657f52
--- /dev/null
+++ b/pkg/scan/events/scan_noop.go
@@ -0,0 +1,11 @@
+//go:build !stats
+// +build !stats
+
+package events
+
+// AddScanEvent is a no-op function
+func AddScanEvent(event ScanEvent) {
+}
+
+func InitWithConfig(config *ScanConfig, statsDirectory string) {
+}
diff --git a/pkg/scan/events/stats_build.go b/pkg/scan/events/stats_build.go
new file mode 100644
index 0000000000..0f01724411
--- /dev/null
+++ b/pkg/scan/events/stats_build.go
@@ -0,0 +1,80 @@
+//go:build stats
+// +build stats
+
+package events
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+var _ ScanEventWorker = &ScanStatsWorker{}
+
+var defaultWorker = &ScanStatsWorker{}
+
+// ScanStatsWorker is a worker for scanning stats
+// This tracks basic stats in jsonlines format
+// in given directory or a default directory with name stats_{timestamp} in the current directory
+type ScanStatsWorker struct {
+	config    *ScanConfig
+	m         *sync.Mutex
+	directory string
+	enc       *json.Encoder
+}
+
+// Init initializes the scan stats worker
+func InitWithConfig(config *ScanConfig, statsDirectory string) {
+	currentTime := time.Now().Format("20060102150405")
+	dirName := fmt.Sprintf("nuclei-stats-%s", currentTime)
+	err := os.Mkdir(dirName, 0755)
+	if err != nil {
+		panic(err)
+	}
+	// save the config to the directory
+	bin, err := json.MarshalIndent(config, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+	err = os.WriteFile(filepath.Join(dirName, ConfigFile), bin, 0755)
+	if err != nil {
+		panic(err)
+	}
+	defaultWorker = &ScanStatsWorker{config: config, m: &sync.Mutex{}, directory: dirName}
+	err = defaultWorker.initEventsFile()
+	if err != nil {
+		panic(err)
+	}
+}
+
+// initEventsFile initializes the events file for the worker
+func (s *ScanStatsWorker) initEventsFile() error {
+	f, err := os.Create(filepath.Join(s.directory, EventsFile))
+	if err != nil {
+		return err
+	}
+	s.enc = json.NewEncoder(f)
+	return nil
+}
+
+// AddScanEvent adds a scan event to the worker
+func (s *ScanStatsWorker) AddScanEvent(event ScanEvent) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	err := s.enc.Encode(event)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// AddScanEvent adds a scan event to the worker
+func AddScanEvent(event ScanEvent) {
+	if defaultWorker == nil {
+		return
+	}
+	defaultWorker.AddScanEvent(event)
+}
diff --git a/pkg/scan/events/utils.go b/pkg/scan/events/utils.go
new file mode 100644
index 0000000000..edd7b09ae5
--- /dev/null
+++ b/pkg/scan/events/utils.go
@@ -0,0 +1,45 @@
+package events
+
+import (
+	"time"
+)
+
+type ScanEventWorker interface {
+	// AddScanEvent adds a scan event to the worker
+	AddScanEvent(event ScanEvent)
+}
+
+// Track scan start / finish status
+type ScanStatus string
+
+const (
+	ScanStarted  ScanStatus = "scan_start"
+	ScanFinished ScanStatus = "scan_end"
+)
+
+const (
+	ConfigFile = "config.json"
+	EventsFile = "events.jsonl"
+)
+
+// ScanEvent represents a single scan event with its metadata
+type ScanEvent struct {
+	Target       string     `json:"target" yaml:"target"`
+	TemplateType string     `json:"template_type" yaml:"template_type"`
+	TemplateID   string     `json:"template_id" yaml:"template_id"`
+	TemplatePath string     `json:"template_path" yaml:"template_path"`
+	MaxRequests  int        `json:"max_requests" yaml:"max_requests"`
+	Time         time.Time  `json:"time" yaml:"time"`
+	EventType    ScanStatus `json:"event_type" yaml:"event_type"`
+}
+
+// ScanConfig is only in context of scan event analysis
+type ScanConfig struct {
+	Name                string `json:"name" yaml:"name"`
+	TargetCount         int    `json:"target_count" yaml:"target_count"`
+	TemplatesCount      int    `json:"templates_count" yaml:"templates_count"`
+	TemplateConcurrency int    `json:"template_concurrency" yaml:"template_concurrency"`
+	PayloadConcurrency  int    `json:"payload_concurrency" yaml:"payload_concurrency"`
+	JsConcurrency       int    `json:"js_concurrency" yaml:"js_concurrency"`
+	Retries             int    `json:"retries" yaml:"retries"`
+}
diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go
index a5e310b7c6..8851349019 100644
--- a/pkg/scan/scan_context.go
+++ b/pkg/scan/scan_context.go
@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 )
 
+
 type ScanContextOption func(*ScanContext)
 
 func WithEvents() ScanContextOption {
diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go
index e434f2173c..fc504171e6 100644
--- a/pkg/tmplexec/exec.go
+++ b/pkg/tmplexec/exec.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 	"sync/atomic"
+	"time"
 
 	"github.com/dop251/goja"
 	"github.com/projectdiscovery/gologger"
@@ -14,6 +15,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
+	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto"
@@ -92,6 +94,31 @@ func (e *TemplateExecuter) Requests() int {
 
 // Execute executes the protocol group and returns true or false if results were found.
 func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
+
+	// === when nuclei is built with -tags=stats ===
+	// Note: this is no-op (empty functions) when nuclei is built in normal or without -tags=stats
+	events.AddScanEvent(events.ScanEvent{
+		Target:       ctx.Input.MetaInput.Input,
+		Time:         time.Now(),
+		EventType:    events.ScanStarted,
+		TemplateType: e.getTemplateType(),
+		TemplateID:   e.options.TemplateID,
+		TemplatePath: e.options.TemplatePath,
+		MaxRequests:  e.Requests(),
+	})
+	defer func() {
+		events.AddScanEvent(events.ScanEvent{
+			Target:       ctx.Input.MetaInput.Input,
+			Time:         time.Now(),
+			EventType:    events.ScanFinished,
+			TemplateType: e.getTemplateType(),
+			TemplateID:   e.options.TemplateID,
+			TemplatePath: e.options.TemplatePath,
+			MaxRequests:  e.Requests(),
+		})
+	}()
+	// ==== end of stats ====
+
 	// executed contains status of execution if it was successfully executed or not
 	// doesn't matter if it was matched or not
 	executed := &atomic.Bool{}
@@ -182,3 +209,17 @@ func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.
 	ctx.LogError(err)
 	return ctx.GenerateResult(), err
 }
+
+// getTemplateType returns the template type of the template
+func (e *TemplateExecuter) getTemplateType() string {
+	if len(e.requests) == 0 {
+		return "null"
+	}
+	if e.options.Flow != "" {
+		return "flow"
+	}
+	if len(e.requests) > 1 {
+		return "multiprotocol"
+	}
+	return e.requests[0].Type().String()
+}
diff --git a/pkg/types/types.go b/pkg/types/types.go
index c251dc0c85..b6bdcaee55 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -419,7 +419,9 @@ func DefaultOptions() *Options {
 		BulkSize:                25,
 		TemplateThreads:         25,
 		HeadlessBulkSize:        10,
+		PayloadConcurrency:      25,
 		HeadlessTemplateThreads: 10,
+		ProbeConcurrency:        50,
 		Timeout:                 5,
 		Retries:                 1,
 		MaxHostError:            30,

From 84582183dce9046fdeb086dd3d4251ee701cbfea Mon Sep 17 00:00:00 2001
From: lvyaoting <166296299+lvyaoting@users.noreply.github.com>
Date: Tue, 16 Apr 2024 19:27:51 +0800
Subject: [PATCH 32/58] chore: fix function names in comment (#5008)

Signed-off-by: lvyaoting <lvyaoting@outlook.com>
---
 cmd/tmc/main.go | 2 +-
 lib/config.go   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/tmc/main.go b/cmd/tmc/main.go
index cd552acf49..eae38a1ca7 100644
--- a/cmd/tmc/main.go
+++ b/cmd/tmc/main.go
@@ -266,7 +266,7 @@ func enhanceTemplate(data string) (string, bool, error) {
 	return data, false, errorutil.New("template enhance failed")
 }
 
-// formatTemplateData formats template data using templateman format api
+// formatTemplate formats template data using templateman format api
 func formatTemplate(data string) (string, bool, error) {
 	resp, err := retryablehttp.DefaultClient().Post(fmt.Sprintf("%s/format", tmBaseUrl), "application/x-yaml", strings.NewReader(data))
 	if err != nil {
diff --git a/lib/config.go b/lib/config.go
index 5b112f8d25..ddcf3b88b4 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -314,7 +314,7 @@ func WithScanStrategy(strategy string) NucleiSDKOptions {
 // OutputWriter
 type OutputWriter output.Writer
 
-// UseWriter allows setting custom output writer
+// UseOutputWriter allows setting custom output writer
 // by default a mock writer is used with user defined callback
 // if outputWriter is used callback will be ignored
 func UseOutputWriter(writer OutputWriter) NucleiSDKOptions {

From e432e4a4c2f884fc39eeec504abe28194679d6a2 Mon Sep 17 00:00:00 2001
From: 4shen0ne <33086594+zrquan@users.noreply.github.com>
Date: Wed, 17 Apr 2024 00:12:44 +0800
Subject: [PATCH 33/58] Fix typos (#5038)

* Fix typos

- Add a quantifier to make the description unambiguous
- Add a missing verb

* Fix href of FAQs
---
 README.md    | 2 +-
 README_CN.md | 8 ++++----
 README_ID.md | 2 +-
 README_KR.md | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 07bce472b5..319f478c1b 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@
   <a href="#install-nuclei">Install</a> •
   <a href="https://docs.projectdiscovery.io/tools/nuclei/">Documentation</a> •
   <a href="#credits">Credits</a> •
-  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
+  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">FAQs</a> •
   <a href="https://discord.gg/projectdiscovery">Join Discord</a>
 </p>
 
diff --git a/README_CN.md b/README_CN.md
index 095f9c109f..25e39b2ae6 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -23,8 +23,8 @@
   <a href="#对于安全工程师">对于安全工程师</a> •
   <a href="#对于开发和组织">对于开发者</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">文档</a> •
-  <a href="#c致谢">致谢</a> •
-  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">常见问题</a> •
+  <a href="#致谢">致谢</a> •
+  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">常见问题</a> •
   <a href="https://discord.gg/projectdiscovery">加入Discord</a>
 </p>
 
@@ -39,7 +39,7 @@
 
 Nuclei使用零误报的定制模板向目标发送请求，同时可以对主机进行批量快速扫描。Nuclei提供TCP、DNS、HTTP、FILE等各类协议的扫描，通过强大且灵活的模板，可以使用Nuclei模拟各种安全检查。
 
-我们的[模板仓库](https://github.com/projectdiscovery/nuclei-templates)包含**超过300**安全研究员和工程师提供的模板。
+我们的[模板仓库](https://github.com/projectdiscovery/nuclei-templates)包含**超过300名**安全研究员和工程师提供的模板。
 
 
 
@@ -439,7 +439,7 @@ Nuclei构建很简单，通过数百名安全研究员的社区模板，Nuclei
 </a>
 </p>
 
-另外您可以其他类似的开源项目：
+另外您可以了解其他类似的开源项目：
 
 [FFuF](https://github.com/ffuf/ffuf), [Qsfuzz](https://github.com/ameenmaali/qsfuzz), [Inception](https://github.com/proabiral/inception), [Snallygaster](https://github.com/hannob/snallygaster), [Gofingerprint](https://github.com/Static-Flow/gofingerprint), [Sn1per](https://github.com/1N3/Sn1per/tree/master/templates), [Google tsunami](https://github.com/google/tsunami-security-scanner), [Jaeles](https://github.com/jaeles-project/jaeles), [ChopChop](https://github.com/michelin/ChopChop)
 
diff --git a/README_ID.md b/README_ID.md
index 776c585480..73f0d86e8a 100644
--- a/README_ID.md
+++ b/README_ID.md
@@ -24,7 +24,7 @@
   <a href="#untuk-pengembang-dan-organisasi">Untuk Pengembang</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">Dokumentasi</a> •
   <a href="#kredit">Kredit</a> •
-  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">Tanya Jawab</a> •
+  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">Tanya Jawab</a> •
   <a href="https://discord.gg/projectdiscovery">Gabung Discord</a>
 </p>
 
diff --git a/README_KR.md b/README_KR.md
index 394ece3349..db6820dd64 100644
--- a/README_KR.md
+++ b/README_KR.md
@@ -23,7 +23,7 @@
   <a href="#개발자를-위한">개발자를 위한</a> •
   <a href="https://nuclei.projectdiscovery.io/nuclei/get-started/">문서</a> •
   <a href="#credits">Credits</a> •
-  <a href="https://nuclei.projectdiscovery.io/faq/nuclei/">FAQs</a> •
+  <a href="https://docs.projectdiscovery.io/tools/nuclei/faq">FAQs</a> •
   <a href="https://discord.gg/projectdiscovery"> Discord 참가</a>
 </p>
 

From a4ba5cd1cb0be308fb5529ae81f2dab5ddde4c37 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Thu, 18 Apr 2024 15:06:15 +0530
Subject: [PATCH 34/58] Fix panic with template validation (#5065)

---
 pkg/utils/insertion_ordered_map.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pkg/utils/insertion_ordered_map.go b/pkg/utils/insertion_ordered_map.go
index 0e3b495d9b..20e6367362 100644
--- a/pkg/utils/insertion_ordered_map.go
+++ b/pkg/utils/insertion_ordered_map.go
@@ -40,6 +40,9 @@ func (insertionOrderedStringMap *InsertionOrderedStringMap) UnmarshalYAML(unmars
 	}
 	insertionOrderedStringMap.values = make(map[string]interface{})
 	for _, v := range data {
+		if v.Key == nil {
+			continue
+		}
 		insertionOrderedStringMap.Set(v.Key.(string), toString(v.Value))
 	}
 	return nil

From 3a3db67248937af9c5406105411effd404493cd0 Mon Sep 17 00:00:00 2001
From: Ice3man <nizamulrana@gmail.com>
Date: Thu, 18 Apr 2024 16:49:28 +0530
Subject: [PATCH 35/58] feat: katana jsonl input format not working fix (#5063)

---
 pkg/input/formats/json/json.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/pkg/input/formats/json/json.go b/pkg/input/formats/json/json.go
index fecb1c6b99..69e628c684 100644
--- a/pkg/input/formats/json/json.go
+++ b/pkg/input/formats/json/json.go
@@ -28,9 +28,10 @@ var _ formats.Format = &JSONFormat{}
 type proxifyRequest struct {
 	URL     string `json:"url"`
 	Request struct {
-		Header map[string]string `json:"header"`
-		Body   string            `json:"body"`
-		Raw    string            `json:"raw"`
+		Header   map[string]string `json:"header"`
+		Body     string            `json:"body"`
+		Raw      string            `json:"raw"`
+		Endpoint string            `json:"endpoint"`
 	} `json:"request"`
 }
 
@@ -63,6 +64,9 @@ func (j *JSONFormat) Parse(input string, resultsCb formats.ParseReqRespCallback)
 			return errors.Wrap(err, "could not decode json file")
 		}
 
+		if request.URL == "" && request.Request.Endpoint != "" {
+			request.URL = request.Request.Endpoint
+		}
 		rawRequest, err := types.ParseRawRequestWithURL(request.Request.Raw, request.URL)
 		if err != nil {
 			gologger.Warning().Msgf("jsonl: Could not parse raw request %s: %s\n", request.URL, err)

From 66da73c1b1f320817a2a7d33404de5c8f9435fb6 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Thu, 18 Apr 2024 17:43:46 +0530
Subject: [PATCH 36/58] Fix panic err using flow templates with workflow
 (#5064)

* Fix panic err using flow templates with workflows

* Misc update

* skip test if pdcp keys are not present

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
---
 pkg/input/provider/list/hmap_test.go |  8 ++++++++
 pkg/tmplexec/exec.go                 | 22 +++++++++++++++++++---
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/pkg/input/provider/list/hmap_test.go b/pkg/input/provider/list/hmap_test.go
index c3fb4a6699..95fc57f2d9 100644
--- a/pkg/input/provider/list/hmap_test.go
+++ b/pkg/input/provider/list/hmap_test.go
@@ -13,6 +13,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/expand"
+	"github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/stretchr/testify/require"
 )
 
@@ -153,6 +154,13 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
 }
 
 func Test_expandASNInputValue(t *testing.T) {
+	// skip this test if pdcp keys are not present
+	h := pdcp.PDCPCredHandler{}
+	creds, err := h.GetCreds()
+	if err != nil || creds == nil || creds.APIKey == "" {
+		t.Logf("Skipping asnmap test as pdcp keys are not present")
+		t.SkipNow()
+	}
 	tests := []struct {
 		asn                string
 		expectedOutputFile string
diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go
index fc504171e6..c510131da2 100644
--- a/pkg/tmplexec/exec.go
+++ b/pkg/tmplexec/exec.go
@@ -205,9 +205,25 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
-	err := e.engine.ExecuteWithResults(ctx)
-	ctx.LogError(err)
-	return ctx.GenerateResult(), err
+	var errx error
+	if e.options.Flow != "" {
+		flowexec, err := flow.NewFlowExecutor(e.requests, ctx, e.options, e.results, e.program)
+		if err != nil {
+			ctx.LogError(err)
+			return nil, fmt.Errorf("could not create flow executor: %s", err)
+		}
+		if err := flowexec.Compile(); err != nil {
+			ctx.LogError(err)
+			return nil, err
+		}
+		errx = flowexec.ExecuteWithResults(ctx)
+	} else {
+		errx = e.engine.ExecuteWithResults(ctx)
+	}
+	if errx != nil {
+		ctx.LogError(errx)
+	}
+	return ctx.GenerateResult(), errx
 }
 
 // getTemplateType returns the template type of the template

From bf0cae3a108773c260debab825f9ee4a8a2b71c6 Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Tue, 23 Apr 2024 13:39:35 +0530
Subject: [PATCH 37/58] Fix panic with fuzz template (#5068)

* Fix panic with fuzz template

* Fix multiple mode in fuzzing

* Add test
---
 cmd/integration-test/fuzz.go                | 27 +++++++++++++++++++++
 integration_tests/fuzz/fuzz-multi-mode.yaml | 27 +++++++++++++++++++++
 pkg/fuzz/component/headers.go               |  3 +++
 pkg/fuzz/execute.go                         |  8 ++++++
 4 files changed, 65 insertions(+)
 create mode 100644 integration_tests/fuzz/fuzz-multi-mode.yaml

diff --git a/cmd/integration-test/fuzz.go b/cmd/integration-test/fuzz.go
index f5e71774d0..e4834d4b96 100644
--- a/cmd/integration-test/fuzz.go
+++ b/cmd/integration-test/fuzz.go
@@ -18,6 +18,7 @@ const (
 
 var fuzzingTestCases = []TestCaseInfo{
 	{Path: "fuzz/fuzz-mode.yaml", TestCase: &fuzzModeOverride{}},
+	{Path: "fuzz/fuzz-multi-mode.yaml", TestCase: &fuzzMultipleMode{}},
 	{Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}},
 	{Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}},
 	{Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}},
@@ -174,3 +175,29 @@ func (h *HeadlessFuzzingQuery) Execute(filePath string) error {
 	}
 	return expectResultsCount(got, 2)
 }
+
+type fuzzMultipleMode struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *fuzzMultipleMode) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		xClientId := r.Header.Get("X-Client-Id")
+		xSecretId := r.Header.Get("X-Secret-Id")
+		if xClientId != "nuclei-v3" || xSecretId != "nuclei-v3" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html")
+		resp := fmt.Sprintf("<html><body><h1>This is multi-mode fuzzing test: %v <h1></body></html>", xClientId)
+		fmt.Fprint(w, resp)
+	})
+	ts := httptest.NewTLSServer(router)
+	defer ts.Close()
+
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?url=https://scanme.sh", debug, "-jsonl", "-fuzz")
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(got, 1)
+}
\ No newline at end of file
diff --git a/integration_tests/fuzz/fuzz-multi-mode.yaml b/integration_tests/fuzz/fuzz-multi-mode.yaml
new file mode 100644
index 0000000000..b8a15aa2ec
--- /dev/null
+++ b/integration_tests/fuzz/fuzz-multi-mode.yaml
@@ -0,0 +1,27 @@
+id: fuzz-multi-mode-test
+
+info:
+  name: multi-mode fuzzing test
+  author: pdteam
+  severity: info
+
+http:
+  - payloads:
+      inject:
+        - nuclei-v1
+        - nuclei-v2
+        - nuclei-v3
+
+    fuzzing:
+      - part: header
+        type: replace
+        mode: multiple
+        fuzz:
+          X-Client-Id: "{{inject}}"
+          X-Secret-Id: "{{inject}}"
+          
+    matchers-condition: or
+    matchers:
+      - type: word
+        words:
+          - "nuclei-v3"
\ No newline at end of file
diff --git a/pkg/fuzz/component/headers.go b/pkg/fuzz/component/headers.go
index fa0f5e1126..fd356b6314 100644
--- a/pkg/fuzz/component/headers.go
+++ b/pkg/fuzz/component/headers.go
@@ -83,6 +83,9 @@ func (q *Header) Delete(key string) error {
 func (q *Header) Rebuild() (*retryablehttp.Request, error) {
 	cloned := q.req.Clone(context.Background())
 	q.value.parsed.Iterate(func(key string, value any) bool {
+		if strings.TrimSpace(key) == "" {
+			return true
+		}
 		if strings.EqualFold(key, "Host") {
 			return true
 		}
diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go
index c8054bf80b..23b3e6e976 100644
--- a/pkg/fuzz/execute.go
+++ b/pkg/fuzz/execute.go
@@ -211,6 +211,14 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo
 		})
 		// if mode is multiple now build and execute it
 		if rule.modeType == multipleModeType {
+			rule.Fuzz.KV.Iterate(func(key, value string) bool {
+				var evaluated string
+				evaluated, input.InteractURLs = rule.executeEvaluate(input, key, "", value, input.InteractURLs)
+				if err := ruleComponent.SetValue(key, evaluated); err != nil {
+					return true
+				}
+				return true
+			})
 			req, err := ruleComponent.Rebuild()
 			if err != nil {
 				return err

From 6cd0d0517415a2c20aeba628df2c8fb45231cdb9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 23 Apr 2024 13:41:47 +0530
Subject: [PATCH 38/58] chore(deps): bump
 github.com/projectdiscovery/retryablehttp-go (#5073)

Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.55 to 1.0.57.
- [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases)
- [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.55...v1.0.57)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/retryablehttp-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod |  4 ++--
 go.sum | 10 ++++------
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/go.mod b/go.mod
index 4cf94ec00c..07f3bf8a38 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
 	github.com/projectdiscovery/interactsh v1.1.9
 	github.com/projectdiscovery/rawhttp v0.1.45
 	github.com/projectdiscovery/retryabledns v1.0.58
-	github.com/projectdiscovery/retryablehttp-go v1.0.55
+	github.com/projectdiscovery/retryablehttp-go v1.0.57
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -94,7 +94,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.40
-	github.com/projectdiscovery/utils v0.0.88
+	github.com/projectdiscovery/utils v0.0.89
 	github.com/projectdiscovery/wappalyzergo v0.0.116
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index 888602a714..7e6a96bae7 100644
--- a/go.sum
+++ b/go.sum
@@ -341,8 +341,6 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
 github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwSmwnCg4bxg=
-github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
 github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
 github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -876,8 +874,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
 github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
-github.com/projectdiscovery/retryablehttp-go v1.0.55 h1:ADgugnl9jKkNXn5m/Zd8TGPq1P7GplYlqUNKm/qTmls=
-github.com/projectdiscovery/retryablehttp-go v1.0.55/go.mod h1:Kpvh4ruFPOEPYaYxgbFmlvBJr4lJKqpcbGvx1j0r/Ng=
+github.com/projectdiscovery/retryablehttp-go v1.0.57 h1:OGfUXKXV4bE5msGxeRrNtMaDg2l8U1JcLXmwG7yXWrY=
+github.com/projectdiscovery/retryablehttp-go v1.0.57/go.mod h1:Lo2EU1wV1draQ/dHuiSkokW4gZ216F/qi/t12DIdMbA=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -888,8 +886,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
 github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs=
-github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
+github.com/projectdiscovery/utils v0.0.89 h1:ruH2bSkpX/rB7EPp2EV/rWyAubQVxCVU38nRcLp4L1w=
+github.com/projectdiscovery/utils v0.0.89/go.mod h1:Dwh5cxn7y97jvyYG3GmBvj0negfH9IjH15qXnzFNtOI=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
 github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

From 60b9d4db4bb5033401fd30c20fd9e6ebebd9c292 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 23 Apr 2024 13:41:55 +0530
Subject: [PATCH 39/58] chore(deps): bump github.com/projectdiscovery/goflags
 (#5074)

Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.48 to 0.1.49.
- [Release notes](https://github.com/projectdiscovery/goflags/releases)
- [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.48...v0.1.49)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/goflags
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 07f3bf8a38..30ddbc6ecb 100644
--- a/go.mod
+++ b/go.mod
@@ -81,7 +81,7 @@ require (
 	github.com/projectdiscovery/dsl v0.0.52
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
-	github.com/projectdiscovery/goflags v0.1.48
+	github.com/projectdiscovery/goflags v0.1.49
 	github.com/projectdiscovery/gologger v1.1.12
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.2
diff --git a/go.sum b/go.sum
index 7e6a96bae7..dbcf574dc7 100644
--- a/go.sum
+++ b/go.sum
@@ -842,8 +842,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm
 github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.48 h1:iJgq+YuQOEm0V7BqlV/0wx7GaI2rmGiFzPkvL/4sCj0=
-github.com/projectdiscovery/goflags v0.1.48/go.mod h1:C9Qqyivehk5iYCf3VAC3+85y60Qbde1ZBMqe5wHhcVw=
+github.com/projectdiscovery/goflags v0.1.49 h1:0e9wya431WDeVm8ZtlyqBQ+rwnhDjUswDMcS0did9Tg=
+github.com/projectdiscovery/goflags v0.1.49/go.mod h1:f0zRbaa5QLrjfJQ5v0efvq8EhkDGhCm9h0hsahjjKFc=
 github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
 github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=

From 2fc1be213a6c6914e3368248ea24eb258400881c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 23 Apr 2024 13:42:19 +0530
Subject: [PATCH 40/58] chore(deps): bump github.com/projectdiscovery/ratelimit
 (#5077)

Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.35 to 0.0.38.
- [Release notes](https://github.com/projectdiscovery/ratelimit/releases)
- [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.35...v0.0.38)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/ratelimit
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 30ddbc6ecb..2854eee23c 100644
--- a/go.mod
+++ b/go.mod
@@ -88,7 +88,7 @@ require (
 	github.com/projectdiscovery/httpx v1.6.0
 	github.com/projectdiscovery/mapcidr v1.1.16
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
-	github.com/projectdiscovery/ratelimit v0.0.35
+	github.com/projectdiscovery/ratelimit v0.0.38
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6
diff --git a/go.sum b/go.sum
index dbcf574dc7..6eda8f7d0b 100644
--- a/go.sum
+++ b/go.sum
@@ -866,8 +866,8 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
 github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
-github.com/projectdiscovery/ratelimit v0.0.35 h1:epEzFATOcXZ4tssV4Hax5Op9lrbUnQMEGMV5PoUpTKc=
-github.com/projectdiscovery/ratelimit v0.0.35/go.mod h1:mPqa8UpV5I7eAN5/ZcsjLiXMhjtVvZRrHtpBRsTPuyA=
+github.com/projectdiscovery/ratelimit v0.0.38 h1:TCcXXjL3LwB+Es1dtW50FSoQsAQBHH0PqQ2VGZhp50U=
+github.com/projectdiscovery/ratelimit v0.0.38/go.mod h1:ig2wgB2qY2YKFKu0Lm1R/AxjXsGyFceEvC+ATAMOFgc=
 github.com/projectdiscovery/rawhttp v0.1.45 h1:5jssUybhRz2ljI3bW4kUO9MTvZ37ArG8pISYtgqzwBg=
 github.com/projectdiscovery/rawhttp v0.1.45/go.mod h1:oLWuZkyKi4rddPoLEN3TeAqyLvFNfr3dxmzV2kwpPvs=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=

From 5dacb255ac0b8557fd73da8fe76f43f9749dc9a8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 23 Apr 2024 14:41:46 +0530
Subject: [PATCH 41/58] chore(deps): bump github.com/projectdiscovery/useragent
 (#5075)

Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.40 to 0.0.47.
- [Release notes](https://github.com/projectdiscovery/useragent/releases)
- [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.40...v0.0.47)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/useragent
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 2854eee23c..6dbdbf7dae 100644
--- a/go.mod
+++ b/go.mod
@@ -93,7 +93,7 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
-	github.com/projectdiscovery/useragent v0.0.40
+	github.com/projectdiscovery/useragent v0.0.47
 	github.com/projectdiscovery/utils v0.0.89
 	github.com/projectdiscovery/wappalyzergo v0.0.116
 	github.com/redis/go-redis/v9 v9.1.0
diff --git a/go.sum b/go.sum
index 6eda8f7d0b..4c781f73ce 100644
--- a/go.sum
+++ b/go.sum
@@ -884,8 +884,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
 github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
 github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
-github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
-github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
+github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0=
+github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M=
 github.com/projectdiscovery/utils v0.0.89 h1:ruH2bSkpX/rB7EPp2EV/rWyAubQVxCVU38nRcLp4L1w=
 github.com/projectdiscovery/utils v0.0.89/go.mod h1:Dwh5cxn7y97jvyYG3GmBvj0negfH9IjH15qXnzFNtOI=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=

From 61e9be530f3c9c6a27cd9b862c4d7e446f7ed7af Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Tue, 23 Apr 2024 16:04:32 +0530
Subject: [PATCH 42/58] Fix: `skip-variables-check` option in self-contained
 templates (#5053)

* fix: skip-variables-check option in self-contained templates

* Update build workflow envs
---
 .github/workflows/build-test.yml    | 2 ++
 pkg/protocols/http/build_request.go | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
index aa0d8aecfe..75591aa8b8 100644
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@@ -42,6 +42,7 @@ jobs:
       - name: Test
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
         run: go test ./...
 
       - name: Integration Tests
@@ -49,6 +50,7 @@ jobs:
         env:
           GH_ACTION: true
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
         run: |
           chmod +x run.sh
           bash run.sh ${{ matrix.os }}
diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go
index bae9a37af0..18b75c72bd 100644
--- a/pkg/protocols/http/build_request.go
+++ b/pkg/protocols/http/build_request.go
@@ -277,7 +277,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 			return nil, fmt.Errorf("malformed request supplied")
 		}
 
-		if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
+		if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil && !r.request.SkipVariablesCheck {
 			return nil, ErrUnresolvedVars.Msgf(parts[1])
 		}
 
@@ -296,7 +296,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 		}
 		return r.generateRawRequest(ctx, data, parsed, values, payloads)
 	}
-	if err := expressions.ContainsUnresolvedVariables(data); err != nil {
+	if err := expressions.ContainsUnresolvedVariables(data); err != nil && !r.request.SkipVariablesCheck {
 		// early exit: if there are any unresolved variables in `path` after evaluation
 		// then return early since this will definitely fail
 		return nil, ErrUnresolvedVars.Msgf(data)

From 465894df159c1ebf5a0ede6f3eef56c95a36b952 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Tue, 23 Apr 2024 13:34:52 +0300
Subject: [PATCH 43/58] disable thread count warning upon validate (#5078)

---
 internal/runner/runner.go  |  2 ++
 pkg/protocols/http/http.go | 12 ++++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index 174c06f0f7..ce8cf4030c 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -52,6 +52,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
+	httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -684,6 +685,7 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 	} else {
 		stats.DisplayAsWarning(templates.SkippedCodeTmplTamperedStats)
 	}
+	stats.DisplayAsWarning(httpProtocol.SetThreadToCountZero)
 	stats.ForceDisplayWarning(templates.SkippedUnsignedStats)
 	stats.ForceDisplayWarning(templates.SkippedRequestSignatureStats)
 
diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go
index b98fd917a5..c20cbbfa66 100644
--- a/pkg/protocols/http/http.go
+++ b/pkg/protocols/http/http.go
@@ -9,7 +9,6 @@ import (
 	json "github.com/json-iterator/go"
 	"github.com/pkg/errors"
 
-	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
@@ -19,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
 	"github.com/projectdiscovery/rawhttp"
 	"github.com/projectdiscovery/retryablehttp-go"
 	fileutil "github.com/projectdiscovery/utils/file"
@@ -455,7 +455,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 			}
 		}
 		if hasNamedInternalExtractor && hasMultipleRequests {
-			gologger.Warning().Label(options.TemplateID).Msgf("Setting thread count to 0 because dynamic extractors are not supported with payloads yet")
+			stats.Increment(SetThreadToCountZero)
 			request.Threads = 0
 		} else {
 			// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
@@ -490,3 +490,11 @@ func (request *Request) Requests() int {
 	}
 	return len(request.Path)
 }
+
+const (
+	SetThreadToCountZero = "set-thread-count-to-zero"
+)
+
+func init() {
+	stats.NewEntry(SetThreadToCountZero, "Setting thread count to 0 for %d templates, dynamic extractors are not supported with payloads yet")
+}

From ea3705eb64188c1389b1d98adb91eb16e731bfdd Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Tue, 23 Apr 2024 14:26:33 +0300
Subject: [PATCH 44/58] fix openapi import nil deref (#5080)

---
 pkg/input/formats/openapi/generator.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkg/input/formats/openapi/generator.go b/pkg/input/formats/openapi/generator.go
index 090b761c3a..6f10c56d09 100644
--- a/pkg/input/formats/openapi/generator.go
+++ b/pkg/input/formats/openapi/generator.go
@@ -179,6 +179,10 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
 	for _, parameter := range reqParams {
 		value := parameter.Value
 
+		if value.Schema == nil || value.Schema.Value == nil {
+			continue
+		}
+
 		// paramValue or default value to use
 		var paramValue interface{}
 

From 4b9c3b8d16ceca39f9892550467bca2fa1a68635 Mon Sep 17 00:00:00 2001
From: scottdharvey <scottharvey@gmail.com>
Date: Tue, 23 Apr 2024 06:41:12 -0700
Subject: [PATCH 45/58] Feat 5059 (#5060)

* change catalog

* add usesuppliedcatalog option

* add catalog nil check

* Update config.go
---
 lib/config.go      | 9 +++++++++
 lib/sdk.go         | 4 ++--
 lib/sdk_private.go | 4 +++-
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/lib/config.go b/lib/config.go
index ddcf3b88b4..a453fb74c7 100644
--- a/lib/config.go
+++ b/lib/config.go
@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/ratelimit"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
@@ -425,3 +426,11 @@ func SignedTemplatesOnly() NucleiSDKOptions {
 		return nil
 	}
 }
+
+// WithCatalog uses a supplied catalog
+func WithCatalog(cat catalog.Catalog) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.catalog = cat
+		return nil
+	}
+}
diff --git a/lib/sdk.go b/lib/sdk.go
index 4bee921c31..c80abb0615 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -6,7 +6,7 @@ import (
 	"io"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
-	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
@@ -63,7 +63,7 @@ type NucleiEngine struct {
 
 	// unexported core fields
 	interactshClient *interactsh.Client
-	catalog          *disk.DiskCatalog
+	catalog          catalog.Catalog
 	rateLimiter      *ratelimit.Limiter
 	store            *loader.Store
 	httpxClient      providerTypes.InputLivenessProbe
diff --git a/lib/sdk_private.go b/lib/sdk_private.go
index c76970c91d..89628cc933 100644
--- a/lib/sdk_private.go
+++ b/lib/sdk_private.go
@@ -146,7 +146,9 @@ func (e *NucleiEngine) init() error {
 		return err
 	}
 
-	e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
+	if e.catalog == nil {
+		e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
+	}
 
 	e.executerOpts = protocols.ExecutorOptions{
 		Output:          e.customWriter,

From e480d131f1b8545070fb0e7fef7a4aaac423d485 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Tue, 23 Apr 2024 16:47:26 +0300
Subject: [PATCH 46/58] add query param (#4894)

* include params

* add query var

* override params for base url var
---
 pkg/protocols/utils/variables.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/pkg/protocols/utils/variables.go b/pkg/protocols/utils/variables.go
index d1810d19b5..90c786923b 100644
--- a/pkg/protocols/utils/variables.go
+++ b/pkg/protocols/utils/variables.go
@@ -24,6 +24,7 @@ func init() {
 		Host:     "Host",
 		Port:     "Port",
 		Path:     "Path",
+		Query:    "Query",
 		File:     "File",
 		Scheme:   "Scheme",
 		Input:    "Input",
@@ -44,6 +45,7 @@ const (
 	Host
 	Port
 	Path
+	Query
 	File
 	Scheme
 	Input
@@ -162,6 +164,12 @@ func generateVariables(inputURL *urlutil.URL, removeTrailingSlash bool) map[stri
 			knownVariables[v] = port
 		case Path:
 			knownVariables[v] = requestPath
+		case Query:
+			if queryParams := urlutil.GetParams(parsed.URL.Query()); len(queryParams) > 0 {
+				knownVariables[v] = "?" + queryParams.Encode()
+			} else {
+				knownVariables[v] = ""
+			}
 		case File:
 			knownVariables[v] = base
 		case Scheme:

From 8676cb6daf83f6dee18899f13cdb7faf0a0fbf69 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Wed, 24 Apr 2024 10:35:05 +0300
Subject: [PATCH 47/58] add response read timeout flag (#4944)

* add response read timeout flag

* fix test

* bump utils

* fix network tests

* fix incorrect unit of response-read-timeout unit

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
---
 cmd/nuclei/main.go               |  1 +
 pkg/protocols/network/request.go | 13 ++-----------
 pkg/testutils/testutils.go       |  1 +
 pkg/types/types.go               |  3 +++
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index a90c4ed52e..9b68dc11b6 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -298,6 +298,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"),
 		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 10*1024*1024, "max response size to read in bytes"),
 		flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"),
+		flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"),
 		flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
 		flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
 	)
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 146c657072..1f51ebe119 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -36,12 +36,6 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
-var (
-	// TODO: make this configurable
-	// DefaultReadTimeout is the default read timeout for network requests
-	DefaultReadTimeout = time.Duration(5) * time.Second
-)
-
 var _ protocols.Request = &Request{}
 
 // Type returns the type of the protocol request
@@ -295,7 +289,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		}
 
 		if input.Read > 0 {
-			buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), DefaultReadTimeout)
+			buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.ResponseReadTimeout)
 			if err != nil {
 				return errorutil.NewWithErr(err).Msgf("could not read response from connection")
 			}
@@ -345,7 +339,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		bufferSize = -1
 	}
 
-	final, err := ConnReadNWithTimeout(conn, int64(bufferSize), DefaultReadTimeout)
+	final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.Options.ResponseReadTimeout)
 	if err != nil {
 		request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
 		gologger.Verbose().Msgf("could not read more data from %s: %s", actualAddress, err)
@@ -446,9 +440,6 @@ func getAddress(toTest string) (string, error) {
 }
 
 func ConnReadNWithTimeout(conn net.Conn, n int64, timeout time.Duration) ([]byte, error) {
-	if timeout == 0 {
-		timeout = DefaultReadTimeout
-	}
 	if n == -1 {
 		// if n is -1 then read all available data from connection
 		return reader.ConnReadNWithTimeout(conn, -1, timeout)
diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go
index e59aa015e2..faa21938bb 100644
--- a/pkg/testutils/testutils.go
+++ b/pkg/testutils/testutils.go
@@ -72,6 +72,7 @@ var DefaultOptions = &types.Options{
 	InteractionsPollDuration:   5,
 	GitHubTemplateRepo:         []string{},
 	GitHubToken:                "",
+	ResponseReadTimeout:        time.Second * 5,
 }
 
 // TemplateInfo contains info for a mock executed template.
diff --git a/pkg/types/types.go b/pkg/types/types.go
index b6bdcaee55..caa1ced75f 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -289,6 +289,8 @@ type Options struct {
 	ResponseReadSize int
 	// ResponseSaveSize is the maximum size of response to save
 	ResponseSaveSize int
+	// ResponseReadTimeout is response read timeout in seconds
+	ResponseReadTimeout time.Duration
 	// Health Check
 	HealthCheck bool
 	// Time to wait between each input read operation before closing the stream
@@ -427,6 +429,7 @@ func DefaultOptions() *Options {
 		MaxHostError:            30,
 		ResponseReadSize:        10 * 1024 * 1024,
 		ResponseSaveSize:        1024 * 1024,
+		ResponseReadTimeout:     5 * time.Second,
 	}
 }
 

From 515f7c12bb6b274f8fe90ce0c669992ca9fc29f5 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Wed, 24 Apr 2024 13:05:40 +0530
Subject: [PATCH 48/58] fix go install failing (#5083)

* Fix panic with fuzz template

* Fix multiple mode in fuzzing

* Add test

* remove fork: use official go-echarts

* bump lint action to v4

---------

Co-authored-by: Ramana Reddy <ramanaredy.manda@gmail.com>
---
 .github/workflows/lint-test.yml |  2 +-
 go.mod                          |  2 --
 go.sum                          |  2 --
 pkg/scan/charts/echarts.go      | 41 +++++++++++++++++++++++++--------
 4 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml
index 4292a42f19..8b4695dc83 100644
--- a/.github/workflows/lint-test.yml
+++ b/.github/workflows/lint-test.yml
@@ -21,7 +21,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.6.0
+        uses: golangci/golangci-lint-action@v4.0.0
         with:
           version: latest
           args: --timeout 5m
\ No newline at end of file
diff --git a/go.mod b/go.mod
index 6dbdbf7dae..7ffdcc20bd 100644
--- a/go.mod
+++ b/go.mod
@@ -349,5 +349,3 @@ require (
 
 // https://go.dev/ref/mod#go-mod-file-retract
 retract v3.2.0 // retract due to broken js protocol issue
-
-replace github.com/go-echarts/go-echarts/v2 => github.com/tarunKoyalwar/go-echarts/v2 v2.1.1
diff --git a/go.sum b/go.sum
index 4c781f73ce..07a7bc8d22 100644
--- a/go.sum
+++ b/go.sum
@@ -1013,8 +1013,6 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
-github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 h1:5fsXGPmK+i18J8cDgxy7AJkiXWBARpVTb0Gbv+bAzPo=
-github.com/tarunKoyalwar/go-echarts/v2 v2.1.1/go.mod h1:VEeyPT5Odx/UHeuxtIAHGu2+87MWGA5OBaZ120NFi/w=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
 github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
diff --git a/pkg/scan/charts/echarts.go b/pkg/scan/charts/echarts.go
index bf70fee710..a2a9815888 100644
--- a/pkg/scan/charts/echarts.go
+++ b/pkg/scan/charts/echarts.go
@@ -38,16 +38,18 @@ func (s *ScanEventsCharts) allCharts(c echo.Context) *components.Page {
 	page := components.NewPage()
 	page.PageTitle = "Nuclei Charts"
 	line1 := s.totalRequestsOverTime(c)
-	line1.SetSpacerHeight(SpacerHeight)
+	// line1.SetSpacerHeight(SpacerHeight)
 	kline := s.topSlowTemplates(c)
-	kline.SetSpacerHeight(SpacerHeight)
+	// kline.SetSpacerHeight(SpacerHeight)
 	line2 := s.requestsVSInterval(c)
-	line2.SetSpacerHeight(SpacerHeight)
+	// line2.SetSpacerHeight(SpacerHeight)
 	line3 := s.concurrencyVsTime(c)
-	line3.SetSpacerHeight(SpacerHeight)
+	// line3.SetSpacerHeight(SpacerHeight)
 	page.AddCharts(line1, kline, line2, line3)
-	page.Validate()
 	page.SetLayout(components.PageCenterLayout)
+	page.Theme = "dark"
+	page.Validate()
+
 	return page
 }
 
@@ -59,7 +61,12 @@ func (s *ScanEventsCharts) TotalRequestsOverTime(c echo.Context) error {
 // totalRequestsOverTime generates a line chart showing total requests count over time
 func (s *ScanEventsCharts) totalRequestsOverTime(c echo.Context) *charts.Line {
 	line := charts.NewLine()
-	line.SetCaption("Chart Shows Total Requests Count Over Time (for each/all Protocols)")
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{
+			Title:    "Nuclei: Total Requests vs Time",
+			Subtitle: "Chart Shows Total Requests Count Over Time (for each/all Protocols)",
+		}),
+	)
 
 	var startTime time.Time = time.Now()
 	var endTime time.Time
@@ -120,8 +127,12 @@ func (s *ScanEventsCharts) TopSlowTemplates(c echo.Context) error {
 // topSlowTemplates generates a Kline chart showing the top slow templates by time taken
 func (s *ScanEventsCharts) topSlowTemplates(c echo.Context) *charts.Kline {
 	kline := charts.NewKLine()
-	kline.SetCaption(fmt.Sprintf("Chart Shows Top Slow Templates (by time taken) (Top %v)", TopK))
-
+	kline.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{
+			Title:    "Nuclei: Top Slow Templates",
+			Subtitle: fmt.Sprintf("Chart Shows Top Slow Templates (by time taken) (Top %v)", TopK),
+		}),
+	)
 	ids := map[string][]int64{}
 	var startTime time.Time = time.Now()
 	for _, event := range s.data {
@@ -200,7 +211,12 @@ func (s *ScanEventsCharts) RequestsVSInterval(c echo.Context) error {
 // requestsVSInterval generates a line chart showing requests per second over time
 func (s *ScanEventsCharts) requestsVSInterval(c echo.Context) *charts.Line {
 	line := charts.NewLine()
-	line.SetCaption("Chart Shows RPS (Requests Per Second) Over Time")
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{
+			Title:    "Nuclei: Requests Per Second vs Time",
+			Subtitle: "Chart Shows RPS (Requests Per Second) Over Time",
+		}),
+	)
 
 	sort.Slice(s.data, func(i, j int) bool {
 		return s.data[i].Time.Before(s.data[j].Time)
@@ -267,7 +283,12 @@ func (s *ScanEventsCharts) ConcurrencyVsTime(c echo.Context) error {
 // concurrencyVsTime generates a line chart showing concurrency (total workers) over time
 func (s *ScanEventsCharts) concurrencyVsTime(c echo.Context) *charts.Line {
 	line := charts.NewLine()
-	line.SetCaption("Chart Shows Concurrency (Total Workers) Over Time")
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{
+			Title:    "Nuclei: Concurrency vs Time",
+			Subtitle: "Chart Shows Concurrency (Total Workers) Over Time",
+		}),
+	)
 
 	dataset := sliceutil.Clone(s.data)
 

From cbe7322019dda42f9d1aee61120e3073ee49fada Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 24 Apr 2024 08:36:04 +0100
Subject: [PATCH 49/58] Exposing embedded api for settings control in CLI
 modality (#5030)

* exposing settings api

* adding probe concurrency

* adding js pool size control

* adding json tags
---
 cmd/nuclei/main.go              |   1 +
 internal/httpapi/apiendpoint.go | 112 ++++++++++++++++++++++++++++++++
 internal/runner/inputs.go       |  15 +----
 internal/runner/runner.go       |  17 ++++-
 pkg/testutils/testutils.go      |   1 +
 pkg/types/types.go              |   2 +
 6 files changed, 134 insertions(+), 14 deletions(-)
 create mode 100644 internal/httpapi/apiendpoint.go

diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index 9b68dc11b6..a084f270b4 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -301,6 +301,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"),
 		flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
 		flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
+		flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
 	)
 
 	flagSet.CreateGroup("interactsh", "interactsh",
diff --git a/internal/httpapi/apiendpoint.go b/internal/httpapi/apiendpoint.go
new file mode 100644
index 0000000000..2cd055eeea
--- /dev/null
+++ b/internal/httpapi/apiendpoint.go
@@ -0,0 +1,112 @@
+package httpapi
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+)
+
+type Concurrency struct {
+	BulkSize              int    `json:"bulk_size"`
+	Threads               int    `json:"threads"`
+	RateLimit             int    `json:"rate_limit"`
+	RateLimitDuration     string `json:"rate_limit_duration"`
+	PayloadConcurrency    int    `json:"payload_concurrency"`
+	ProbeConcurrency      int    `json:"probe_concurrency"`
+	JavascriptConcurrency int    `json:"javascript_concurrency"`
+}
+
+// Server represents the HTTP server that handles the concurrency settings endpoints.
+type Server struct {
+	addr   string
+	config *types.Options
+}
+
+// New creates a new instance of Server.
+func New(addr string, config *types.Options) *Server {
+	return &Server{
+		addr:   addr,
+		config: config,
+	}
+}
+
+// Start initializes the server and its routes, then starts listening on the specified address.
+func (s *Server) Start() error {
+	http.HandleFunc("/api/concurrency", s.handleConcurrency)
+	if err := http.ListenAndServe(s.addr, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+// handleConcurrency routes the request based on its method to the appropriate handler.
+func (s *Server) handleConcurrency(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case http.MethodGet:
+		s.getSettings(w, r)
+	case http.MethodPut:
+		s.updateSettings(w, r)
+	default:
+		http.Error(w, "Unsupported HTTP method", http.StatusMethodNotAllowed)
+	}
+}
+
+// GetSettings handles GET requests and returns the current concurrency settings
+func (s *Server) getSettings(w http.ResponseWriter, _ *http.Request) {
+	concurrencySettings := Concurrency{
+		BulkSize:              s.config.BulkSize,
+		Threads:               s.config.TemplateThreads,
+		RateLimit:             s.config.RateLimit,
+		RateLimitDuration:     s.config.RateLimitDuration.String(),
+		PayloadConcurrency:    s.config.PayloadConcurrency,
+		ProbeConcurrency:      s.config.ProbeConcurrency,
+		JavascriptConcurrency: compiler.PoolingJsVmConcurrency,
+	}
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(concurrencySettings); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+// UpdateSettings handles PUT requests to update the concurrency settings
+func (s *Server) updateSettings(w http.ResponseWriter, r *http.Request) {
+	var newSettings Concurrency
+	if err := json.NewDecoder(r.Body).Decode(&newSettings); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if newSettings.RateLimitDuration != "" {
+		if duration, err := time.ParseDuration(newSettings.RateLimitDuration); err == nil {
+			s.config.RateLimitDuration = duration
+		} else {
+			http.Error(w, "Invalid duration format", http.StatusBadRequest)
+			return
+		}
+	}
+	if newSettings.BulkSize > 0 {
+		s.config.BulkSize = newSettings.BulkSize
+	}
+	if newSettings.Threads > 0 {
+		s.config.TemplateThreads = newSettings.Threads
+	}
+	if newSettings.RateLimit > 0 {
+		s.config.RateLimit = newSettings.RateLimit
+	}
+	if newSettings.PayloadConcurrency > 0 {
+		s.config.PayloadConcurrency = newSettings.PayloadConcurrency
+	}
+	if newSettings.ProbeConcurrency > 0 {
+		s.config.ProbeConcurrency = newSettings.ProbeConcurrency
+	}
+	if newSettings.JavascriptConcurrency > 0 {
+		compiler.PoolingJsVmConcurrency = newSettings.JavascriptConcurrency
+		s.config.JsConcurrency = newSettings.JavascriptConcurrency // no-op on speed change
+	}
+
+	w.WriteHeader(http.StatusOK)
+}
diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go
index 87fbb573e1..047360aee3 100644
--- a/internal/runner/inputs.go
+++ b/internal/runner/inputs.go
@@ -16,8 +16,6 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
-var GlobalProbeBulkSize = 50
-
 // initializeTemplatesHTTPInput initializes the http form of input
 // for any loaded http templates if input is in non-standard format.
 func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
@@ -31,11 +29,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	gologger.Info().Msgf("Running httpx on input host")
 
-	var bulkSize = GlobalProbeBulkSize
-	if r.options.BulkSize > GlobalProbeBulkSize {
-		bulkSize = r.options.BulkSize
-	}
-
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
@@ -45,10 +38,8 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 		return nil, errors.Wrap(err, "could not create httpx client")
 	}
 
-	shouldFollowGlobalProbeBulkSize := bulkSize == GlobalProbeBulkSize
-
 	// Probe the non-standard URLs and store them in cache
-	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
+	swg, err := syncutil.New(syncutil.WithSize(r.options.BulkSize))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create adaptive group")
 	}
@@ -58,8 +49,8 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 			return true
 		}
 
-		if shouldFollowGlobalProbeBulkSize && swg.Size != GlobalProbeBulkSize {
-			swg.Resize(GlobalProbeBulkSize)
+		if r.options.ProbeConcurrency > 0 && swg.Size != r.options.ProbeConcurrency {
+			swg.Resize(r.options.ProbeConcurrency)
 		}
 
 		swg.Add()
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index ce8cf4030c..ab2fe5de93 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -32,6 +32,7 @@ import (
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v3/internal/httpapi"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
@@ -89,8 +90,9 @@ type Runner struct {
 	pdcpUploadErrMsg string
 	inputProvider    provider.InputProvider
 	//general purpose temporary directory
-	tmpDir string
-	parser parser.Parser
+	tmpDir          string
+	parser          parser.Parser
+	httpApiEndpoint *httpapi.Server
 }
 
 const pprofServerAddress = "127.0.0.1:8086"
@@ -222,6 +224,17 @@ func New(options *types.Options) (*Runner, error) {
 		}()
 	}
 
+	if options.HttpApiEndpoint != "" {
+		apiServer := httpapi.New(options.HttpApiEndpoint, options)
+		gologger.Info().Msgf("Listening api endpoint on: %s", options.HttpApiEndpoint)
+		runner.httpApiEndpoint = apiServer
+		go func() {
+			if err := apiServer.Start(); err != nil {
+				gologger.Error().Msgf("Failed to start API server: %s", err)
+			}
+		}()
+	}
+
 	if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
 		os.Exit(0)
 	}
diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go
index faa21938bb..68410f7f1d 100644
--- a/pkg/testutils/testutils.go
+++ b/pkg/testutils/testutils.go
@@ -55,6 +55,7 @@ var DefaultOptions = &types.Options{
 	Retries:                    1,
 	RateLimit:                  150,
 	RateLimitDuration:          time.Second,
+	ProbeConcurrency:           50,
 	ProjectPath:                "",
 	Severities:                 severity.Severities{},
 	Targets:                    []string{},
diff --git a/pkg/types/types.go b/pkg/types/types.go
index caa1ced75f..d7dd41f22d 100644
--- a/pkg/types/types.go
+++ b/pkg/types/types.go
@@ -391,6 +391,8 @@ type Options struct {
 	ProbeConcurrency int
 	// Dast only runs DAST templates
 	DAST bool
+	// HttpApiEndpoint is the experimental http api endpoint
+	HttpApiEndpoint string
 }
 
 // ShouldLoadResume resume file

From d85267df31a48b3786a0bdcf48d23d6afe056618 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 24 Apr 2024 13:32:18 +0530
Subject: [PATCH 50/58] chore(deps): bump github.com/projectdiscovery/mapcidr
 (#5076)

Bumps [github.com/projectdiscovery/mapcidr](https://github.com/projectdiscovery/mapcidr) from 1.1.16 to 1.1.34.
- [Release notes](https://github.com/projectdiscovery/mapcidr/releases)
- [Changelog](https://github.com/projectdiscovery/mapcidr/blob/main/.goreleaser.yml)
- [Commits](https://github.com/projectdiscovery/mapcidr/compare/v1.1.16...v1.1.34)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/mapcidr
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 7ffdcc20bd..ca84f14b92 100644
--- a/go.mod
+++ b/go.mod
@@ -86,7 +86,7 @@ require (
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.2
 	github.com/projectdiscovery/httpx v1.6.0
-	github.com/projectdiscovery/mapcidr v1.1.16
+	github.com/projectdiscovery/mapcidr v1.1.34
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
 	github.com/projectdiscovery/ratelimit v0.0.38
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
diff --git a/go.sum b/go.sum
index 07a7bc8d22..e8de0c6ac6 100644
--- a/go.sum
+++ b/go.sum
@@ -860,8 +860,8 @@ github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:M
 github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk=
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
-github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo=
-github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q=
+github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM=
+github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
 github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=

From 21aec3062a8d2bdb2bbd8bdb6a915157b6a2cd6a Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Wed, 24 Apr 2024 14:03:06 +0300
Subject: [PATCH 51/58] update utils (#5089)

---
 go.mod | 4 ++--
 go.sum | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index ca84f14b92..35a39ccc08 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.67
+	github.com/projectdiscovery/fastdialer v0.0.68
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
 	github.com/projectdiscovery/rawhttp v0.1.45
@@ -94,7 +94,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
 	github.com/projectdiscovery/useragent v0.0.47
-	github.com/projectdiscovery/utils v0.0.89
+	github.com/projectdiscovery/utils v0.0.91
 	github.com/projectdiscovery/wappalyzergo v0.0.116
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
diff --git a/go.sum b/go.sum
index e8de0c6ac6..aed5e024e4 100644
--- a/go.sum
+++ b/go.sum
@@ -341,6 +341,7 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
 github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
 github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
 github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -836,6 +837,8 @@ github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7S
 github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
 github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E=
 github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84=
+github.com/projectdiscovery/fastdialer v0.0.68 h1:JuIrr8aVGdGWkEwL4axsJWAWDY2uviSqBB0TCekeCOo=
+github.com/projectdiscovery/fastdialer v0.0.68/go.mod h1:asHSBFJgmwrXpiegcrcAgOyd/QewCVgeI4idH55+v7M=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@@ -888,6 +891,8 @@ github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLP
 github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M=
 github.com/projectdiscovery/utils v0.0.89 h1:ruH2bSkpX/rB7EPp2EV/rWyAubQVxCVU38nRcLp4L1w=
 github.com/projectdiscovery/utils v0.0.89/go.mod h1:Dwh5cxn7y97jvyYG3GmBvj0negfH9IjH15qXnzFNtOI=
+github.com/projectdiscovery/utils v0.0.91 h1:aHAAnC0qX9pJZrWq4Qpl2PSTYLrSCL1dm1QWLjprE2w=
+github.com/projectdiscovery/utils v0.0.91/go.mod h1:O/6U3ZoU+tNw4lKurdjyVMZPVXL5IYq0YeaDc15PRls=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
 github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

From e450dee03d9adac3f59fabba92243430b89b693e Mon Sep 17 00:00:00 2001
From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com>
Date: Wed, 24 Apr 2024 18:44:59 +0530
Subject: [PATCH 52/58] Preserve reference links as case-sensitive while
 unmarshalling (#5098)

---
 pkg/model/model_test.go                        | 4 ++--
 pkg/model/types/stringslice/stringslice_raw.go | 9 +++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/pkg/model/model_test.go b/pkg/model/model_test.go
index 3f87add2c9..cd2826405c 100644
--- a/pkg/model/model_test.go
+++ b/pkg/model/model_test.go
@@ -80,7 +80,7 @@ func TestUnmarshal(t *testing.T) {
 	templateName := "Test Template"
 	authors := []string{"forgedhallpass", "ice3man"}
 	tags := []string{"cve", "misc"}
-	references := []string{"http://test.com", "http://domain.com"}
+	references := []string{"http://test.com", "http://Domain.com"}
 
 	dynamicKey1 := "customDynamicKey1"
 	dynamicKey2 := "customDynamicKey2"
@@ -109,7 +109,7 @@ func TestUnmarshal(t *testing.T) {
   author: ` + strings.Join(authors, ", ") + `
   tags: ` + strings.Join(tags, ", ") + `
   severity: critical
-  reference: ` + strings.Join(references, ", ") + `
+  reference: ` + strings.Join(references, ",") + `
   metadata:
      ` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1].(string) + `
      ` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2].(string) + `
diff --git a/pkg/model/types/stringslice/stringslice_raw.go b/pkg/model/types/stringslice/stringslice_raw.go
index 7d9e470bfa..fa96c2cc3f 100644
--- a/pkg/model/types/stringslice/stringslice_raw.go
+++ b/pkg/model/types/stringslice/stringslice_raw.go
@@ -12,6 +12,15 @@ func (rawStringSlice *RawStringSlice) Normalize(value string) string {
 	return value
 }
 
+func (rawStringSlice *RawStringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	marshalledSlice, err := marshalStringToSlice(unmarshal)
+	if err != nil {
+		return err
+	}
+	rawStringSlice.Value = marshalledSlice
+	return nil
+}
+
 func (rawStringSlice RawStringSlice) JSONSchemaAlias() any {
 	return StringOrSlice("")
 }

From cfe6f5da581942d290a5c9c739c8791564b480d7 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Wed, 24 Apr 2024 16:19:39 +0300
Subject: [PATCH 53/58] fix tests (#5092)

---
 pkg/protocols/dns/request_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/protocols/dns/request_test.go b/pkg/protocols/dns/request_test.go
index 7275097b2c..f32a8c622b 100644
--- a/pkg/protocols/dns/request_test.go
+++ b/pkg/protocols/dns/request_test.go
@@ -37,7 +37,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
 				Name:  "test",
 				Part:  "raw",
 				Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
-				Words: []string{"93.184.216.34"},
+				Words: []string{"93.184.215.14"},
 			}},
 			Extractors: []*extractors.Extractor{{
 				Part:  "raw",
@@ -64,7 +64,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
 	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
 	require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
-	require.Equal(t, "93.184.216.34", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+	require.Equal(t, "93.184.215.14", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
 	finalEvent = nil
 
 	t.Run("url-to-domain", func(t *testing.T) {
@@ -79,6 +79,6 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
 	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
 	require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
-	require.Equal(t, "93.184.216.34", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+	require.Equal(t, "93.184.215.14", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
 	finalEvent = nil
 }

From 3dfcec0a36dbf5ebc529ce0478076279cb975b71 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Wed, 24 Apr 2024 19:34:13 +0530
Subject: [PATCH 54/58] missing mhe check in http payloads (#5099)

* go mod tidy

* fix spm missing hosterrorcheck + improvements

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
---
 go.sum                                        |   6 +-
 .../common/hosterrorscache/hosterrorscache.go |   2 +-
 pkg/protocols/http/httputils/spm.go           |  50 +++++++-
 pkg/protocols/http/request.go                 | 118 +++++++++++++++---
 4 files changed, 145 insertions(+), 31 deletions(-)

diff --git a/go.sum b/go.sum
index aed5e024e4..ce3f2b214d 100644
--- a/go.sum
+++ b/go.sum
@@ -341,6 +341,8 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
 github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwSmwnCg4bxg=
+github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
 github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
 github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
 github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
@@ -835,8 +837,6 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
 github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
 github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
-github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E=
-github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84=
 github.com/projectdiscovery/fastdialer v0.0.68 h1:JuIrr8aVGdGWkEwL4axsJWAWDY2uviSqBB0TCekeCOo=
 github.com/projectdiscovery/fastdialer v0.0.68/go.mod h1:asHSBFJgmwrXpiegcrcAgOyd/QewCVgeI4idH55+v7M=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
@@ -889,8 +889,6 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
 github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0=
 github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M=
-github.com/projectdiscovery/utils v0.0.89 h1:ruH2bSkpX/rB7EPp2EV/rWyAubQVxCVU38nRcLp4L1w=
-github.com/projectdiscovery/utils v0.0.89/go.mod h1:Dwh5cxn7y97jvyYG3GmBvj0negfH9IjH15qXnzFNtOI=
 github.com/projectdiscovery/utils v0.0.91 h1:aHAAnC0qX9pJZrWq4Qpl2PSTYLrSCL1dm1QWLjprE2w=
 github.com/projectdiscovery/utils v0.0.91/go.mod h1:O/6U3ZoU+tNw4lKurdjyVMZPVXL5IYq0YeaDc15PRls=
 github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
index 2eb12cfcb4..3ada7718a8 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -124,7 +124,7 @@ func (c *Cache) MarkFailed(value string, err error) {
 	_ = c.failedTargets.Set(finalValue, existingCacheItemValue)
 }
 
-var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer)`)
+var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer|i/o timeout|could not connect to any address found for host)`)
 
 // checkError checks if an error represents a type that should be
 // added to the host skipping table.
diff --git a/pkg/protocols/http/httputils/spm.go b/pkg/protocols/http/httputils/spm.go
index 52d13f06ff..5e20fea58d 100644
--- a/pkg/protocols/http/httputils/spm.go
+++ b/pkg/protocols/http/httputils/spm.go
@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	syncutil "github.com/projectdiscovery/utils/sync"
+	"golang.org/x/exp/maps"
 )
 
 // WorkPoolType is the type of work pool to use
@@ -19,7 +20,7 @@ const (
 
 // StopAtFirstMatchHandler is a handler that executes
 // request and stops on first match
-type StopAtFirstMatchHandler[T any] struct {
+type StopAtFirstMatchHandler[T comparable] struct {
 	once sync.Once
 	// Result Channel
 	ResultChan chan T
@@ -33,12 +34,14 @@ type StopAtFirstMatchHandler[T any] struct {
 	ctx         context.Context
 	cancel      context.CancelFunc
 	internalWg  *sync.WaitGroup
-	results     []T
+	results     map[T]struct{}
+	onResult    func(T)
 	stopEnabled bool
+	maxResults  int
 }
 
 // NewBlockingSPMHandler creates a new stop at first match handler
-func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *StopAtFirstMatchHandler[T] {
+func NewBlockingSPMHandler[T comparable](ctx context.Context, size int, maxResults int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
 
 	awg, _ := syncutil.New(syncutil.WithSize(size))
@@ -51,6 +54,8 @@ func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *Stop
 		ctx:         ctx1,
 		cancel:      cancel,
 		stopEnabled: spm,
+		results:     make(map[T]struct{}),
+		maxResults:  maxResults,
 	}
 	s.internalWg.Add(1)
 	go s.run(ctx)
@@ -58,7 +63,7 @@ func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *Stop
 }
 
 // NewNonBlockingSPMHandler creates a new stop at first match handler
-func NewNonBlockingSPMHandler[T any](ctx context.Context, spm bool) *StopAtFirstMatchHandler[T] {
+func NewNonBlockingSPMHandler[T comparable](ctx context.Context, maxResults int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
 	s := &StopAtFirstMatchHandler[T]{
 		ResultChan:  make(chan T, 1),
@@ -68,6 +73,8 @@ func NewNonBlockingSPMHandler[T any](ctx context.Context, spm bool) *StopAtFirst
 		ctx:         ctx1,
 		cancel:      cancel,
 		stopEnabled: spm,
+		results:     make(map[T]struct{}),
+		maxResults:  maxResults,
 	}
 	s.internalWg.Add(1)
 	go s.run(ctx)
@@ -82,6 +89,25 @@ func (h *StopAtFirstMatchHandler[T]) Trigger() {
 	}
 }
 
+// Cancel cancels spm context
+func (h *StopAtFirstMatchHandler[T]) Cancel() {
+	h.cancel()
+}
+
+// SetOnResult callback
+// this is not thread safe
+func (h *StopAtFirstMatchHandler[T]) SetOnResultCallback(fn func(T)) {
+	if h.onResult != nil {
+		tmp := h.onResult
+		h.onResult = func(t T) {
+			tmp(t)
+			fn(t)
+		}
+	} else {
+		h.onResult = fn
+	}
+}
+
 // MatchCallback is called when a match is found
 // input fn should be the callback that is intended to be called
 // if stop at first is enabled and other conditions are met
@@ -104,7 +130,14 @@ func (h *StopAtFirstMatchHandler[T]) run(ctx context.Context) {
 			if !ok {
 				return
 			}
-			h.results = append(h.results, val)
+			if h.onResult != nil {
+				h.onResult(val)
+			}
+			if len(h.results) >= h.maxResults {
+				// skip or do not store the result
+				continue
+			}
+			h.results[val] = struct{}{}
 		}
 	}
 }
@@ -114,6 +147,11 @@ func (h *StopAtFirstMatchHandler[T]) Done() <-chan struct{} {
 	return h.ctx.Done()
 }
 
+// Cancelled returns true if the context is cancelled
+func (h *StopAtFirstMatchHandler[T]) Cancelled() bool {
+	return h.ctx.Err() != nil
+}
+
 // FoundFirstMatch returns true if first match was found
 // in stop at first match mode
 func (h *StopAtFirstMatchHandler[T]) FoundFirstMatch() bool {
@@ -168,5 +206,5 @@ func (h *StopAtFirstMatchHandler[T]) Wait() {
 
 // CombinedResults returns the combined results
 func (h *StopAtFirstMatchHandler[T]) CombinedResults() []T {
-	return h.results
+	return maps.Keys(h.results)
 }
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 5d64216f6a..d966a4bb78 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -47,6 +47,9 @@ import (
 
 const (
 	defaultMaxWorkers = 150
+	// max unique errors to store & combine
+	// when executing requests in parallel
+	maxErrorsWhenParallel = 3
 )
 
 var (
@@ -111,7 +114,7 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
 	}
 
 	shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
-	spmHandler := httputils.NewNonBlockingSPMHandler[error](ctx, shouldStop)
+	spmHandler := httputils.NewNonBlockingSPMHandler[error](ctx, maxErrorsWhenParallel, shouldStop)
 	gotMatches := &atomic.Bool{}
 	// wrappedCallback is a callback that wraps the original callback
 	// to implement stop at first match logic
@@ -132,12 +135,29 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
 		}
 	}
 
+	// look for unresponsive hosts and cancel inflight requests as well
+	spmHandler.SetOnResultCallback(func(err error) {
+		if err == nil {
+			return
+		}
+		// marks thsi host as unresponsive if applicable
+		request.markUnresponsiveHost(input, err)
+		if request.isUnresponsiveHost(input) {
+			// stop all inflight requests
+			spmHandler.Cancel()
+		}
+	})
+
 	for i := 0; i < request.RaceNumberRequests; i++ {
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+			// stop sending more requests condition is met
+			break
+		}
 		spmHandler.Acquire()
 		// execute http request
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
 				// stop sending more requests condition is met
 				return
 			}
@@ -175,7 +195,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 	// Stop-at-first-match logic while executing requests
 	// parallely using threads
 	shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
-	spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, shouldStop)
+	spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, maxErrorsWhenParallel, shouldStop)
 	// wrappedCallback is a callback that wraps the original callback
 	// to implement stop at first match logic
 	wrappedCallback := func(event *output.InternalWrappedEvent) {
@@ -194,6 +214,19 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		}
 	}
 
+	// look for unresponsive hosts and cancel inflight requests as well
+	spmHandler.SetOnResultCallback(func(err error) {
+		if err == nil {
+			return
+		}
+		// marks thsi host as unresponsive if applicable
+		request.markUnresponsiveHost(input, err)
+		if request.isUnresponsiveHost(input) {
+			// stop all inflight requests
+			spmHandler.Cancel()
+		}
+	})
+
 	// iterate payloads and make requests
 	generator := request.newGenerator(false)
 	for {
@@ -207,6 +240,11 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 			spmHandler.Resize(request.options.Options.PayloadConcurrency)
 		}
 
+		// break if stop at first match is found or host is unresponsive
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+			break
+		}
+
 		ctx := request.newContext(input)
 		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
 		if err != nil {
@@ -222,19 +260,21 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		spmHandler.Acquire()
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
+				return
+			}
+			// putting ratelimiter here prevents any unnecessary waiting if any
+			request.options.RateLimitTake()
+
+			// after ratelimit take, check if we need to stop
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
 				return
 			}
 
 			select {
 			case <-spmHandler.Done():
 				return
-			case spmHandler.ResultChan <- func() error {
-				// putting ratelimiter here prevents any unnecessary waiting if any
-				request.options.RateLimitTake()
-				previous := make(map[string]interface{})
-				return request.executeRequest(input, httpRequest, previous, false, wrappedCallback, 0)
-			}():
+			case spmHandler.ResultChan <- request.executeRequest(input, httpRequest, make(map[string]interface{}), false, wrappedCallback, 0):
 				return
 			}
 		}(generatedHttpRequest)
@@ -276,12 +316,10 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		maxWorkers = pipeOptions.MaxPendingRequests
 	}
 
-	// Stop-at-first-match logic while executing requests
-	// parallely using threads
 	// Stop-at-first-match logic while executing requests
 	// parallely using threads
 	shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
-	spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, shouldStop)
+	spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, maxErrorsWhenParallel, shouldStop)
 	// wrappedCallback is a callback that wraps the original callback
 	// to implement stop at first match logic
 	wrappedCallback := func(event *output.InternalWrappedEvent) {
@@ -300,11 +338,29 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		}
 	}
 
+	// look for unresponsive hosts and cancel inflight requests as well
+	spmHandler.SetOnResultCallback(func(err error) {
+		if err == nil {
+			return
+		}
+		// marks thsi host as unresponsive if applicable
+		request.markUnresponsiveHost(input, err)
+		if request.isUnresponsiveHost(input) {
+			// stop all inflight requests
+			spmHandler.Cancel()
+		}
+	})
+
 	for {
 		inputData, payloads, ok := generator.nextValue()
 		if !ok {
 			break
 		}
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
+			// skip if first match is found
+			break
+		}
+
 		ctx := request.newContext(input)
 		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
 		if err != nil {
@@ -318,11 +374,10 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		spmHandler.Acquire()
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
 				// skip if first match is found
 				return
 			}
-
 			select {
 			case <-spmHandler.Done():
 				return
@@ -398,7 +453,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				input.MetaInput.Input = generatedHttpRequest.URL()
 			}
 			// Check if hosts keep erroring
-			if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.ID()) {
+			if request.isUnresponsiveHost(input) {
 				return true, nil
 			}
 			var gotMatches bool
@@ -437,12 +492,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				return true, nil
 			}
 			if execReqErr != nil {
-				if request.options.HostErrorsCache != nil {
-					request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
-				}
+				// if applicable mark the host as unresponsive
+				request.markUnresponsiveHost(input, execReqErr)
 				requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL())
+				request.options.Progress.IncrementFailedRequestsBy(1)
+			} else {
+				request.options.Progress.IncrementRequests()
 			}
-			request.options.Progress.IncrementRequests()
 
 			// If this was a match, and we want to stop at first match, skip all further requests.
 			shouldStopAtFirstMatch := generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch
@@ -482,6 +538,10 @@ const drainReqSize = int64(8 * 1024)
 
 // executeRequest executes the actual generated request and returns error if occurred
 func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) {
+	// Check if hosts keep erroring
+	if request.isUnresponsiveHost(input) {
+		return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input)
+	}
 
 	// wrap one more callback for validation and fixing event
 	callback := func(event *output.InternalWrappedEvent) {
@@ -985,3 +1045,21 @@ func (request *Request) newContext(input *contextargs.Context) context.Context {
 	}
 	return context.Background()
 }
+
+// markUnresponsiveHost checks if the error is a unreponsive host error and marks it
+func (request *Request) markUnresponsiveHost(input *contextargs.Context, err error) {
+	if err == nil {
+		return
+	}
+	if request.options.HostErrorsCache != nil {
+		request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
+	}
+}
+
+// isUnresponsiveHost checks if the error is a unreponsive based on its execution history
+func (request *Request) isUnresponsiveHost(input *contextargs.Context) bool {
+	if request.options.HostErrorsCache != nil {
+		return request.options.HostErrorsCache.Check(input.MetaInput.ID())
+	}
+	return false
+}

From 0b82e8b7aa84ec637f0e2cb33da0757622c956ef Mon Sep 17 00:00:00 2001
From: Ice3man <nizamulrana@gmail.com>
Date: Thu, 25 Apr 2024 15:37:56 +0530
Subject: [PATCH 55/58] feat: added support for context cancellation to engine
 (#5096)

* feat: added support for context cancellation to engine

* misc

* feat: added contexts everywhere

* misc

* misc

* use granular http timeouts and increase http timeout to 30s using multiplier

* track response header timeout in mhe

* update responseHeaderTimeout to 5sec

* skip failing windows test

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
---
 cmd/integration-test/library.go               |  2 +-
 internal/runner/lazy.go                       |  4 +-
 internal/runner/runner.go                     |  2 +-
 lib/multi.go                                  |  2 +-
 lib/sdk.go                                    |  3 +-
 pkg/core/execute_options.go                   | 37 +++++++++++++------
 pkg/core/executors.go                         | 35 +++++++++++++-----
 pkg/core/workflow_execute.go                  |  6 +--
 pkg/core/workflow_execute_test.go             | 25 +++++++------
 pkg/input/provider/list/hmap_test.go          |  4 ++
 pkg/js/compiler/compiler.go                   |  8 ++--
 pkg/protocols/code/code.go                    |  1 +
 pkg/protocols/code/code_test.go               |  3 +-
 .../common/automaticscan/automaticscan.go     |  9 +++--
 .../common/contextargs/contextargs.go         | 16 ++++++--
 .../common/hosterrorscache/hosterrorscache.go |  2 +-
 pkg/protocols/dns/request.go                  |  6 +++
 pkg/protocols/dns/request_test.go             |  5 ++-
 pkg/protocols/file/request_test.go            |  3 +-
 .../headless/engine/page_actions_test.go      |  5 ++-
 pkg/protocols/headless/request.go             |  2 +-
 pkg/protocols/http/build_request_test.go      | 16 ++++----
 .../http/httpclientpool/clientpool.go         | 25 +++++++++----
 pkg/protocols/http/request.go                 | 27 ++++++++++++--
 pkg/protocols/http/request_annotations.go     |  3 +-
 pkg/protocols/http/request_fuzz.go            | 12 ++++++
 pkg/protocols/http/request_test.go            |  9 +++--
 pkg/protocols/javascript/js.go                | 17 ++++++++-
 pkg/protocols/network/request.go              | 12 ++++++
 pkg/protocols/network/request_test.go         |  7 ++--
 pkg/protocols/protocols.go                    |  3 +-
 pkg/protocols/ssl/ssl_test.go                 |  3 +-
 pkg/protocols/utils/variables_test.go         |  3 +-
 pkg/scan/scan_context.go                      | 13 +++++--
 pkg/templates/cluster.go                      |  2 +-
 pkg/tmplexec/flow/flow_executor.go            |  6 +++
 pkg/tmplexec/flow/flow_executor_test.go       | 28 +++++++-------
 pkg/tmplexec/generic/exec.go                  |  6 +++
 pkg/tmplexec/multiproto/multi.go              | 12 ++++++
 pkg/tmplexec/multiproto/multi_test.go         |  8 ++--
 40 files changed, 279 insertions(+), 113 deletions(-)

diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go
index a714b29abb..1324688e05 100644
--- a/cmd/integration-test/library.go
+++ b/cmd/integration-test/library.go
@@ -128,7 +128,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
 	}
 	store.Load()
 
-	_ = engine.Execute(store.Templates(), provider.NewSimpleInputProviderWithUrls(templateURL))
+	_ = engine.Execute(context.Background(), store.Templates(), provider.NewSimpleInputProviderWithUrls(templateURL))
 	engine.WorkPool().Wait() // Wait for the scan to finish
 
 	return results, nil
diff --git a/internal/runner/lazy.go b/internal/runner/lazy.go
index b61dd5515a..eb41374513 100644
--- a/internal/runner/lazy.go
+++ b/internal/runner/lazy.go
@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
@@ -71,7 +72,8 @@ func GetLazyAuthFetchCallback(opts *AuthLazyFetchOptions) authx.LazyFetchSecret
 		tmpl := tmpls[0]
 		// add args to tmpl here
 		vars := map[string]interface{}{}
-		ctx := scan.NewScanContext(contextargs.NewWithInput(d.Input))
+		mainCtx := context.Background()
+		ctx := scan.NewScanContext(mainCtx, contextargs.NewWithInput(mainCtx, d.Input))
 		for _, v := range d.Variables {
 			vars[v.Key] = v.Value
 			ctx.Input.Add(v.Key, v.Value)
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index ab2fe5de93..43f16ff10b 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -669,7 +669,7 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine)
 	if r.inputProvider == nil {
 		return nil, errors.New("no input provider found")
 	}
-	results := engine.ExecuteScanWithOpts(finalTemplates, r.inputProvider, r.options.DisableClustering)
+	results := engine.ExecuteScanWithOpts(context.Background(), finalTemplates, r.inputProvider, r.options.DisableClustering)
 	return results, nil
 }
 
diff --git a/lib/multi.go b/lib/multi.go
index 6fa791a2f0..bdcee79677 100644
--- a/lib/multi.go
+++ b/lib/multi.go
@@ -138,7 +138,7 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ..
 	engine := core.New(tmpEngine.opts)
 	engine.SetExecuterOptions(unsafeOpts.executerOpts)
 
-	_ = engine.ExecuteScanWithOpts(store.Templates(), inputProvider, false)
+	_ = engine.ExecuteScanWithOpts(context.Background(), store.Templates(), inputProvider, false)
 
 	engine.WorkPool().Wait()
 	return nil
diff --git a/lib/sdk.go b/lib/sdk.go
index c80abb0615..f6b5ea7c43 100644
--- a/lib/sdk.go
+++ b/lib/sdk.go
@@ -3,6 +3,7 @@ package nuclei
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"io"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
@@ -210,7 +211,7 @@ func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.Result
 	}
 	e.resultCallbacks = append(e.resultCallbacks, filtered...)
 
-	_ = e.engine.ExecuteScanWithOpts(e.store.Templates(), e.inputProvider, false)
+	_ = e.engine.ExecuteScanWithOpts(context.Background(), e.store.Templates(), e.inputProvider, false)
 	defer e.engine.WorkPool().Wait()
 	return nil
 }
diff --git a/pkg/core/execute_options.go b/pkg/core/execute_options.go
index 93f197fc2b..4d27b5f660 100644
--- a/pkg/core/execute_options.go
+++ b/pkg/core/execute_options.go
@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"sync"
 	"sync/atomic"
 
@@ -20,18 +21,18 @@ import (
 //
 // All the execution logic for the templates/workflows happens in this part
 // of the engine.
-func (e *Engine) Execute(templates []*templates.Template, target provider.InputProvider) *atomic.Bool {
-	return e.ExecuteScanWithOpts(templates, target, false)
+func (e *Engine) Execute(ctx context.Context, templates []*templates.Template, target provider.InputProvider) *atomic.Bool {
+	return e.ExecuteScanWithOpts(ctx, templates, target, false)
 }
 
 // ExecuteWithResults a list of templates with results
-func (e *Engine) ExecuteWithResults(templatesList []*templates.Template, target provider.InputProvider, callback func(*output.ResultEvent)) *atomic.Bool {
+func (e *Engine) ExecuteWithResults(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, callback func(*output.ResultEvent)) *atomic.Bool {
 	e.Callback = callback
-	return e.ExecuteScanWithOpts(templatesList, target, false)
+	return e.ExecuteScanWithOpts(ctx, templatesList, target, false)
 }
 
 // ExecuteScanWithOpts executes scan with given scanStrategy
-func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
+func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
 	results := &atomic.Bool{}
 	selfcontainedWg := &sync.WaitGroup{}
 
@@ -83,14 +84,14 @@ func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target
 	}
 
 	// Execute All SelfContained in parallel
-	e.executeAllSelfContained(selfContained, results, selfcontainedWg)
+	e.executeAllSelfContained(ctx, selfContained, results, selfcontainedWg)
 
 	strategyResult := &atomic.Bool{}
 	switch e.options.ScanStrategy {
 	case scanstrategy.TemplateSpray.String():
-		strategyResult = e.executeTemplateSpray(filtered, target)
+		strategyResult = e.executeTemplateSpray(ctx, filtered, target)
 	case scanstrategy.HostSpray.String():
-		strategyResult = e.executeHostSpray(filtered, target)
+		strategyResult = e.executeHostSpray(ctx, filtered, target)
 	}
 
 	results.CompareAndSwap(false, strategyResult.Load())
@@ -100,7 +101,7 @@ func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target
 }
 
 // executeTemplateSpray executes scan using template spray strategy where targets are iterated over each template
-func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
+func (e *Engine) executeTemplateSpray(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
 	results := &atomic.Bool{}
 
 	// wp is workpool that contains different waitgroups for
@@ -108,6 +109,12 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 	wp := e.GetWorkPool()
 
 	for _, template := range templatesList {
+		select {
+		case <-ctx.Done():
+			return results
+		default:
+		}
+
 		// resize check point - nop if there are no changes
 		wp.RefreshWithConfig(e.GetWorkPoolConfig())
 
@@ -125,7 +132,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 			// All other request types are executed here
 			// Note: executeTemplateWithTargets creates goroutines and blocks
 			// given template is executed on all targets
-			e.executeTemplateWithTargets(tpl, target, results)
+			e.executeTemplateWithTargets(ctx, tpl, target, results)
 		}(template)
 	}
 	wp.Wait()
@@ -133,15 +140,21 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 }
 
 // executeHostSpray executes scan using host spray strategy where templates are iterated over each target
-func (e *Engine) executeHostSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
+func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
 	results := &atomic.Bool{}
 	wp, _ := syncutil.New(syncutil.WithSize(e.options.BulkSize + e.options.HeadlessBulkSize))
 
 	target.Iterate(func(value *contextargs.MetaInput) bool {
+		select {
+		case <-ctx.Done():
+			return false
+		default:
+		}
+
 		wp.Add()
 		go func(targetval *contextargs.MetaInput) {
 			defer wp.Done()
-			e.executeTemplatesOnTarget(templatesList, targetval, results)
+			e.executeTemplatesOnTarget(ctx, templatesList, targetval, results)
 		}(value)
 		return true
 	})
diff --git a/pkg/core/executors.go b/pkg/core/executors.go
index 14fb75c631..89c85b2ad7 100644
--- a/pkg/core/executors.go
+++ b/pkg/core/executors.go
@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"sync"
 	"sync/atomic"
 
@@ -17,14 +18,14 @@ import (
 // Executors are low level executors that deals with template execution on a target
 
 // executeAllSelfContained executes all self contained templates that do not use `target`
-func (e *Engine) executeAllSelfContained(alltemplates []*templates.Template, results *atomic.Bool, sg *sync.WaitGroup) {
+func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*templates.Template, results *atomic.Bool, sg *sync.WaitGroup) {
 	for _, v := range alltemplates {
 		sg.Add(1)
 		go func(template *templates.Template) {
 			defer sg.Done()
 			var err error
 			var match bool
-			ctx := scan.NewScanContext(contextargs.New())
+			ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
 			if e.Callback != nil {
 				if results, err := template.Executer.ExecuteWithResults(ctx); err != nil {
 					for _, result := range results {
@@ -45,7 +46,7 @@ func (e *Engine) executeAllSelfContained(alltemplates []*templates.Template, res
 }
 
 // executeTemplateWithTarget executes a given template on x targets (with a internal targetpool(i.e concurrency))
-func (e *Engine) executeTemplateWithTargets(template *templates.Template, target provider.InputProvider, results *atomic.Bool) {
+func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templates.Template, target provider.InputProvider, results *atomic.Bool) {
 	// this is target pool i.e max target to execute
 	wg := e.workPool.InputPool(template.Type())
 
@@ -77,6 +78,12 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 	}
 
 	target.Iterate(func(scannedValue *contextargs.MetaInput) bool {
+		select {
+		case <-ctx.Done():
+			return false // exit
+		default:
+		}
+
 		// Best effort to track the host progression
 		// skips indexes lower than the minimum in-flight at interruption time
 		var skip bool
@@ -114,9 +121,9 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 
 			var match bool
 			var err error
-			ctxArgs := contextargs.New()
+			ctxArgs := contextargs.New(ctx)
 			ctxArgs.MetaInput = value
-			ctx := scan.NewScanContext(ctxArgs)
+			ctx := scan.NewScanContext(ctx, ctxArgs)
 			switch template.Type() {
 			case types.WorkflowProtocol:
 				match = e.executeWorkflow(ctx, template.CompiledWorkflow)
@@ -149,7 +156,7 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 }
 
 // executeTemplatesOnTarget execute given templates on given single target
-func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, target *contextargs.MetaInput, results *atomic.Bool) {
+func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*templates.Template, target *contextargs.MetaInput, results *atomic.Bool) {
 	// all templates are executed on single target
 
 	// wp is workpool that contains different waitgroups for
@@ -158,6 +165,12 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	wp := e.GetWorkPool()
 
 	for _, tpl := range alltemplates {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
 		// resize check point - nop if there are no changes
 		wp.RefreshWithConfig(e.GetWorkPoolConfig())
 
@@ -173,9 +186,9 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 
 			var match bool
 			var err error
-			ctxArgs := contextargs.New()
+			ctxArgs := contextargs.New(ctx)
 			ctxArgs.MetaInput = value
-			ctx := scan.NewScanContext(ctxArgs)
+			ctx := scan.NewScanContext(ctx, ctxArgs)
 			switch template.Type() {
 			case types.WorkflowProtocol:
 				match = e.executeWorkflow(ctx, template.CompiledWorkflow)
@@ -230,9 +243,11 @@ func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs
 	go func(tpl *templates.Template) {
 		defer wg.Done()
 
-		ctxArgs := contextargs.New()
+		// TODO: Workflows are a no-op for now. We need to
+		// implement them in the future with context cancellation
+		ctxArgs := contextargs.New(context.Background())
 		ctxArgs.MetaInput = value
-		ctx := scan.NewScanContext(ctxArgs)
+		ctx := scan.NewScanContext(context.Background(), ctxArgs)
 		match, err := template.Executer.Execute(ctx)
 		if err != nil {
 			gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err)
diff --git a/pkg/core/workflow_execute.go b/pkg/core/workflow_execute.go
index c17a1af8d5..d0d3ede004 100644
--- a/pkg/core/workflow_execute.go
+++ b/pkg/core/workflow_execute.go
@@ -21,7 +21,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 
 	// at this point we should be at the start root execution of a workflow tree, hence we create global shared instances
 	workflowCookieJar, _ := cookiejar.New(nil)
-	ctxArgs := contextargs.New()
+	ctxArgs := contextargs.New(ctx.Context())
 	ctxArgs.MetaInput = ctx.Input.MetaInput
 	ctxArgs.CookieJar = workflowCookieJar
 
@@ -139,7 +139,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 							defer swg.Done()
 
 							// create a new context with the same input but with unset callbacks
-							subCtx := scan.NewScanContext(ctx.Input)
+							subCtx := scan.NewScanContext(ctx.Context(), ctx.Input)
 							if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, w); err != nil {
 								gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err)
 							}
@@ -165,7 +165,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 
 			go func(template *workflows.WorkflowTemplate) {
 				// create a new context with the same input but with unset callbacks
-				subCtx := scan.NewScanContext(ctx.Input)
+				subCtx := scan.NewScanContext(ctx.Context(), ctx.Input)
 				if err := e.runWorkflowStep(template, subCtx, results, swg, w); err != nil {
 					gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
 				}
diff --git a/pkg/core/workflow_execute_test.go b/pkg/core/workflow_execute_test.go
index f3d6a7f231..0c478a5e13 100644
--- a/pkg/core/workflow_execute_test.go
+++ b/pkg/core/workflow_execute_test.go
@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"testing"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
@@ -25,8 +26,8 @@ func TestWorkflowsSimple(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 }
@@ -49,8 +50,8 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 
@@ -77,8 +78,8 @@ func TestWorkflowsSubtemplates(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 
@@ -103,8 +104,8 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.False(t, matched, "could not get correct match value")
 
@@ -134,8 +135,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 
@@ -165,8 +166,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.False(t, matched, "could not get correct match value")
 
diff --git a/pkg/input/provider/list/hmap_test.go b/pkg/input/provider/list/hmap_test.go
index 95fc57f2d9..b08782e124 100644
--- a/pkg/input/provider/list/hmap_test.go
+++ b/pkg/input/provider/list/hmap_test.go
@@ -3,6 +3,7 @@ package list
 import (
 	"net"
 	"os"
+	"runtime"
 	"strconv"
 	"strings"
 	"testing"
@@ -77,6 +78,9 @@ func (m *mockDnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 }
 
 func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping test see: https://github.com/projectdiscovery/nuclei/issues/5097")
+	}
 	srv := &dns.Server{Addr: ":" + strconv.Itoa(61234), Net: "udp"}
 	srv.Handler = &mockDnsHandler{}
 
diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go
index 265bbaf04b..05eacda6c6 100644
--- a/pkg/js/compiler/compiler.go
+++ b/pkg/js/compiler/compiler.go
@@ -37,6 +37,8 @@ type ExecuteOptions struct {
 	// Source is original source of the script
 	Source *string
 
+	Context context.Context
+
 	// Manually exported objects
 	exports map[string]interface{}
 }
@@ -77,13 +79,13 @@ func (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error
 	if err != nil {
 		return nil, err
 	}
-	return c.ExecuteWithOptions(p, args, &ExecuteOptions{})
+	return c.ExecuteWithOptions(p, args, &ExecuteOptions{Context: context.Background()})
 }
 
 // ExecuteWithOptions executes a script with the provided options.
 func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) {
 	if opts == nil {
-		opts = &ExecuteOptions{}
+		opts = &ExecuteOptions{Context: context.Background()}
 	}
 	if args == nil {
 		args = NewExecuteArgs()
@@ -105,7 +107,7 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
 	}
 
 	// execute with context and timeout
-	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(opts.Timeout)*time.Second)
+	ctx, cancel := context.WithTimeout(opts.Context, time.Duration(opts.Timeout)*time.Second)
 	defer cancel()
 	// execute the script
 	results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {
diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go
index 1b59de3efd..70dda72548 100644
--- a/pkg/protocols/code/code.go
+++ b/pkg/protocols/code/code.go
@@ -199,6 +199,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				Source:   &request.PreCondition,
 				Callback: registerPreConditionFunctions,
 				Cleanup:  cleanUpPreConditionFunctions,
+				Context:  input.Context(),
 			})
 		if err != nil {
 			return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
diff --git a/pkg/protocols/code/code_test.go b/pkg/protocols/code/code_test.go
index 1ace1388f3..320f7c5481 100644
--- a/pkg/protocols/code/code_test.go
+++ b/pkg/protocols/code/code_test.go
@@ -3,6 +3,7 @@
 package code
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -31,7 +32,7 @@ func TestCodeProtocol(t *testing.T) {
 	require.Nil(t, err, "could not compile code request")
 
 	var gotEvent output.InternalEvent
-	ctxArgs := contextargs.NewWithInput("")
+	ctxArgs := contextargs.NewWithInput(context.Background(), "")
 	err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {
 		gotEvent = event.InternalEvent
 	})
diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go
index 7119377b01..4567f498d9 100644
--- a/pkg/protocols/common/automaticscan/automaticscan.go
+++ b/pkg/protocols/common/automaticscan/automaticscan.go
@@ -1,6 +1,7 @@
 package automaticscan
 
 import (
+	"context"
 	"io"
 	"net/http"
 	"os"
@@ -189,7 +190,7 @@ func (s *Service) executeAutomaticScanOnTarget(input *contextargs.MetaInput) {
 	execOptions.Progress = &testutils.MockProgressClient{} // stats are not supported yet due to centralized logic and cannot be reinitialized
 	eng.SetExecuterOptions(execOptions)
 
-	tmp := eng.ExecuteScanWithOpts(finalTemplates, provider.NewSimpleInputProviderWithUrls(input.Input), true)
+	tmp := eng.ExecuteScanWithOpts(context.Background(), finalTemplates, provider.NewSimpleInputProviderWithUrls(input.Input), true)
 	s.hasResults.Store(tmp.Load())
 }
 
@@ -244,7 +245,9 @@ func (s *Service) getTagsUsingWappalyzer(input *contextargs.MetaInput) []string
 
 // getTagsUsingDetectionTemplates returns tags using detection templates
 func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) ([]string, int) {
-	ctxArgs := contextargs.NewWithInput(input.Input)
+	ctx := context.Background()
+
+	ctxArgs := contextargs.NewWithInput(ctx, input.Input)
 
 	// execute tech detection templates on target
 	tags := map[string]struct{}{}
@@ -256,7 +259,7 @@ func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) (
 		sg.Add()
 		go func(template *templates.Template) {
 			defer sg.Done()
-			ctx := scan.NewScanContext(ctxArgs)
+			ctx := scan.NewScanContext(ctx, ctxArgs)
 			ctx.OnResult = func(event *output.InternalWrappedEvent) {
 				if event == nil {
 					return
diff --git a/pkg/protocols/common/contextargs/contextargs.go b/pkg/protocols/common/contextargs/contextargs.go
index 4ebaa15615..8e8f0361c4 100644
--- a/pkg/protocols/common/contextargs/contextargs.go
+++ b/pkg/protocols/common/contextargs/contextargs.go
@@ -1,6 +1,7 @@
 package contextargs
 
 import (
+	"context"
 	"net/http/cookiejar"
 	"strings"
 	"sync/atomic"
@@ -19,6 +20,8 @@ var (
 
 // Context implements a shared context struct to share information across multiple templates within a workflow
 type Context struct {
+	ctx context.Context
+
 	// Meta is the target for the executor
 	MetaInput *MetaInput
 
@@ -30,17 +33,18 @@ type Context struct {
 }
 
 // Create a new contextargs instance
-func New() *Context {
-	return NewWithInput("")
+func New(ctx context.Context) *Context {
+	return NewWithInput(ctx, "")
 }
 
 // Create a new contextargs instance with input string
-func NewWithInput(input string) *Context {
+func NewWithInput(ctx context.Context, input string) *Context {
 	jar, err := cookiejar.New(nil)
 	if err != nil {
 		gologger.Error().Msgf("contextargs: could not create cookie jar: %s\n", err)
 	}
 	return &Context{
+		ctx:       ctx,
 		MetaInput: &MetaInput{Input: input},
 		CookieJar: jar,
 		args: &mapsutil.SyncLockMap[string, interface{}]{
@@ -50,6 +54,11 @@ func NewWithInput(input string) *Context {
 	}
 }
 
+// Context returns the context of the current contextargs
+func (ctx *Context) Context() context.Context {
+	return ctx.ctx
+}
+
 // Set the specific key-value pair
 func (ctx *Context) Set(key string, value interface{}) {
 	_ = ctx.args.Set(key, value)
@@ -158,6 +167,7 @@ func (ctx *Context) HasArgs() bool {
 
 func (ctx *Context) Clone() *Context {
 	newCtx := &Context{
+		ctx:       ctx.ctx,
 		MetaInput: ctx.MetaInput.Clone(),
 		args:      ctx.args.Clone(),
 		CookieJar: ctx.CookieJar,
diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
index 3ada7718a8..3abd1ec55f 100644
--- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go
+++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go
@@ -124,7 +124,7 @@ func (c *Cache) MarkFailed(value string, err error) {
 	_ = c.failedTargets.Set(finalValue, existingCacheItemValue)
 }
 
-var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer|i/o timeout|could not connect to any address found for host)`)
+var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer|i/o timeout|could not connect to any address found for host|timeout awaiting response headers)`)
 
 // checkError checks if an error represents a type that should be
 // added to the host skipping table.
diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go
index d4e70e13e9..608e4730ac 100644
--- a/pkg/protocols/dns/request.go
+++ b/pkg/protocols/dns/request.go
@@ -80,6 +80,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 				break
 			}
 
+			select {
+			case <-input.Context().Done():
+				return input.Context().Err()
+			default:
+			}
+
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
 				swg.Resize(request.options.Options.PayloadConcurrency)
diff --git a/pkg/protocols/dns/request_test.go b/pkg/protocols/dns/request_test.go
index f32a8c622b..81f0b98ab2 100644
--- a/pkg/protocols/dns/request_test.go
+++ b/pkg/protocols/dns/request_test.go
@@ -1,6 +1,7 @@
 package dns
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -54,7 +55,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	t.Run("domain-valid", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput("example.com")
+		ctxArgs := contextargs.NewWithInput(context.Background(), "example.com")
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
@@ -70,7 +71,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	t.Run("url-to-domain", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		err := request.ExecuteWithResults(contextargs.NewWithInput("https://example.com"), metadata, previous, func(event *output.InternalWrappedEvent) {
+		err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), "https://example.com"), metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
 		require.Nil(t, err, "could not execute dns request")
diff --git a/pkg/protocols/file/request_test.go b/pkg/protocols/file/request_test.go
index ff41e3e8be..7c69cf5bce 100644
--- a/pkg/protocols/file/request_test.go
+++ b/pkg/protocols/file/request_test.go
@@ -1,6 +1,7 @@
 package file
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"testing"
@@ -67,7 +68,7 @@ func TestFileExecuteWithResults(t *testing.T) {
 	t.Run("valid", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(tempDir)
+		ctxArgs := contextargs.NewWithInput(context.Background(), tempDir)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
diff --git a/pkg/protocols/headless/engine/page_actions_test.go b/pkg/protocols/headless/engine/page_actions_test.go
index a4b69eeff0..e6699d6408 100644
--- a/pkg/protocols/headless/engine/page_actions_test.go
+++ b/pkg/protocols/headless/engine/page_actions_test.go
@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"math/rand"
@@ -595,7 +596,7 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle
 	ts := httptest.NewServer(http.HandlerFunc(handler))
 	defer ts.Close()
 
-	input := contextargs.NewWithInput(ts.URL)
+	input := contextargs.NewWithInput(context.Background(), ts.URL)
 	input.CookieJar, err = cookiejar.New(nil)
 	require.Nil(t, err)
 
@@ -674,7 +675,7 @@ func TestBlockedHeadlessURLS(t *testing.T) {
 			{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
 		}
 
-		data, page, err := instance.Run(contextargs.NewWithInput(ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test
+		data, page, err := instance.Run(contextargs.NewWithInput(context.Background(), ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test
 		require.Error(t, err, "expected error for url %s got %v", testcase, data)
 		require.True(t, stringsutil.ContainsAny(err.Error(), "net::ERR_ACCESS_DENIED", "failed to parse url", "Cannot navigate to invalid URL", "net::ERR_ABORTED", "net::ERR_INVALID_URL"), "found different error %v for testcases %v", err, testcase)
 		require.Len(t, data, 0, "expected no data for url %s got %v", testcase, data)
diff --git a/pkg/protocols/headless/request.go b/pkg/protocols/headless/request.go
index f45afdd35e..05bf5d3548 100644
--- a/pkg/protocols/headless/request.go
+++ b/pkg/protocols/headless/request.go
@@ -44,7 +44,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 		if err != nil {
 			return err
 		}
-		input = contextargs.NewWithInput(url)
+		input = contextargs.NewWithInput(input.Context(), url)
 	}
 
 	if request.options.Browser.UserAgent() == "" {
diff --git a/pkg/protocols/http/build_request_test.go b/pkg/protocols/http/build_request_test.go
index bc87549f03..4405bd10b7 100644
--- a/pkg/protocols/http/build_request_test.go
+++ b/pkg/protocols/http/build_request_test.go
@@ -40,7 +40,7 @@ func TestMakeRequestFromModal(t *testing.T) {
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	if req.request.URL == nil {
 		t.Fatalf("url is nil in generator make")
@@ -70,13 +70,13 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com/test.php"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com/test.php"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path")
 
 	generator = request.newGenerator(false)
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com/test/"), inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com/test/"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path")
 }
@@ -110,13 +110,13 @@ Accept-Encoding: gzip`},
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw")
 
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw")
@@ -151,13 +151,13 @@ Accept-Encoding: gzip`},
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw")
 
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw")
@@ -195,7 +195,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
 	require.Nil(t, err, "could not create interactsh client")
 
 	inputData, payloads, _ := generator.nextValue()
-	got, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	got, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 
 	// check if all the interactsh markers are replaced with unique urls
diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go
index 1560e787a6..3e2baf55a5 100644
--- a/pkg/protocols/http/httpclientpool/clientpool.go
+++ b/pkg/protocols/http/httpclientpool/clientpool.go
@@ -35,8 +35,18 @@ var (
 	forceMaxRedirects int
 	normalClient      *retryablehttp.Client
 	clientPool        *mapsutil.SyncLockMap[string, *retryablehttp.Client]
+	// ResponseHeaderTimeout is the timeout for response headers
+	// to be read from the server (this prevents infinite hang started by server if any)
+	ResponseHeaderTimeout = time.Duration(5) * time.Second
+	// HttpTimeoutMultiplier is the multiplier for the http timeout
+	HttpTimeoutMultiplier = 3
 )
 
+// GetHttpTimeout returns the http timeout for the client
+func GetHttpTimeout(opts *types.Options) time.Duration {
+	return time.Duration(opts.Timeout*HttpTimeoutMultiplier) * time.Second
+}
+
 // Init initializes the clientpool implementation
 func Init(options *types.Options) error {
 	// Don't create clients if already created in the past.
@@ -139,7 +149,7 @@ func GetRawHTTP(options *types.Options) *rawhttp.Client {
 		} else if Dialer != nil {
 			rawHttpOptions.FastDialer = Dialer
 		}
-		rawHttpOptions.Timeout = time.Duration(options.Timeout) * time.Second
+		rawHttpOptions.Timeout = GetHttpTimeout(options)
 		rawHttpClient = rawhttp.NewClient(rawHttpOptions)
 	}
 	return rawHttpClient
@@ -237,11 +247,12 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 			}
 			return Dialer.DialTLS(ctx, network, addr)
 		},
-		MaxIdleConns:        maxIdleConns,
-		MaxIdleConnsPerHost: maxIdleConnsPerHost,
-		MaxConnsPerHost:     maxConnsPerHost,
-		TLSClientConfig:     tlsConfig,
-		DisableKeepAlives:   disableKeepAlives,
+		MaxIdleConns:          maxIdleConns,
+		MaxIdleConnsPerHost:   maxIdleConnsPerHost,
+		MaxConnsPerHost:       maxConnsPerHost,
+		TLSClientConfig:       tlsConfig,
+		DisableKeepAlives:     disableKeepAlives,
+		ResponseHeaderTimeout: ResponseHeaderTimeout,
 	}
 
 	if types.ProxyURL != "" {
@@ -288,7 +299,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 		CheckRedirect: makeCheckRedirectFunc(redirectFlow, maxRedirects),
 	}
 	if !configuration.NoTimeout {
-		httpclient.Timeout = time.Duration(options.Timeout) * time.Second
+		httpclient.Timeout = GetHttpTimeout(options)
 	}
 	client := retryablehttp.NewWithHTTPClient(httpclient, retryableHttpOptions)
 	if jar != nil {
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index d966a4bb78..545247ef56 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -235,6 +235,12 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 			break
 		}
 
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		// resize check point - nop if there are no changes
 		if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
 			spmHandler.Resize(request.options.Options.PayloadConcurrency)
@@ -356,6 +362,13 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		if !ok {
 			break
 		}
+
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
 			// skip if first match is found
 			break
@@ -433,8 +446,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			request.options.RateLimitTake()
 
 			ctx := request.newContext(input)
-			ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second)
+			ctxWithTimeout, cancel := context.WithTimeout(ctx, httpclientpool.GetHttpTimeout(request.options.Options))
 			defer cancel()
+
 			generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
 			if err != nil {
 				if err == types.ErrNoMoreRequests {
@@ -512,6 +526,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 		if !ok {
 			break
 		}
+
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		var gotErr error
 		var skip bool
 		if len(gotDynamicValues) > 0 {
@@ -1041,9 +1062,9 @@ func (request *Request) pruneSignatureInternalValues(maps ...map[string]interfac
 
 func (request *Request) newContext(input *contextargs.Context) context.Context {
 	if input.MetaInput.CustomIP != "" {
-		return context.WithValue(context.Background(), fastdialer.IP, input.MetaInput.CustomIP)
+		return context.WithValue(input.Context(), fastdialer.IP, input.MetaInput.CustomIP)
 	}
-	return context.Background()
+	return input.Context()
 }
 
 // markUnresponsiveHost checks if the error is a unreponsive host error and marks it
diff --git a/pkg/protocols/http/request_annotations.go b/pkg/protocols/http/request_annotations.go
index 6325e4e734..67bf7c1671 100644
--- a/pkg/protocols/http/request_annotations.go
+++ b/pkg/protocols/http/request_annotations.go
@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/retryablehttp-go"
 	iputil "github.com/projectdiscovery/utils/ip"
 	stringsutil "github.com/projectdiscovery/utils/strings"
@@ -124,7 +125,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
 			}
 		} else {
 			//nolint:govet // cancelled automatically by withTimeout
-			ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), time.Duration(r.options.Options.Timeout)*time.Second)
+			ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), httpclientpool.GetHttpTimeout(r.options.Options))
 			request = request.Clone(ctx)
 		}
 	}
diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go
index aed17f54e0..fc2a4e7576 100644
--- a/pkg/protocols/http/request_fuzz.go
+++ b/pkg/protocols/http/request_fuzz.go
@@ -110,9 +110,21 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 func (request *Request) executeAllFuzzingRules(input *contextargs.Context, values map[string]interface{}, baseRequest *retryablehttp.Request, callback protocols.OutputEventCallback) error {
 	applicable := false
 	for _, rule := range request.Fuzzing {
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		err := rule.Execute(&fuzz.ExecuteRuleInput{
 			Input: input,
 			Callback: func(gr fuzz.GeneratedRequest) bool {
+				select {
+				case <-input.Context().Done():
+					return false
+				default:
+				}
+
 				// TODO: replace this after scanContext Refactor
 				return request.executeGeneratedFuzzingRequest(gr, input, callback)
 			},
diff --git a/pkg/protocols/http/request_test.go b/pkg/protocols/http/request_test.go
index cd8e860b07..c0bd2bb346 100644
--- a/pkg/protocols/http/request_test.go
+++ b/pkg/protocols/http/request_test.go
@@ -1,6 +1,7 @@
 package http
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -83,7 +84,7 @@ Disallow: /c`))
 	t.Run("test", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++
@@ -159,7 +160,7 @@ func TestDisableTE(t *testing.T) {
 	t.Run("test", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++
@@ -172,7 +173,7 @@ func TestDisableTE(t *testing.T) {
 	t.Run("test2", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request2.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++
@@ -242,7 +243,7 @@ func TestReqURLPattern(t *testing.T) {
 	t.Run("test", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++
diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go
index 54bb378839..61257a968b 100644
--- a/pkg/protocols/javascript/js.go
+++ b/pkg/protocols/javascript/js.go
@@ -154,6 +154,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 		opts := &compiler.ExecuteOptions{
 			Timeout: request.Timeout,
 			Source:  &request.Init,
+			Context: context.Background(),
 		}
 		// register 'export' function to export variables from init code
 		// these are saved in args and are available in pre-condition and request code
@@ -343,7 +344,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 		argsCopy.TemplateCtx = templateCtx.GetAll()
 
 		result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, argsCopy,
-			&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.PreCondition})
+			&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.PreCondition, Context: target.Context()})
 		if err != nil {
 			return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
 		}
@@ -373,6 +374,12 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 				return nil
 			}
 
+			select {
+			case <-input.Context().Done():
+				return input.Context().Err()
+			default:
+			}
+
 			if err := request.executeRequestWithPayloads(hostPort, input, hostname, value, payloadValues, func(result *output.InternalWrappedEvent) {
 				if result.OperatorsResult != nil && result.OperatorsResult.Matched {
 					gotMatches = true
@@ -419,6 +426,12 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 				break
 			}
 
+			select {
+			case <-input.Context().Done():
+				return
+			default:
+			}
+
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && sg.Size != request.options.Options.PayloadConcurrency {
 				sg.Resize(request.options.Options.PayloadConcurrency)
@@ -486,7 +499,7 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 	}
 
 	results, err := request.options.JsCompiler.ExecuteWithOptions(request.scriptCompiled, argsCopy,
-		&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.Code})
+		&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.Code, Context: input.Context()})
 	if err != nil {
 		// shouldn't fail even if it returned error instead create a failure event
 		results = compiler.ExecuteResult{"success": false, "error": err.Error()}
diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go
index 1f51ebe119..23eed8cb0f 100644
--- a/pkg/protocols/network/request.go
+++ b/pkg/protocols/network/request.go
@@ -139,6 +139,12 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps
 	variables = generators.MergeMaps(variablesMap, variables, request.options.Constants)
 
 	for _, kv := range request.addresses {
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		actualAddress := replacer.Replace(kv.address, variables)
 
 		if visited.Has(actualAddress) && !request.options.Options.DisableClustering {
@@ -186,6 +192,12 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 				break
 			}
 
+			select {
+			case <-input.Context().Done():
+				return input.Context().Err()
+			default:
+			}
+
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
 				swg.Resize(request.options.Options.PayloadConcurrency)
diff --git a/pkg/protocols/network/request_test.go b/pkg/protocols/network/request_test.go
index bf0cd531d4..1945888e9b 100644
--- a/pkg/protocols/network/request_test.go
+++ b/pkg/protocols/network/request_test.go
@@ -1,6 +1,7 @@
 package network
 
 import (
+	"context"
 	"encoding/hex"
 	"fmt"
 	"net/http"
@@ -65,7 +66,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 	t.Run("domain-valid", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(parsed.Host)
+		ctxArgs := contextargs.NewWithInput(context.Background(), parsed.Host)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
@@ -81,7 +82,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 	t.Run("invalid-port-override", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput("127.0.0.1:11211")
+		ctxArgs := contextargs.NewWithInput(context.Background(), "127.0.0.1:11211")
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
@@ -95,7 +96,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 	t.Run("hex-to-string", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(parsed.Host)
+		ctxArgs := contextargs.NewWithInput(context.Background(), parsed.Host)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go
index 6328bc36a3..6b4904c8d8 100644
--- a/pkg/protocols/protocols.go
+++ b/pkg/protocols/protocols.go
@@ -1,6 +1,7 @@
 package protocols
 
 import (
+	"context"
 	"encoding/base64"
 	"sync/atomic"
 
@@ -173,7 +174,7 @@ func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contexta
 	templateCtx, ok := e.templateCtxStore.Get(scanId)
 	if !ok {
 		// if template context does not exist create new and add it to store and return it
-		templateCtx = contextargs.New()
+		templateCtx = contextargs.New(context.Background())
 		templateCtx.MetaInput = input
 		_ = e.templateCtxStore.Set(scanId, templateCtx)
 	}
diff --git a/pkg/protocols/ssl/ssl_test.go b/pkg/protocols/ssl/ssl_test.go
index 009cf98d32..59c9f85f3e 100644
--- a/pkg/protocols/ssl/ssl_test.go
+++ b/pkg/protocols/ssl/ssl_test.go
@@ -1,6 +1,7 @@
 package ssl
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -28,7 +29,7 @@ func TestSSLProtocol(t *testing.T) {
 	require.Nil(t, err, "could not compile ssl request")
 
 	var gotEvent output.InternalEvent
-	ctxArgs := contextargs.NewWithInput("scanme.sh:443")
+	ctxArgs := contextargs.NewWithInput(context.Background(), "scanme.sh:443")
 	err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {
 		gotEvent = event.InternalEvent
 	})
diff --git a/pkg/protocols/utils/variables_test.go b/pkg/protocols/utils/variables_test.go
index c24305017c..b83529499d 100644
--- a/pkg/protocols/utils/variables_test.go
+++ b/pkg/protocols/utils/variables_test.go
@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"context"
 	"reflect"
 	"testing"
 
@@ -49,7 +50,7 @@ func TestHTTPVariables(t *testing.T) {
 	require.Equal(t, values["Hostname"], "foobar.com", "incorrect hostname")
 
 	baseURL = "http://scanme.sh"
-	ctxArgs := contextargs.NewWithInput(baseURL)
+	ctxArgs := contextargs.NewWithInput(context.Background(), baseURL)
 	ctxArgs.MetaInput.CustomIP = "1.2.3.4"
 	values = GenerateVariablesWithContextArgs(ctxArgs, true)
 
diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go
index 8851349019..69fbfde4c0 100644
--- a/pkg/scan/scan_context.go
+++ b/pkg/scan/scan_context.go
@@ -10,7 +10,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 )
 
-
 type ScanContextOption func(*ScanContext)
 
 func WithEvents() ScanContextOption {
@@ -20,7 +19,8 @@ func WithEvents() ScanContextOption {
 }
 
 type ScanContext struct {
-	context.Context
+	ctx context.Context
+
 	// exported / configurable fields
 	Input *contextargs.Context
 
@@ -43,8 +43,13 @@ type ScanContext struct {
 }
 
 // NewScanContext creates a new scan context using input
-func NewScanContext(input *contextargs.Context) *ScanContext {
-	return &ScanContext{Input: input}
+func NewScanContext(ctx context.Context, input *contextargs.Context) *ScanContext {
+	return &ScanContext{ctx: ctx, Input: input}
+}
+
+// Context returns the context of the scan
+func (s *ScanContext) Context() context.Context {
+	return s.ctx
 }
 
 // GenerateResult returns final results slice from all events
diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go
index b1e0b56e2d..97e7b67bd0 100644
--- a/pkg/templates/cluster.go
+++ b/pkg/templates/cluster.go
@@ -303,7 +303,7 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
-	scanCtx := scan.NewScanContext(ctx.Input)
+	scanCtx := scan.NewScanContext(ctx.Context(), ctx.Input)
 	dynamicValues := make(map[string]interface{})
 
 	inputItem := ctx.Input.Clone()
diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go
index 31aa9f0f4d..c3f0014cab 100644
--- a/pkg/tmplexec/flow/flow_executor.go
+++ b/pkg/tmplexec/flow/flow_executor.go
@@ -174,6 +174,12 @@ func (f *FlowExecutor) Compile() error {
 
 // ExecuteWithResults executes the flow and returns results
 func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
+	select {
+	case <-ctx.Context().Done():
+		return ctx.Context().Err()
+	default:
+	}
+
 	f.ctx.Input = ctx.Input
 	// -----Load all types of variables-----
 	// add all input args to template context
diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go
index bd3a526d2f..b47b38a2a0 100644
--- a/pkg/tmplexec/flow/flow_executor_test.go
+++ b/pkg/tmplexec/flow/flow_executor_test.go
@@ -55,8 +55,8 @@ func TestFlowTemplateWithIndex(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("hackerone.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "hackerone.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
 	require.True(t, gotresults)
@@ -74,8 +74,8 @@ func TestFlowTemplateWithID(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	target := contextargs.NewWithInput("hackerone.com")
-	ctx := scan.NewScanContext(target)
+	target := contextargs.NewWithInput(context.Background(), "hackerone.com")
+	ctx := scan.NewScanContext(context.Background(), target)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
 	require.True(t, gotresults)
@@ -96,8 +96,8 @@ func TestFlowWithProtoPrefix(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("hackerone.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "hackerone.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
 	require.True(t, gotresults)
@@ -116,8 +116,8 @@ func TestFlowWithConditionNegative(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("scanme.sh")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "scanme.sh")
+	ctx := scan.NewScanContext(context.Background(), input)
 	// expect no results and verify thant dns request is executed and http is not
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
@@ -137,8 +137,8 @@ func TestFlowWithConditionPositive(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("blog.projectdiscovery.io")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	ctx := scan.NewScanContext(context.Background(), input)
 	// positive match . expect results also verify that both dns() and http() were executed
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
@@ -158,8 +158,8 @@ func TestFlowWithNoMatchers(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("blog.projectdiscovery.io")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	ctx := scan.NewScanContext(context.Background(), input)
 	// positive match . expect results also verify that both dns() and http() were executed
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
@@ -174,8 +174,8 @@ func TestFlowWithNoMatchers(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	anotherInput := contextargs.NewWithInput("blog.projectdiscovery.io")
-	anotherCtx := scan.NewScanContext(anotherInput)
+	anotherInput := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	anotherCtx := scan.NewScanContext(context.Background(), anotherInput)
 	// positive match . expect results also verify that both dns() and http() were executed
 	gotresults, err = Template.Executer.Execute(anotherCtx)
 	require.Nil(t, err, "could not execute template")
diff --git a/pkg/tmplexec/generic/exec.go b/pkg/tmplexec/generic/exec.go
index e47cfd7305..29ace4477d 100644
--- a/pkg/tmplexec/generic/exec.go
+++ b/pkg/tmplexec/generic/exec.go
@@ -45,6 +45,12 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error {
 	previous := mapsutil.NewSyncLockMap[string, any]()
 
 	for _, req := range g.requests {
+		select {
+		case <-ctx.Context().Done():
+			return ctx.Context().Err()
+		default:
+		}
+
 		inputItem := ctx.Input.Clone()
 		if g.options.InputHelper != nil && ctx.Input.MetaInput.Input != "" {
 			if inputItem.MetaInput.Input = g.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" {
diff --git a/pkg/tmplexec/multiproto/multi.go b/pkg/tmplexec/multiproto/multi.go
index 997e62243d..58858f971e 100644
--- a/pkg/tmplexec/multiproto/multi.go
+++ b/pkg/tmplexec/multiproto/multi.go
@@ -44,6 +44,12 @@ func (m *MultiProtocol) Compile() error {
 
 // ExecuteWithResults executes the template and returns results
 func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
+	select {
+	case <-ctx.Context().Done():
+		return ctx.Context().Err()
+	default:
+	}
+
 	// put all readonly args into template context
 	m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(m.readOnlyArgs)
 
@@ -96,6 +102,12 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
 
 	// execute all protocols in the queue
 	for _, req := range m.requests {
+		select {
+		case <-ctx.Context().Done():
+			return ctx.Context().Err()
+		default:
+		}
+
 		values := m.options.GetTemplateCtx(ctx.Input.MetaInput).GetAll()
 		err := req.ExecuteWithResults(ctx.Input, output.InternalEvent(values), nil, multiProtoCallback)
 		// if error skip execution of next protocols
diff --git a/pkg/tmplexec/multiproto/multi_test.go b/pkg/tmplexec/multiproto/multi_test.go
index b632680298..4f2aa25e4e 100644
--- a/pkg/tmplexec/multiproto/multi_test.go
+++ b/pkg/tmplexec/multiproto/multi_test.go
@@ -54,8 +54,8 @@ func TestMultiProtoWithDynamicExtractor(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("blog.projectdiscovery.io")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	ctx := scan.NewScanContext(context.Background(), input)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
 	require.True(t, gotresults)
@@ -71,8 +71,8 @@ func TestMultiProtoWithProtoPrefix(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("blog.projectdiscovery.io")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	ctx := scan.NewScanContext(context.Background(), input)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
 	require.True(t, gotresults)

From c8cda14e4183e1c2fe4551368cc7bd84ae479ff8 Mon Sep 17 00:00:00 2001
From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
Date: Thu, 25 Apr 2024 13:58:37 +0300
Subject: [PATCH 56/58] remove default val in CLI and increase `MaxBodyRead` to
 10mb (#5100)

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
---
 cmd/nuclei/main.go            | 2 +-
 pkg/protocols/http/request.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
index a084f270b4..242a902f55 100644
--- a/cmd/nuclei/main.go
+++ b/cmd/nuclei/main.go
@@ -296,7 +296,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.Interface, "interface", "i", "", "network interface to use for network scan"),
 		flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"),
 		flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"),
-		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 10*1024*1024, "max response size to read in bytes"),
+		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
 		flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"),
 		flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"),
 		flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go
index 545247ef56..b2628f007a 100644
--- a/pkg/protocols/http/request.go
+++ b/pkg/protocols/http/request.go
@@ -53,7 +53,7 @@ const (
 )
 
 var (
-	MaxBodyRead = int64(1 << 22) // 4MB using shift operator
+	MaxBodyRead = int64(10 * 1024 * 1024) // 10MB
 	// ErrMissingVars is error occured when variables are missing
 	ErrMissingVars = errors.New("stop execution due to unresolved variables")
 )

From 295f45807e65e1bee16dce573b217a0dfff67002 Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Thu, 25 Apr 2024 16:29:52 +0530
Subject: [PATCH 57/58] version update

---
 pkg/catalog/config/constants.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go
index 3dca0be63b..2b6793634f 100644
--- a/pkg/catalog/config/constants.go
+++ b/pkg/catalog/config/constants.go
@@ -31,7 +31,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.2.4`
+	Version = `v3.2.5`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

From 7ce1b3e43d92a56a33ddf2ebf3e15876fb39388d Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
Date: Thu, 25 Apr 2024 16:55:48 +0530
Subject: [PATCH 58/58] flow: fix empty template.xxx in flow (#5106)

---
 pkg/tmplexec/flow/flow_executor.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go
index c3f0014cab..6a1813efd9 100644
--- a/pkg/tmplexec/flow/flow_executor.go
+++ b/pkg/tmplexec/flow/flow_executor.go
@@ -226,7 +226,11 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
 		}
 	}
 	// register template object
-	if err := runtime.Set("template", f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()); err != nil {
+	tmplObj := f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()
+	if tmplObj == nil {
+		tmplObj = map[string]interface{}{}
+	}
+	if err := runtime.Set("template", tmplObj); err != nil {
 		return err
 	}