feat: added tracking for status code, waf-detection & grouped errors

cmd/nuclei/main.go

@@ -446,6 +446,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "display statistics in JSONL(ines) format"),
 		flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"),
 		flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", 9092, "port to expose nuclei metrics on"),
+		flagSet.BoolVarP(&options.HTTPStats, "http-stats", "hps", false, "enable http status capturing (experimental)"),
 	)
 
 	flagSet.CreateGroup("cloud", "Cloud",

internal/runner/runner.go

@@ -19,6 +19,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
+	outputstats "github.com/projectdiscovery/nuclei/v3/pkg/output/stats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan/events"
 	uncoverlib "github.com/projectdiscovery/uncover"
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
@@ -92,6 +93,8 @@ type Runner struct {
 	pdcpUploadErrMsg   string
 	inputProvider      provider.InputProvider
 	fuzzFrequencyCache *frequency.Tracker
+	httpStats          *outputstats.Tracker
+
 	//general purpose temporary directory
 	tmpDir          string
 	parser          parser.Parser
@@ -256,6 +259,10 @@ func New(options *types.Options) (*Runner, error) {
 	}
 	// setup a proxy writer to automatically upload results to PDCP
 	runner.output = runner.setupPDCPUpload(outputWriter)
+	if options.HTTPStats {
+		runner.httpStats = outputstats.NewTracker()
+		runner.output = output.NewMultiWriter(runner.output, output.NewTrackerWriter(runner.httpStats))
+	}
 
 	if options.JSONL && options.EnableProgressBar {
 		options.StatsJSON = true
@@ -362,6 +369,9 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions,
 
 // Close releases all the resources and cleans up
 func (r *Runner) Close() {
+	if r.httpStats != nil {
+		r.httpStats.DisplayTopStats(r.options.NoColor)
+	}
 	// dump hosterrors cache
 	if r.hostErrors != nil {
 		r.hostErrors.Close()

pkg/output/multi_writer.go

@@ -8,6 +8,8 @@ type MultiWriter struct {
 	writers []Writer
 }
 
+var _ Writer = &MultiWriter{}
+
 // NewMultiWriter creates a new MultiWriter instance
 func NewMultiWriter(writers ...Writer) *MultiWriter {
 	return &MultiWriter{writers: writers}
@@ -57,3 +59,9 @@ func (mw *MultiWriter) WriteStoreDebugData(host, templateID, eventType string, d
 		writer.WriteStoreDebugData(host, templateID, eventType, data)
 	}
 }
+
+func (mw *MultiWriter) RequestStatsLog(statusCode, response string) {
+	for _, writer := range mw.writers {
+		writer.RequestStatsLog(statusCode, response)
+	}
+}

pkg/output/output.go

@@ -50,6 +50,8 @@ type Writer interface {
 	WriteFailure(*InternalWrappedEvent) error
 	// Request logs a request in the trace log
 	Request(templateID, url, requestType string, err error)
+	// RequestStatsLog logs a request stats log
+	RequestStatsLog(statusCode, response string)
 	//  WriteStoreDebugData writes the request/response debug data to file
 	WriteStoreDebugData(host, templateID, eventType string, data string)
 }
@@ -75,6 +77,8 @@ type StandardWriter struct {
 	KeysToRedact          []string
 }
 
+var _ Writer = &StandardWriter{}
+
 var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
 
 // InternalEvent is an internal output generation structure for nuclei.
@@ -351,15 +355,33 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques
 	if w.traceFile == nil && w.errorFile == nil {
 		return
 	}
+
+	request := getJSONLogRequestFromError(templatePath, input, requestType, requestErr)
+	if w.timestamp {
+		ts := time.Now()
+		request.Timestamp = &ts
+	}
+	data, err := jsoniter.Marshal(request)
+	if err != nil {
+		return
+	}
+
+	if w.traceFile != nil {
+		_, _ = w.traceFile.Write(data)
+	}
+
+	if requestErr != nil && w.errorFile != nil {
+		_, _ = w.errorFile.Write(data)
+	}
+}
+
+func getJSONLogRequestFromError(templatePath, input, requestType string, requestErr error) *JSONLogRequest {
 	request := &JSONLogRequest{
 		Template: templatePath,
 		Input:    input,
 		Type:     requestType,
 	}
-	if w.timestamp {
-		ts := time.Now()
-		request.Timestamp = &ts
-	}
+
 	parsed, _ := urlutil.ParseAbsoluteURL(input, false)
 	if parsed != nil {
 		request.Address = parsed.Hostname()
@@ -397,18 +419,7 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques
 	if val := errkit.GetAttrValue(requestErr, "address"); val.Any() != nil {
 		request.Address = val.String()
 	}
-	data, err := jsoniter.Marshal(request)
-	if err != nil {
-		return
-	}
-
-	if w.traceFile != nil {
-		_, _ = w.traceFile.Write(data)
-	}
-
-	if requestErr != nil && w.errorFile != nil {
-		_, _ = w.errorFile.Write(data)
-	}
+	return request
 }
 
 // Colorizer returns the colorizer instance for writer
@@ -540,12 +551,14 @@ func tryParseCause(err error) error {
 	if strings.HasPrefix(msg, "ReadStatusLine:") {
 		// last index is actual error (from rawhttp)
 		parts := strings.Split(msg, ":")
-		return errkit.New("%s", strings.TrimSpace(parts[len(parts)-1]))
+		return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
 	}
 	if strings.Contains(msg, "read ") {
 		// same here
 		parts := strings.Split(msg, ":")
-		return errkit.New("%s", strings.TrimSpace(parts[len(parts)-1]))
+		return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
 	}
 	return err
 }
+
+func (w *StandardWriter) RequestStatsLog(statusCode, response string) {}

pkg/output/output_stats.go

@@ -0,0 +1,51 @@
+package output
+
+import (
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output/stats"
+)
+
+// StatsOutputWriter implements writer interface for stats observation
+type StatsOutputWriter struct {
+	colorizer aurora.Aurora
+	Tracker   *stats.Tracker
+}
+
+var _ Writer = &StatsOutputWriter{}
+
+// NewStatsOutputWriter returns a new StatsOutputWriter instance.
+func NewTrackerWriter(t *stats.Tracker) *StatsOutputWriter {
+	return &StatsOutputWriter{
+		colorizer: aurora.NewAurora(true),
+		Tracker:   t,
+	}
+}
+
+func (tw *StatsOutputWriter) Close() {}
+
+func (tw *StatsOutputWriter) Colorizer() aurora.Aurora {
+	return tw.colorizer
+}
+
+func (tw *StatsOutputWriter) Write(event *ResultEvent) error {
+	return nil
+}
+
+func (tw *StatsOutputWriter) WriteFailure(event *InternalWrappedEvent) error {
+	return nil
+}
+
+func (tw *StatsOutputWriter) Request(templateID, url, requestType string, err error) {
+	if err == nil {
+		return
+	}
+	jsonReq := getJSONLogRequestFromError(templateID, url, requestType, err)
+	tw.Tracker.TrackErrorKind(jsonReq.Error)
+}
+
+func (tw *StatsOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {}
+
+func (tw *StatsOutputWriter) RequestStatsLog(statusCode, response string) {
+	tw.Tracker.TrackStatusCode(statusCode)
+	tw.Tracker.TrackWAFDetected(response)
+}