Fixes to recursion and wordlist handling for queued jobs (#537)

CHANGELOG.md

@@ -2,6 +2,8 @@
 - master
   - New
   - Changed
+    - Fixed a bug with recursion, introduced in the 1.4.0 release
+    - Recursion now works better with multiple wordlists, disabling unnecessary wordlists for queued jobs where needed
   
 - v1.4.0
   - New

pkg/ffuf/interfaces.go

@@ -17,7 +17,9 @@ type RunnerProvider interface {
 
 //InputProvider interface handles the input data for RunnerProvider
 type InputProvider interface {
+	ActivateKeywords([]string)
 	AddProvider(InputProviderConfig) error
+	Keywords() []string
 	Next() bool
 	Position() int
 	Reset()
@@ -34,6 +36,9 @@ type InternalInputProvider interface {
 	IncrementPosition()
 	Value() []byte
 	Total() int
+	Active() bool
+	Enable()
+	Disable()
 }
 
 //OutputProvider is responsible of providing output from the RunnerProvider

pkg/ffuf/job.go

@@ -167,6 +167,17 @@ func (j *Job) jobsInQueue() bool {
 func (j *Job) prepareQueueJob() {
 	j.Config.Url = j.queuejobs[j.queuepos].Url
 	j.currentDepth = j.queuejobs[j.queuepos].depth
+
+	//Find all keywords present in new queued job
+	kws := j.Input.Keywords()
+	found_kws := make([]string, 0)
+	for _, k := range kws {
+		if RequestContainsKeyword(j.queuejobs[j.queuepos].req, k) {
+			found_kws = append(found_kws, k)
+		}
+	}
+	//And activate / disable inputproviders as needed
+	j.Input.ActivateKeywords(found_kws)
 	j.queuepos += 1
 }
 
@@ -407,7 +418,7 @@ func (j *Job) handleGreedyRecursionJob(resp Response) {
 	// Handle greedy recursion strategy. Match has been determined before calling handleRecursionJob
 	if j.Config.RecursionDepth == 0 || j.currentDepth < j.Config.RecursionDepth {
 		recUrl := resp.Request.Url + "/" + "FUZZ"
-		newJob := QueueJob{Url: recUrl, depth: j.currentDepth + 1}
+		newJob := QueueJob{Url: recUrl, depth: j.currentDepth + 1, req: RecursionRequest(j.Config, recUrl)}
 		j.queuejobs = append(j.queuejobs, newJob)
 		j.Output.Info(fmt.Sprintf("Adding a new job to the queue: %s", recUrl))
 	} else {
@@ -425,7 +436,7 @@ func (j *Job) handleDefaultRecursionJob(resp Response) {
 	}
 	if j.Config.RecursionDepth == 0 || j.currentDepth < j.Config.RecursionDepth {
 		// We have yet to reach the maximum recursion depth
-		newJob := QueueJob{Url: recUrl, depth: j.currentDepth + 1, req: BaseRequest(j.Config)}
+		newJob := QueueJob{Url: recUrl, depth: j.currentDepth + 1, req: RecursionRequest(j.Config, recUrl)}
 		j.queuejobs = append(j.queuejobs, newJob)
 		j.Output.Info(fmt.Sprintf("Adding a new job to the queue: %s", recUrl))
 	} else {

pkg/ffuf/request.go

@@ -32,6 +32,13 @@ func BaseRequest(conf *Config) Request {
 	return req
 }
 
+// RecursionRequest returns a base request for a recursion target
+func RecursionRequest(conf *Config, path string) Request {
+	r := BaseRequest(conf)
+	r.Url = path
+	return r
+}
+
 // CopyRequest performs a deep copy of a request and returns a new struct
 func CopyRequest(basereq *Request) Request {
 	var req Request

pkg/ffuf/util.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"strings"
 )
 
 //used for random string generation in calibration function
@@ -43,6 +44,28 @@ func FileExists(path string) bool {
 	return !md.IsDir()
 }
 
+//RequestContainsKeyword checks if a keyword is present in any field of a request
+func RequestContainsKeyword(req Request, kw string) bool {
+	if strings.Contains(req.Host, kw) {
+		return true
+	}
+	if strings.Contains(req.Url, kw) {
+		return true
+	}
+	if strings.Contains(req.Method, kw) {
+		return true
+	}
+	if strings.Contains(string(req.Data), kw) {
+		return true
+	}
+	for k, v := range req.Headers {
+		if strings.Contains(k, kw) || strings.Contains(v, kw) {
+			return true
+		}
+	}
+	return false
+}
+
 //Version returns the ffuf version string
 func Version() string {
 	return fmt.Sprintf("%s%s", VERSION, VERSION_APPENDIX)

pkg/input/command.go

@@ -12,6 +12,7 @@ import (
 type CommandInput struct {
 	config  *ffuf.Config
 	count   int
+	active  bool
 	keyword string
 	command string
 	shell   string
@@ -19,6 +20,7 @@ type CommandInput struct {
 
 func NewCommandInput(keyword string, value string, conf *ffuf.Config) (*CommandInput, error) {
 	var cmd CommandInput
+	cmd.active = true
 	cmd.keyword = keyword
 	cmd.config = conf
 	cmd.count = 0
@@ -74,3 +76,15 @@ func (c *CommandInput) Value() []byte {
 func (c *CommandInput) Total() int {
 	return c.config.InputNum
 }
+
+func (c *CommandInput) Active() bool {
+	return c.active
+}
+
+func (c *CommandInput) Enable() {
+	c.active = true
+}
+
+func (c *CommandInput) Disable() {
+	c.active = false
+}

pkg/input/input.go

@@ -51,11 +51,31 @@ func (i *MainInputProvider) AddProvider(provider ffuf.InputProviderConfig) error
 	return nil
 }
 
+// ActivateKeywords enables / disables wordlists based on list of active keywords
+func (i *MainInputProvider) ActivateKeywords(kws []string) {
+	for _, p := range i.Providers {
+		if sliceContains(kws, p.Keyword()) {
+			p.Active()
+		} else {
+			p.Disable()
+		}
+	}
+}
+
 //Position will return the current position of progress
 func (i *MainInputProvider) Position() int {
 	return i.position
 }
 
+//Keywords returns a slice of all keywords in the inputprovider
+func (i *MainInputProvider) Keywords() []string {
+	kws := make([]string, 0)
+	for _, p := range i.Providers {
+		kws = append(kws, p.Keyword())
+	}
+	return kws
+}
+
 //Next will increment the cursor position, and return a boolean telling if there's inputs left
 func (i *MainInputProvider) Next() bool {
 	if i.position >= i.Total() {
@@ -91,6 +111,10 @@ func (i *MainInputProvider) Reset() {
 func (i *MainInputProvider) pitchforkValue() map[string][]byte {
 	values := make(map[string][]byte)
 	for _, p := range i.Providers {
+		if !p.Active() {
+			// The inputprovider is disabled
+			continue
+		}
 		if !p.Next() {
 			// Loop to beginning if the inputprovider has been exhausted
 			p.ResetPosition()
@@ -108,7 +132,11 @@ func (i *MainInputProvider) clusterbombValue() map[string][]byte {
 	// Should we signal the next InputProvider in the slice to increment
 	signalNext := false
 	first := true
-	for index, p := range i.Providers {
+	index := 0
+	for _, p := range i.Providers {
+		if !p.Active() {
+			continue
+		}
 		if signalNext {
 			p.IncrementPosition()
 			signalNext = false
@@ -130,18 +158,24 @@ func (i *MainInputProvider) clusterbombValue() map[string][]byte {
 			p.IncrementPosition()
 			first = false
 		}
+		index += 1
 	}
 	return values
 }
 
 func (i *MainInputProvider) clusterbombIteratorReset() {
-	for index, p := range i.Providers {
+	index := 0
+	for _, p := range i.Providers {
+		if !p.Active() {
+			continue
+		}
 		if index < i.msbIterator {
 			p.ResetPosition()
 		}
 		if index == i.msbIterator {
 			p.IncrementPosition()
 		}
+		index += 1
 	}
 }
 
@@ -150,6 +184,9 @@ func (i *MainInputProvider) Total() int {
 	count := 0
 	if i.Config.InputMode == "pitchfork" {
 		for _, p := range i.Providers {
+			if !p.Active() {
+				continue
+			}
 			if p.Total() > count {
 				count = p.Total()
 			}
@@ -158,8 +195,21 @@ func (i *MainInputProvider) Total() int {
 	if i.Config.InputMode == "clusterbomb" || i.Config.InputMode == "sniper" {
 		count = 1
 		for _, p := range i.Providers {
+			if !p.Active() {
+				continue
+			}
 			count = count * p.Total()
 		}
 	}
 	return count
 }
+
+//sliceContains is a helper function that returns true if a string is included in a string slice
+func sliceContains(sslice []string, str string) bool {
+	for _, v := range sslice {
+		if v == str {
+			return true
+		}
+	}
+	return false
+}

pkg/input/wordlist.go

@@ -10,6 +10,7 @@ import (
 )
 
 type WordlistInput struct {
+	active   bool
 	config   *ffuf.Config
 	data     [][]byte
 	position int
@@ -18,6 +19,7 @@ type WordlistInput struct {
 
 func NewWordlistInput(keyword string, value string, conf *ffuf.Config) (*WordlistInput, error) {
 	var wl WordlistInput
+	wl.active = true
 	wl.keyword = keyword
 	wl.config = conf
 	wl.position = 0
@@ -75,6 +77,21 @@ func (w *WordlistInput) Total() int {
 	return len(w.data)
 }
 
+//Active returns boolean if the inputprovider is active
+func (w *WordlistInput) Active() bool {
+	return w.active
+}
+
+//Enable sets the inputprovider as active
+func (w *WordlistInput) Enable() {
+	w.active = true
+}
+
+//Disable disables the inputprovider
+func (w *WordlistInput) Disable() {
+	w.active = false
+}
+
 //validFile checks that the wordlist file exists and can be read
 func (w *WordlistInput) validFile(path string) (bool, error) {
 	_, err := os.Stat(path)