Refactor calibration and filter addition / removal to correct modules. (#34)

main.go

@@ -90,56 +90,27 @@ func main() {
 		flag.Usage()
 		os.Exit(1)
 	}
-	if err := prepareFilters(&opts, &conf); err != nil {
-		fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err)
-		flag.Usage()
-		os.Exit(1)
-	}
-
 	job, err := prepareJob(&conf)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err)
 		flag.Usage()
 		os.Exit(1)
 	}
-
+	if err := prepareFilters(&opts, &conf); err != nil {
-	if conf.AutoCalibration {
+		fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err)
-		// Handle the calibration
+		flag.Usage()
-		responses, err := job.CalibrateResponses()
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error in autocalibration, exiting: %s\n", err)
 		os.Exit(1)
 	}
-		if len(responses) > 0 {
+
-			calibrateFilters(responses, &conf)
+	if err := filter.CalibrateIfNeeded(job); err != nil {
-		}
+		fmt.Fprintf(os.Stderr, "Error in autocalibration, exiting: %s\n", err)
+		os.Exit(1)
 	}
 
 	// Job handles waiting for goroutines to complete itself
 	job.Start()
 }
 
-func calibrateFilters(responses []ffuf.Response, conf *ffuf.Config) {
-	sizeCalib := make([]string, 0)
-	wordCalib := make([]string, 0)
-	for _, r := range responses {
-		if r.ContentLength > 1 {
-			// Only add if we have an actual size of responses
-			sizeCalib = append(sizeCalib, strconv.FormatInt(r.ContentLength, 10))
-		}
-		if r.ContentWords > 1 {
-			// Only add if we have an actual word length of response
-			wordCalib = append(wordCalib, strconv.FormatInt(r.ContentWords, 10))
-		}
-	}
-	if len(sizeCalib) > 0 {
-		addFilter(conf, "size", strings.Join(sizeCalib, ","))
-	}
-	if len(wordCalib) > 0 {
-		addFilter(conf, "word", strings.Join(wordCalib, ","))
-	}
-}
-
 func prepareJob(conf *ffuf.Config) (*ffuf.Job, error) {
 	errs := ffuf.NewMultierror()
 	// TODO: implement error handling for runnerprovider and outputprovider
@@ -160,6 +131,51 @@ func prepareJob(conf *ffuf.Config) (*ffuf.Job, error) {
 	}, errs.ErrorOrNil()
 }
 
+func prepareFilters(parseOpts *cliOptions, conf *ffuf.Config) error {
+	errs := ffuf.NewMultierror()
+	if parseOpts.filterStatus != "" {
+		if err := filter.AddFilter(conf, "status", parseOpts.filterStatus); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.filterSize != "" {
+		if err := filter.AddFilter(conf, "size", parseOpts.filterSize); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.filterRegexp != "" {
+		if err := filter.AddFilter(conf, "regexp", parseOpts.filterRegexp); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.filterWords != "" {
+		if err := filter.AddFilter(conf, "word", parseOpts.filterWords); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.matcherStatus != "" {
+		if err := filter.AddMatcher(conf, "status", parseOpts.matcherStatus); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.matcherSize != "" {
+		if err := filter.AddMatcher(conf, "size", parseOpts.matcherSize); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.matcherRegexp != "" {
+		if err := filter.AddMatcher(conf, "regexp", parseOpts.matcherRegexp); err != nil {
+			errs.Add(err)
+		}
+	}
+	if parseOpts.matcherWords != "" {
+		if err := filter.AddMatcher(conf, "word", parseOpts.matcherWords); err != nil {
+			errs.Add(err)
+		}
+	}
+	return errs.ErrorOrNil()
+}
+
 func prepareConfig(parseOpts *cliOptions, conf *ffuf.Config) error {
 	//TODO: refactor in a proper flag library that can handle things like required flags
 	errs := ffuf.NewMultierror()
@@ -263,64 +279,3 @@ func prepareConfig(parseOpts *cliOptions, conf *ffuf.Config) error {
 
 	return errs.ErrorOrNil()
 }
-
-func prepareFilters(parseOpts *cliOptions, conf *ffuf.Config) error {
-	errs := ffuf.NewMultierror()
-	if parseOpts.filterStatus != "" {
-		if err := addFilter(conf, "status", parseOpts.filterStatus); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.filterSize != "" {
-		if err := addFilter(conf, "size", parseOpts.filterSize); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.filterRegexp != "" {
-		if err := addFilter(conf, "regexp", parseOpts.filterRegexp); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.filterWords != "" {
-		if err := addFilter(conf, "word", parseOpts.filterWords); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.matcherStatus != "" {
-		if err := addMatcher(conf, "status", parseOpts.matcherStatus); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.matcherSize != "" {
-		if err := addMatcher(conf, "size", parseOpts.matcherSize); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.matcherRegexp != "" {
-		if err := addMatcher(conf, "regexp", parseOpts.matcherRegexp); err != nil {
-			errs.Add(err)
-		}
-	}
-	if parseOpts.matcherWords != "" {
-		if err := addMatcher(conf, "word", parseOpts.matcherWords); err != nil {
-			errs.Add(err)
-		}
-	}
-	return errs.ErrorOrNil()
-}
-
-func addFilter(conf *ffuf.Config, name string, option string) error {
-	newf, err := filter.NewFilterByName(name, option)
-	if err == nil {
-		conf.Filters = append(conf.Filters, newf)
-	}
-	return err
-}
-
-func addMatcher(conf *ffuf.Config, name string, option string) error {
-	newf, err := filter.NewFilterByName(name, option)
-	if err == nil {
-		conf.Matchers = append(conf.Matchers, newf)
-	}
-	return err
-}

pkg/ffuf/config.go

@@ -69,3 +69,31 @@ func NewConfig(ctx context.Context) Config {
 	conf.DirSearchCompat = false
 	return conf
 }
+
+type CliOptions struct {
+	extensions    string
+	delay         string
+	filterStatus  string
+	filterSize    string
+	filterRegexp  string
+	filterWords   string
+	matcherStatus string
+	matcherSize   string
+	matcherRegexp string
+	matcherWords  string
+	proxyURL      string
+	outputFormat  string
+	headers       multiStringFlag
+	showVersion   bool
+}
+
+type multiStringFlag []string
+
+func (m *multiStringFlag) String() string {
+	return ""
+}
+
+func (m *multiStringFlag) Set(value string) error {
+	*m = append(*m, value)
+	return nil
+}

pkg/ffuf/job.go

@@ -129,35 +129,6 @@ func (j *Job) updateProgress() {
 	j.Output.Progress(prog)
 }
 
-//Calibrate runs a self-calibration task for filtering options, requesting random resources and acting accordingly
-func (j *Job) CalibrateResponses() ([]Response, error) {
-	cInputs := make([]string, 0)
-	cInputs = append(cInputs, "admin"+randomString(16)+"/")
-	cInputs = append(cInputs, ".htaccess"+randomString(16))
-	cInputs = append(cInputs, randomString(16)+"/")
-	cInputs = append(cInputs, randomString(16))
-
-	results := make([]Response, 0)
-	for _, input := range cInputs {
-		req, err := j.Runner.Prepare([]byte(input))
-		if err != nil {
-			j.Output.Error(fmt.Sprintf("Encountered an error while preparing request: %s\n", err))
-			j.incError()
-			return results, err
-		}
-		resp, err := j.Runner.Execute(&req)
-		if err != nil {
-			return results, err
-		}
-
-		// Only calibrate on responses that would be matched otherwise
-		if j.isMatch(resp) {
-			results = append(results, resp)
-		}
-	}
-	return results, nil
-}
-
 func (j *Job) isMatch(resp Response) bool {
 	matched := false
 	for _, m := range j.Config.Matchers {
@@ -218,6 +189,35 @@ func (j *Job) runTask(input []byte, retried bool) {
 	return
 }
 
+//CalibrateResponses returns slice of Responses for randomly generated filter autocalibration requests
+func (j *Job) CalibrateResponses() ([]Response, error) {
+	cInputs := make([]string, 0)
+	cInputs = append(cInputs, "admin"+RandomString(16)+"/")
+	cInputs = append(cInputs, ".htaccess"+RandomString(16))
+	cInputs = append(cInputs, RandomString(16)+"/")
+	cInputs = append(cInputs, RandomString(16))
+
+	results := make([]Response, 0)
+	for _, input := range cInputs {
+		req, err := j.Runner.Prepare([]byte(input))
+		if err != nil {
+			j.Output.Error(fmt.Sprintf("Encountered an error while preparing request: %s\n", err))
+			j.incError()
+			return results, err
+		}
+		resp, err := j.Runner.Execute(&req)
+		if err != nil {
+			return results, err
+		}
+
+		// Only calibrate on responses that would be matched otherwise
+		if j.isMatch(resp) {
+			results = append(results, resp)
+		}
+	}
+	return results, nil
+}
+
 func (j *Job) CheckStop() {
 	if j.Counter > 50 {
 		// We have enough samples

pkg/ffuf/util.go

@@ -7,10 +7,25 @@ import (
 //used for random string generation in calibration function
 var chars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
 
-func randomString(n int) string {
+//RandomString returns a random string of length of parameter n
+func RandomString(n int) string {
 	s := make([]rune, n)
 	for i := range s {
 		s[i] = chars[rand.Intn(len(chars))]
 	}
 	return string(s)
 }
+
+//UniqStringSlice returns an unordered slice of unique strings. The duplicates are dropped
+func UniqStringSlice(inslice []string) []string {
+	found := map[string]bool{}
+
+	for _, v := range inslice {
+		found[v] = true
+	}
+	ret := []string{}
+	for k, _ := range found {
+		ret = append(ret, k)
+	}
+	return ret
+}

pkg/filter/filter.go

@@ -2,6 +2,8 @@ package filter
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
 
 	"github.com/ffuf/ffuf/pkg/ffuf"
 )
@@ -21,3 +23,63 @@ func NewFilterByName(name string, value string) (ffuf.FilterProvider, error) {
 	}
 	return nil, fmt.Errorf("Could not create filter with name %s", name)
 }
+
+//AddFilter adds a new filter to Config
+func AddFilter(conf *ffuf.Config, name string, option string) error {
+	newf, err := NewFilterByName(name, option)
+	if err == nil {
+		conf.Filters = append(conf.Filters, newf)
+	}
+	return err
+}
+
+//AddMatcher adds a new matcher to Config
+func AddMatcher(conf *ffuf.Config, name string, option string) error {
+	newf, err := NewFilterByName(name, option)
+	if err == nil {
+		conf.Matchers = append(conf.Matchers, newf)
+	}
+	return err
+}
+
+//CalibrateIfNeeded runs a self-calibration task for filtering options (if needed) by requesting random resources and acting accordingly
+func CalibrateIfNeeded(j *ffuf.Job) error {
+	if !j.Config.AutoCalibration {
+		return nil
+	}
+	// Handle the calibration
+	responses, err := j.CalibrateResponses()
+	if err != nil {
+		return err
+	}
+	if len(responses) > 0 {
+		calibrateFilters(j, responses)
+	}
+	return nil
+}
+
+func calibrateFilters(j *ffuf.Job, responses []ffuf.Response) {
+	sizeCalib := make([]string, 0)
+	wordCalib := make([]string, 0)
+	for _, r := range responses {
+		if r.ContentLength > 1 {
+			// Only add if we have an actual size of responses
+			sizeCalib = append(sizeCalib, strconv.FormatInt(r.ContentLength, 10))
+		}
+		if r.ContentWords > 1 {
+			// Only add if we have an actual word length of response
+			wordCalib = append(wordCalib, strconv.FormatInt(r.ContentWords, 10))
+		}
+	}
+
+	//Remove duplicates
+	sizeCalib = ffuf.UniqStringSlice(sizeCalib)
+	wordCalib = ffuf.UniqStringSlice(wordCalib)
+
+	if len(sizeCalib) > 0 {
+		AddFilter(j.Config, "size", strings.Join(sizeCalib, ","))
+	}
+	if len(wordCalib) > 0 {
+		AddFilter(j.Config, "word", strings.Join(wordCalib, ","))
+	}
+}