From 7fe5786c2469ae81e328db625e20b3947aa8541a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Sun, 28 Apr 2019 19:36:48 +0300 Subject: [PATCH] Refactor calibration and filter addition / removal to correct modules. (#34) --- main.go | 151 +++++++++++++++---------------------------- pkg/ffuf/config.go | 28 ++++++++ pkg/ffuf/job.go | 58 ++++++++--------- pkg/ffuf/util.go | 17 ++++- pkg/filter/filter.go | 62 ++++++++++++++++++ 5 files changed, 188 insertions(+), 128 deletions(-) diff --git a/main.go b/main.go index 8663ba2..a972bca 100644 --- a/main.go +++ b/main.go @@ -90,56 +90,27 @@ func main() { flag.Usage() os.Exit(1) } - if err := prepareFilters(&opts, &conf); err != nil { - fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err) - flag.Usage() - os.Exit(1) - } - job, err := prepareJob(&conf) if err != nil { fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err) flag.Usage() os.Exit(1) } + if err := prepareFilters(&opts, &conf); err != nil { + fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err) + flag.Usage() + os.Exit(1) + } - if conf.AutoCalibration { - // Handle the calibration - responses, err := job.CalibrateResponses() - if err != nil { - fmt.Fprintf(os.Stderr, "Error in autocalibration, exiting: %s\n", err) - os.Exit(1) - } - if len(responses) > 0 { - calibrateFilters(responses, &conf) - } + if err := filter.CalibrateIfNeeded(job); err != nil { + fmt.Fprintf(os.Stderr, "Error in autocalibration, exiting: %s\n", err) + os.Exit(1) } // Job handles waiting for goroutines to complete itself job.Start() } -func calibrateFilters(responses []ffuf.Response, conf *ffuf.Config) { - sizeCalib := make([]string, 0) - wordCalib := make([]string, 0) - for _, r := range responses { - if r.ContentLength > 1 { - // Only add if we have an actual size of responses - sizeCalib = append(sizeCalib, strconv.FormatInt(r.ContentLength, 10)) - } - if r.ContentWords > 1 { - // Only add if we have an actual word length of response - wordCalib = append(wordCalib, strconv.FormatInt(r.ContentWords, 10)) - } - } - if len(sizeCalib) > 0 { - addFilter(conf, "size", strings.Join(sizeCalib, ",")) - } - if len(wordCalib) > 0 { - addFilter(conf, "word", strings.Join(wordCalib, ",")) - } -} - func prepareJob(conf *ffuf.Config) (*ffuf.Job, error) { errs := ffuf.NewMultierror() // TODO: implement error handling for runnerprovider and outputprovider @@ -160,6 +131,51 @@ func prepareJob(conf *ffuf.Config) (*ffuf.Job, error) { }, errs.ErrorOrNil() } +func prepareFilters(parseOpts *cliOptions, conf *ffuf.Config) error { + errs := ffuf.NewMultierror() + if parseOpts.filterStatus != "" { + if err := filter.AddFilter(conf, "status", parseOpts.filterStatus); err != nil { + errs.Add(err) + } + } + if parseOpts.filterSize != "" { + if err := filter.AddFilter(conf, "size", parseOpts.filterSize); err != nil { + errs.Add(err) + } + } + if parseOpts.filterRegexp != "" { + if err := filter.AddFilter(conf, "regexp", parseOpts.filterRegexp); err != nil { + errs.Add(err) + } + } + if parseOpts.filterWords != "" { + if err := filter.AddFilter(conf, "word", parseOpts.filterWords); err != nil { + errs.Add(err) + } + } + if parseOpts.matcherStatus != "" { + if err := filter.AddMatcher(conf, "status", parseOpts.matcherStatus); err != nil { + errs.Add(err) + } + } + if parseOpts.matcherSize != "" { + if err := filter.AddMatcher(conf, "size", parseOpts.matcherSize); err != nil { + errs.Add(err) + } + } + if parseOpts.matcherRegexp != "" { + if err := filter.AddMatcher(conf, "regexp", parseOpts.matcherRegexp); err != nil { + errs.Add(err) + } + } + if parseOpts.matcherWords != "" { + if err := filter.AddMatcher(conf, "word", parseOpts.matcherWords); err != nil { + errs.Add(err) + } + } + return errs.ErrorOrNil() +} + func prepareConfig(parseOpts *cliOptions, conf *ffuf.Config) error { //TODO: refactor in a proper flag library that can handle things like required flags errs := ffuf.NewMultierror() @@ -263,64 +279,3 @@ func prepareConfig(parseOpts *cliOptions, conf *ffuf.Config) error { return errs.ErrorOrNil() } - -func prepareFilters(parseOpts *cliOptions, conf *ffuf.Config) error { - errs := ffuf.NewMultierror() - if parseOpts.filterStatus != "" { - if err := addFilter(conf, "status", parseOpts.filterStatus); err != nil { - errs.Add(err) - } - } - if parseOpts.filterSize != "" { - if err := addFilter(conf, "size", parseOpts.filterSize); err != nil { - errs.Add(err) - } - } - if parseOpts.filterRegexp != "" { - if err := addFilter(conf, "regexp", parseOpts.filterRegexp); err != nil { - errs.Add(err) - } - } - if parseOpts.filterWords != "" { - if err := addFilter(conf, "word", parseOpts.filterWords); err != nil { - errs.Add(err) - } - } - if parseOpts.matcherStatus != "" { - if err := addMatcher(conf, "status", parseOpts.matcherStatus); err != nil { - errs.Add(err) - } - } - if parseOpts.matcherSize != "" { - if err := addMatcher(conf, "size", parseOpts.matcherSize); err != nil { - errs.Add(err) - } - } - if parseOpts.matcherRegexp != "" { - if err := addMatcher(conf, "regexp", parseOpts.matcherRegexp); err != nil { - errs.Add(err) - } - } - if parseOpts.matcherWords != "" { - if err := addMatcher(conf, "word", parseOpts.matcherWords); err != nil { - errs.Add(err) - } - } - return errs.ErrorOrNil() -} - -func addFilter(conf *ffuf.Config, name string, option string) error { - newf, err := filter.NewFilterByName(name, option) - if err == nil { - conf.Filters = append(conf.Filters, newf) - } - return err -} - -func addMatcher(conf *ffuf.Config, name string, option string) error { - newf, err := filter.NewFilterByName(name, option) - if err == nil { - conf.Matchers = append(conf.Matchers, newf) - } - return err -} diff --git a/pkg/ffuf/config.go b/pkg/ffuf/config.go index 1ad37f6..737c805 100644 --- a/pkg/ffuf/config.go +++ b/pkg/ffuf/config.go @@ -69,3 +69,31 @@ func NewConfig(ctx context.Context) Config { conf.DirSearchCompat = false return conf } + +type CliOptions struct { + extensions string + delay string + filterStatus string + filterSize string + filterRegexp string + filterWords string + matcherStatus string + matcherSize string + matcherRegexp string + matcherWords string + proxyURL string + outputFormat string + headers multiStringFlag + showVersion bool +} + +type multiStringFlag []string + +func (m *multiStringFlag) String() string { + return "" +} + +func (m *multiStringFlag) Set(value string) error { + *m = append(*m, value) + return nil +} diff --git a/pkg/ffuf/job.go b/pkg/ffuf/job.go index a0d590c..f0ec2a3 100644 --- a/pkg/ffuf/job.go +++ b/pkg/ffuf/job.go @@ -129,35 +129,6 @@ func (j *Job) updateProgress() { j.Output.Progress(prog) } -//Calibrate runs a self-calibration task for filtering options, requesting random resources and acting accordingly -func (j *Job) CalibrateResponses() ([]Response, error) { - cInputs := make([]string, 0) - cInputs = append(cInputs, "admin"+randomString(16)+"/") - cInputs = append(cInputs, ".htaccess"+randomString(16)) - cInputs = append(cInputs, randomString(16)+"/") - cInputs = append(cInputs, randomString(16)) - - results := make([]Response, 0) - for _, input := range cInputs { - req, err := j.Runner.Prepare([]byte(input)) - if err != nil { - j.Output.Error(fmt.Sprintf("Encountered an error while preparing request: %s\n", err)) - j.incError() - return results, err - } - resp, err := j.Runner.Execute(&req) - if err != nil { - return results, err - } - - // Only calibrate on responses that would be matched otherwise - if j.isMatch(resp) { - results = append(results, resp) - } - } - return results, nil -} - func (j *Job) isMatch(resp Response) bool { matched := false for _, m := range j.Config.Matchers { @@ -218,6 +189,35 @@ func (j *Job) runTask(input []byte, retried bool) { return } +//CalibrateResponses returns slice of Responses for randomly generated filter autocalibration requests +func (j *Job) CalibrateResponses() ([]Response, error) { + cInputs := make([]string, 0) + cInputs = append(cInputs, "admin"+RandomString(16)+"/") + cInputs = append(cInputs, ".htaccess"+RandomString(16)) + cInputs = append(cInputs, RandomString(16)+"/") + cInputs = append(cInputs, RandomString(16)) + + results := make([]Response, 0) + for _, input := range cInputs { + req, err := j.Runner.Prepare([]byte(input)) + if err != nil { + j.Output.Error(fmt.Sprintf("Encountered an error while preparing request: %s\n", err)) + j.incError() + return results, err + } + resp, err := j.Runner.Execute(&req) + if err != nil { + return results, err + } + + // Only calibrate on responses that would be matched otherwise + if j.isMatch(resp) { + results = append(results, resp) + } + } + return results, nil +} + func (j *Job) CheckStop() { if j.Counter > 50 { // We have enough samples diff --git a/pkg/ffuf/util.go b/pkg/ffuf/util.go index db7edea..ef12ffc 100644 --- a/pkg/ffuf/util.go +++ b/pkg/ffuf/util.go @@ -7,10 +7,25 @@ import ( //used for random string generation in calibration function var chars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") -func randomString(n int) string { +//RandomString returns a random string of length of parameter n +func RandomString(n int) string { s := make([]rune, n) for i := range s { s[i] = chars[rand.Intn(len(chars))] } return string(s) } + +//UniqStringSlice returns an unordered slice of unique strings. The duplicates are dropped +func UniqStringSlice(inslice []string) []string { + found := map[string]bool{} + + for _, v := range inslice { + found[v] = true + } + ret := []string{} + for k, _ := range found { + ret = append(ret, k) + } + return ret +} diff --git a/pkg/filter/filter.go b/pkg/filter/filter.go index c9041ac..44e9043 100644 --- a/pkg/filter/filter.go +++ b/pkg/filter/filter.go @@ -2,6 +2,8 @@ package filter import ( "fmt" + "strconv" + "strings" "github.com/ffuf/ffuf/pkg/ffuf" ) @@ -21,3 +23,63 @@ func NewFilterByName(name string, value string) (ffuf.FilterProvider, error) { } return nil, fmt.Errorf("Could not create filter with name %s", name) } + +//AddFilter adds a new filter to Config +func AddFilter(conf *ffuf.Config, name string, option string) error { + newf, err := NewFilterByName(name, option) + if err == nil { + conf.Filters = append(conf.Filters, newf) + } + return err +} + +//AddMatcher adds a new matcher to Config +func AddMatcher(conf *ffuf.Config, name string, option string) error { + newf, err := NewFilterByName(name, option) + if err == nil { + conf.Matchers = append(conf.Matchers, newf) + } + return err +} + +//CalibrateIfNeeded runs a self-calibration task for filtering options (if needed) by requesting random resources and acting accordingly +func CalibrateIfNeeded(j *ffuf.Job) error { + if !j.Config.AutoCalibration { + return nil + } + // Handle the calibration + responses, err := j.CalibrateResponses() + if err != nil { + return err + } + if len(responses) > 0 { + calibrateFilters(j, responses) + } + return nil +} + +func calibrateFilters(j *ffuf.Job, responses []ffuf.Response) { + sizeCalib := make([]string, 0) + wordCalib := make([]string, 0) + for _, r := range responses { + if r.ContentLength > 1 { + // Only add if we have an actual size of responses + sizeCalib = append(sizeCalib, strconv.FormatInt(r.ContentLength, 10)) + } + if r.ContentWords > 1 { + // Only add if we have an actual word length of response + wordCalib = append(wordCalib, strconv.FormatInt(r.ContentWords, 10)) + } + } + + //Remove duplicates + sizeCalib = ffuf.UniqStringSlice(sizeCalib) + wordCalib = ffuf.UniqStringSlice(wordCalib) + + if len(sizeCalib) > 0 { + AddFilter(j.Config, "size", strings.Join(sizeCalib, ",")) + } + if len(wordCalib) > 0 { + AddFilter(j.Config, "word", strings.Join(wordCalib, ",")) + } +}