diff --git a/CHANGELOG.md b/CHANGELOG.md index 283c12d..d7ceaff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - Added full line colors - Added `-json` to emit newline delimited JSON output - Added 500 Internal Server Error to list of status codes matched by default + - New autocalibration options: `-ach`, `-ack` and `-acs`. Revamped the whole autocalibration process - Changed - Fixed an issue where output file was created regardless of `-or` - Fixed an issue where output (often a lot of it) would be printed after entering interactive mode diff --git a/ffufrc.example b/ffufrc.example index 059a8b8..19ea23d 100644 --- a/ffufrc.example +++ b/ffufrc.example @@ -27,6 +27,9 @@ "randomtest", "admin" ] + autocalibration_strategy = "basic" + autocalibration_keyword = "FUZZ" + autocalibration_perhost = false colors = false delay = "" maxtime = 0 diff --git a/help.go b/help.go index 15af7d5..42db3cf 100644 --- a/help.go +++ b/help.go @@ -61,7 +61,7 @@ func Usage() { Description: "", Flags: make([]UsageFlag, 0), Hidden: false, - ExpectedFlags: []string{"ac", "acc", "c", "config", "json", "maxtime", "maxtime-job", "noninteractive", "p", "rate", "s", "sa", "se", "sf", "t", "v", "V"}, + ExpectedFlags: []string{"ac", "acc", "ack", "ach", "acs", "c", "config", "json", "maxtime", "maxtime-job", "noninteractive", "p", "rate", "s", "sa", "se", "sf", "t", "v", "V"}, } u_compat := UsageSection{ Name: "COMPATIBILITY OPTIONS", diff --git a/main.go b/main.go index 48abe64..85ee064 100644 --- a/main.go +++ b/main.go @@ -4,13 +4,13 @@ import ( "context" "flag" "fmt" + "github.com/ffuf/ffuf/pkg/filter" "io/ioutil" "log" "os" "strings" "github.com/ffuf/ffuf/pkg/ffuf" - "github.com/ffuf/ffuf/pkg/filter" "github.com/ffuf/ffuf/pkg/input" "github.com/ffuf/ffuf/pkg/interactive" "github.com/ffuf/ffuf/pkg/output" @@ -62,6 +62,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions { flag.BoolVar(&ignored, "k", false, "Dummy flag for backwards compatibility") flag.BoolVar(&opts.Output.OutputSkipEmptyFile, "or", opts.Output.OutputSkipEmptyFile, "Don't create the output file if we don't have results") flag.BoolVar(&opts.General.AutoCalibration, "ac", opts.General.AutoCalibration, "Automatically calibrate filtering options") + flag.BoolVar(&opts.General.AutoCalibrationPerHost, "ach", opts.General.AutoCalibration, "Per host autocalibration") flag.BoolVar(&opts.General.Colors, "c", opts.General.Colors, "Colorize output.") flag.BoolVar(&opts.General.Json, "json", opts.General.Json, "JSON output, printing newline-delimited JSON records") flag.BoolVar(&opts.General.Noninteractive, "noninteractive", opts.General.Noninteractive, "Disable the interactive console functionality") @@ -84,6 +85,8 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions { flag.IntVar(&opts.HTTP.RecursionDepth, "recursion-depth", opts.HTTP.RecursionDepth, "Maximum recursion depth.") flag.IntVar(&opts.HTTP.Timeout, "timeout", opts.HTTP.Timeout, "HTTP request timeout in seconds.") flag.IntVar(&opts.Input.InputNum, "input-num", opts.Input.InputNum, "Number of inputs to test. Used in conjunction with --input-cmd.") + flag.StringVar(&opts.General.AutoCalibrationKeyword, "ack", opts.General.AutoCalibrationKeyword, "Autocalibration keyword") + flag.StringVar(&opts.General.AutoCalibrationStrategy, "acs", opts.General.AutoCalibrationStrategy, "Autocalibration strategy: \"basic\" or \"advanced\"") flag.StringVar(&opts.General.ConfigFile, "config", "", "Load configuration from a file") flag.StringVar(&opts.Filter.Lines, "fl", opts.Filter.Lines, "Filter by amount of lines in response. Comma separated list of line counts and ranges") flag.StringVar(&opts.Filter.Regexp, "fr", opts.Filter.Regexp, "Filter regexp") @@ -195,17 +198,13 @@ func main() { fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err) os.Exit(1) } - if err := filter.SetupFilters(opts, conf); err != nil { + if err := SetupFilters(opts, conf); err != nil { fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err) Usage() fmt.Fprintf(os.Stderr, "Encountered error(s): %s\n", err) os.Exit(1) } - if err := filter.CalibrateIfNeeded(job); err != nil { - fmt.Fprintf(os.Stderr, "Error in autocalibration, exiting: %s\n", err) - os.Exit(1) - } if !conf.Noninteractive { go func() { err := interactive.Handle(job) @@ -233,3 +232,104 @@ func prepareJob(conf *ffuf.Config) (*ffuf.Job, error) { job.Output = output.NewOutputProviderByName("stdout", conf) return job, errs.ErrorOrNil() } + +func SetupFilters(parseOpts *ffuf.ConfigOptions, conf *ffuf.Config) error { + errs := ffuf.NewMultierror() + conf.MatcherManager = filter.NewMatcherManager() + // If any other matcher is set, ignore -mc default value + matcherSet := false + statusSet := false + warningIgnoreBody := false + flag.Visit(func(f *flag.Flag) { + if f.Name == "mc" { + statusSet = true + } + if f.Name == "ms" { + matcherSet = true + warningIgnoreBody = true + } + if f.Name == "ml" { + matcherSet = true + warningIgnoreBody = true + } + if f.Name == "mr" { + matcherSet = true + } + if f.Name == "mt" { + matcherSet = true + } + if f.Name == "mw" { + matcherSet = true + warningIgnoreBody = true + } + }) + // Only set default matchers if no + if statusSet || !matcherSet { + if err := conf.MatcherManager.AddMatcher("status", parseOpts.Matcher.Status); err != nil { + errs.Add(err) + } + } + + if parseOpts.Filter.Status != "" { + if err := conf.MatcherManager.AddFilter("status", parseOpts.Filter.Status, false); err != nil { + errs.Add(err) + } + } + if parseOpts.Filter.Size != "" { + warningIgnoreBody = true + if err := conf.MatcherManager.AddFilter("size", parseOpts.Filter.Size, false); err != nil { + errs.Add(err) + } + } + if parseOpts.Filter.Regexp != "" { + if err := conf.MatcherManager.AddFilter("regexp", parseOpts.Filter.Regexp, false); err != nil { + errs.Add(err) + } + } + if parseOpts.Filter.Words != "" { + warningIgnoreBody = true + if err := conf.MatcherManager.AddFilter("word", parseOpts.Filter.Words, false); err != nil { + errs.Add(err) + } + } + if parseOpts.Filter.Lines != "" { + warningIgnoreBody = true + if err := conf.MatcherManager.AddFilter("line", parseOpts.Filter.Lines, false); err != nil { + errs.Add(err) + } + } + if parseOpts.Filter.Time != "" { + if err := conf.MatcherManager.AddFilter("time", parseOpts.Filter.Time, false); err != nil { + errs.Add(err) + } + } + if parseOpts.Matcher.Size != "" { + if err := conf.MatcherManager.AddMatcher("size", parseOpts.Matcher.Size); err != nil { + errs.Add(err) + } + } + if parseOpts.Matcher.Regexp != "" { + if err := conf.MatcherManager.AddMatcher("regexp", parseOpts.Matcher.Regexp); err != nil { + errs.Add(err) + } + } + if parseOpts.Matcher.Words != "" { + if err := conf.MatcherManager.AddMatcher("word", parseOpts.Matcher.Words); err != nil { + errs.Add(err) + } + } + if parseOpts.Matcher.Lines != "" { + if err := conf.MatcherManager.AddMatcher("line", parseOpts.Matcher.Lines); err != nil { + errs.Add(err) + } + } + if parseOpts.Matcher.Time != "" { + if err := conf.MatcherManager.AddFilter("time", parseOpts.Matcher.Time, false); err != nil { + errs.Add(err) + } + } + if conf.IgnoreBody && warningIgnoreBody { + fmt.Printf("*** Warning: possible undesired combination of -ignore-body and the response options: fl,fs,fw,ml,ms and mw.\n") + } + return errs.ErrorOrNil() +} diff --git a/pkg/ffuf/autocalibration.go b/pkg/ffuf/autocalibration.go new file mode 100644 index 0000000..62a269f --- /dev/null +++ b/pkg/ffuf/autocalibration.go @@ -0,0 +1,231 @@ +package ffuf + +import ( + "fmt" + "log" + "math/rand" + "strconv" + "time" +) + +func (j *Job) autoCalibrationStrings() map[string][]string { + rand.Seed(time.Now().UnixNano()) + cInputs := make(map[string][]string) + if len(j.Config.AutoCalibrationStrings) < 1 { + cInputs["basic_admin"] = append(cInputs["basic_admin"], "admin"+RandomString(16)) + cInputs["basic_admin"] = append(cInputs["basic_admin"], "admin"+RandomString(8)) + cInputs["htaccess"] = append(cInputs["htaccess"], ".htaccess"+RandomString(16)) + cInputs["htaccess"] = append(cInputs["htaccess"], ".htaccess"+RandomString(8)) + cInputs["basic_random"] = append(cInputs["basic_random"], RandomString(16)) + cInputs["basic_random"] = append(cInputs["basic_random"], RandomString(8)) + if j.Config.AutoCalibrationStrategy == "advanced" { + // Add directory tests and .htaccess too + cInputs["admin_dir"] = append(cInputs["admin_dir"], "admin"+RandomString(16)+"/") + cInputs["admin_dir"] = append(cInputs["admin_dir"], "admin"+RandomString(8)+"/") + cInputs["random_dir"] = append(cInputs["random_dir"], RandomString(16)+"/") + cInputs["random_dir"] = append(cInputs["random_dir"], RandomString(8)+"/") + } + } else { + cInputs["custom"] = append(cInputs["custom"], j.Config.AutoCalibrationStrings...) + } + return cInputs +} + +func (j *Job) calibrationRequest(inputs map[string][]byte) (Response, error) { + basereq := BaseRequest(j.Config) + req, err := j.Runner.Prepare(inputs, &basereq) + if err != nil { + j.Output.Error(fmt.Sprintf("Encountered an error while preparing autocalibration request: %s\n", err)) + j.incError() + log.Printf("%s", err) + return Response{}, err + } + resp, err := j.Runner.Execute(&req) + if err != nil { + j.Output.Error(fmt.Sprintf("Encountered an error while executing autocalibration request: %s\n", err)) + j.incError() + log.Printf("%s", err) + return Response{}, err + } + // Only calibrate on responses that would be matched otherwise + if j.isMatch(resp) { + return resp, nil + } + return resp, fmt.Errorf("Response wouldn't be matched") +} + +//CalibrateForHost runs autocalibration for a specific host +func (j *Job) CalibrateForHost(host string, input map[string][]byte) error { + if j.Config.MatcherManager.CalibratedForDomain(host) { + return nil + } + if input[j.Config.AutoCalibrationKeyword] == nil { + return fmt.Errorf("Autocalibration keyword \"%s\" not found in the request.", j.Config.AutoCalibrationKeyword) + } + cStrings := j.autoCalibrationStrings() + for _, v := range cStrings { + responses := make([]Response, 0) + for _, cs := range v { + input[j.Config.AutoCalibrationKeyword] = []byte(cs) + resp, err := j.calibrationRequest(input) + if err != nil { + continue + } + responses = append(responses, resp) + err = j.calibrateFilters(responses, true) + if err != nil { + j.Output.Error(fmt.Sprintf("%s", err)) + } + } + } + j.Config.MatcherManager.SetCalibratedForHost(host, true) + return nil +} + +//CalibrateResponses returns slice of Responses for randomly generated filter autocalibration requests +func (j *Job) Calibrate(input map[string][]byte) error { + if j.Config.MatcherManager.Calibrated() { + return nil + } + cInputs := j.autoCalibrationStrings() + + for _, v := range cInputs { + responses := make([]Response, 0) + for _, cs := range v { + input[j.Config.AutoCalibrationKeyword] = []byte(cs) + resp, err := j.calibrationRequest(input) + if err != nil { + continue + } + responses = append(responses, resp) + err = j.calibrateFilters(responses, false) + if err != nil { + j.Output.Error(fmt.Sprintf("%s", err)) + } + } + } + j.Config.MatcherManager.SetCalibrated(true) + return nil +} + +//CalibrateIfNeeded runs a self-calibration task for filtering options (if needed) by requesting random resources and +// configuring the filters accordingly +func (j *Job) CalibrateIfNeeded(host string, input map[string][]byte) error { + j.calibMutex.Lock() + defer j.calibMutex.Unlock() + if !j.Config.AutoCalibration { + return nil + } + if j.Config.AutoCalibrationPerHost { + return j.CalibrateForHost(host, input) + } + return j.Calibrate(input) +} + +func (j *Job) calibrateFilters(responses []Response, perHost bool) error { + // Work down from the most specific common denominator + if len(responses) > 0 { + // Content length + baselineSize := responses[0].ContentLength + sizeMatch := true + for _, r := range responses { + if baselineSize != r.ContentLength { + sizeMatch = false + } + } + if sizeMatch { + if perHost { + // Check if already filtered + for _, f := range j.Config.MatcherManager.FiltersForDomain(responses[0].Request.Host) { + match, _ := f.Filter(&responses[0]) + if match { + // Already filtered + return nil + } + } + _ = j.Config.MatcherManager.AddPerDomainFilter(responses[0].Request.Host, "size", strconv.FormatInt(baselineSize, 10)) + return nil + } else { + // Check if already filtered + for _, f := range j.Config.MatcherManager.GetFilters() { + match, _ := f.Filter(&responses[0]) + if match { + // Already filtered + return nil + } + } + _ = j.Config.MatcherManager.AddFilter("size", strconv.FormatInt(baselineSize, 10), false) + return nil + } + } + + // Content words + baselineWords := responses[0].ContentWords + wordsMatch := true + for _, r := range responses { + if baselineWords != r.ContentWords { + wordsMatch = false + } + } + if wordsMatch { + if perHost { + // Check if already filtered + for _, f := range j.Config.MatcherManager.FiltersForDomain(responses[0].Request.Host) { + match, _ := f.Filter(&responses[0]) + if match { + // Already filtered + return nil + } + } + _ = j.Config.MatcherManager.AddPerDomainFilter(responses[0].Request.Host, "word", strconv.FormatInt(baselineWords, 10)) + return nil + } else { + // Check if already filtered + for _, f := range j.Config.MatcherManager.GetFilters() { + match, _ := f.Filter(&responses[0]) + if match { + // Already filtered + return nil + } + } + _ = j.Config.MatcherManager.AddFilter("word", strconv.FormatInt(baselineSize, 10), false) + return nil + } + } + + // Content lines + baselineLines := responses[0].ContentLines + linesMatch := true + for _, r := range responses { + if baselineLines != r.ContentLines { + linesMatch = false + } + } + if linesMatch { + if perHost { + // Check if already filtered + for _, f := range j.Config.MatcherManager.FiltersForDomain(responses[0].Request.Host) { + match, _ := f.Filter(&responses[0]) + if match { + // Already filtered + return nil + } + } + _ = j.Config.MatcherManager.AddPerDomainFilter(responses[0].Request.Host, "line", strconv.FormatInt(baselineLines, 10)) + return nil + } else { + // Check if already filtered + for _, f := range j.Config.MatcherManager.GetFilters() { + match, _ := f.Filter(&responses[0]) + if match { + // Already filtered + return nil + } + } + _ = j.Config.MatcherManager.AddFilter("line", strconv.FormatInt(baselineSize, 10), false) + return nil + } + } + } + return fmt.Errorf("No common filtering values found") +} diff --git a/pkg/ffuf/config.go b/pkg/ffuf/config.go index 48fc86c..7342db4 100644 --- a/pkg/ffuf/config.go +++ b/pkg/ffuf/config.go @@ -5,54 +5,56 @@ import ( ) type Config struct { - AutoCalibration bool `json:"autocalibration"` - AutoCalibrationStrings []string `json:"autocalibration_strings"` - Cancel context.CancelFunc `json:"-"` - Colors bool `json:"colors"` - CommandKeywords []string `json:"-"` - CommandLine string `json:"cmdline"` - ConfigFile string `json:"configfile"` - Context context.Context `json:"-"` - Data string `json:"postdata"` - Delay optRange `json:"delay"` - DirSearchCompat bool `json:"dirsearch_compatibility"` - Extensions []string `json:"extensions"` - Filters map[string]FilterProvider `json:"filters"` - FollowRedirects bool `json:"follow_redirects"` - Headers map[string]string `json:"headers"` - IgnoreBody bool `json:"ignorebody"` - IgnoreWordlistComments bool `json:"ignore_wordlist_comments"` - InputMode string `json:"inputmode"` - InputNum int `json:"cmd_inputnum"` - InputProviders []InputProviderConfig `json:"inputproviders"` - InputShell string `json:"inputshell"` - Json bool `json:"json"` - Matchers map[string]FilterProvider `json:"matchers"` - MaxTime int `json:"maxtime"` - MaxTimeJob int `json:"maxtime_job"` - Method string `json:"method"` - Noninteractive bool `json:"noninteractive"` - OutputDirectory string `json:"outputdirectory"` - OutputFile string `json:"outputfile"` - OutputFormat string `json:"outputformat"` - OutputSkipEmptyFile bool `json:"OutputSkipEmptyFile"` - ProgressFrequency int `json:"-"` - ProxyURL string `json:"proxyurl"` - Quiet bool `json:"quiet"` - Rate int64 `json:"rate"` - Recursion bool `json:"recursion"` - RecursionDepth int `json:"recursion_depth"` - RecursionStrategy string `json:"recursion_strategy"` - ReplayProxyURL string `json:"replayproxyurl"` - SNI string `json:"sni"` - StopOn403 bool `json:"stop_403"` - StopOnAll bool `json:"stop_all"` - StopOnErrors bool `json:"stop_errors"` - Threads int `json:"threads"` - Timeout int `json:"timeout"` - Url string `json:"url"` - Verbose bool `json:"verbose"` - Http2 bool `json:"http2"` + AutoCalibration bool `json:"autocalibration"` + AutoCalibrationKeyword string `json:"autocalibration_keyword"` + AutoCalibrationPerHost bool `json:"autocalibration_perhost"` + AutoCalibrationStrategy string `json:"autocalibration_strategy"` + AutoCalibrationStrings []string `json:"autocalibration_strings"` + Cancel context.CancelFunc `json:"-"` + Colors bool `json:"colors"` + CommandKeywords []string `json:"-"` + CommandLine string `json:"cmdline"` + ConfigFile string `json:"configfile"` + Context context.Context `json:"-"` + Data string `json:"postdata"` + Delay optRange `json:"delay"` + DirSearchCompat bool `json:"dirsearch_compatibility"` + Extensions []string `json:"extensions"` + FollowRedirects bool `json:"follow_redirects"` + Headers map[string]string `json:"headers"` + IgnoreBody bool `json:"ignorebody"` + IgnoreWordlistComments bool `json:"ignore_wordlist_comments"` + InputMode string `json:"inputmode"` + InputNum int `json:"cmd_inputnum"` + InputProviders []InputProviderConfig `json:"inputproviders"` + InputShell string `json:"inputshell"` + Json bool `json:"json"` + MatcherManager MatcherManager `json:"matchers"` + MaxTime int `json:"maxtime"` + MaxTimeJob int `json:"maxtime_job"` + Method string `json:"method"` + Noninteractive bool `json:"noninteractive"` + OutputDirectory string `json:"outputdirectory"` + OutputFile string `json:"outputfile"` + OutputFormat string `json:"outputformat"` + OutputSkipEmptyFile bool `json:"OutputSkipEmptyFile"` + ProgressFrequency int `json:"-"` + ProxyURL string `json:"proxyurl"` + Quiet bool `json:"quiet"` + Rate int64 `json:"rate"` + Recursion bool `json:"recursion"` + RecursionDepth int `json:"recursion_depth"` + RecursionStrategy string `json:"recursion_strategy"` + ReplayProxyURL string `json:"replayproxyurl"` + SNI string `json:"sni"` + StopOn403 bool `json:"stop_403"` + StopOnAll bool `json:"stop_all"` + StopOnErrors bool `json:"stop_errors"` + Threads int `json:"threads"` + Timeout int `json:"timeout"` + Url string `json:"url"` + Verbose bool `json:"verbose"` + Http2 bool `json:"http2"` } type InputProviderConfig struct { @@ -64,6 +66,8 @@ type InputProviderConfig struct { func NewConfig(ctx context.Context, cancel context.CancelFunc) Config { var conf Config + conf.AutoCalibrationKeyword = "FUZZ" + conf.AutoCalibrationStrategy = "basic" conf.AutoCalibrationStrings = make([]string, 0) conf.CommandKeywords = make([]string, 0) conf.Context = ctx @@ -72,7 +76,6 @@ func NewConfig(ctx context.Context, cancel context.CancelFunc) Config { conf.Delay = optRange{0, 0, false, false} conf.DirSearchCompat = false conf.Extensions = make([]string, 0) - conf.Filters = make(map[string]FilterProvider) conf.FollowRedirects = false conf.Headers = make(map[string]string) conf.IgnoreWordlistComments = false @@ -81,7 +84,6 @@ func NewConfig(ctx context.Context, cancel context.CancelFunc) Config { conf.InputShell = "" conf.InputProviders = make([]InputProviderConfig, 0) conf.Json = false - conf.Matchers = make(map[string]FilterProvider) conf.MaxTime = 0 conf.MaxTimeJob = 0 conf.Method = "GET" diff --git a/pkg/ffuf/interfaces.go b/pkg/ffuf/interfaces.go index c36021f..6879992 100644 --- a/pkg/ffuf/interfaces.go +++ b/pkg/ffuf/interfaces.go @@ -2,6 +2,21 @@ package ffuf import "time" +//MatcherManager provides functions for managing matchers and filters +type MatcherManager interface { + SetCalibrated(calibrated bool) + SetCalibratedForHost(host string, calibrated bool) + AddFilter(name string, option string, replace bool) error + AddPerDomainFilter(domain string, name string, option string) error + RemoveFilter(name string) + AddMatcher(name string, option string) error + GetFilters() map[string]FilterProvider + GetMatchers() map[string]FilterProvider + FiltersForDomain(domain string) map[string]FilterProvider + CalibratedForDomain(domain string) bool + Calibrated() bool +} + //FilterProvider is a generic interface for both Matchers and Filters type FilterProvider interface { Filter(response *Response) (bool, error) diff --git a/pkg/ffuf/job.go b/pkg/ffuf/job.go index 539566a..89d4f75 100644 --- a/pkg/ffuf/job.go +++ b/pkg/ffuf/job.go @@ -36,6 +36,7 @@ type Job struct { queuepos int skipQueue bool currentDepth int + calibMutex sync.Mutex pauseWg sync.WaitGroup } @@ -325,7 +326,15 @@ func (j *Job) updateProgress() { func (j *Job) isMatch(resp Response) bool { matched := false - for _, m := range j.Config.Matchers { + var matchers map[string]FilterProvider + var filters map[string]FilterProvider + if j.Config.AutoCalibrationPerHost { + filters = j.Config.MatcherManager.FiltersForDomain(resp.Request.Host) + } else { + filters = j.Config.MatcherManager.GetFilters() + } + matchers = j.Config.MatcherManager.GetMatchers() + for _, m := range matchers { match, err := m.Filter(&resp) if err != nil { continue @@ -338,7 +347,7 @@ func (j *Job) isMatch(resp Response) bool { if !matched { return false } - for _, f := range j.Config.Filters { + for _, f := range filters { fv, err := f.Filter(&resp) if err != nil { continue @@ -360,6 +369,7 @@ func (j *Job) runTask(input map[string][]byte, position int, retried bool) { log.Printf("%s", err) return } + resp, err := j.Runner.Execute(&req) if err != nil { if retried { @@ -386,6 +396,10 @@ func (j *Job) runTask(input map[string][]byte, position int, retried bool) { } } j.pauseWg.Wait() + + // Handle autocalibration, must be done after the actual request to ensure sane value in req.Host + _ = j.CalibrateIfNeeded(req.Host, input) + if j.isMatch(resp) { // Re-send request through replay-proxy if needed if j.ReplayRunner != nil { @@ -444,47 +458,6 @@ func (j *Job) handleDefaultRecursionJob(resp Response) { } } -//CalibrateResponses returns slice of Responses for randomly generated filter autocalibration requests -func (j *Job) CalibrateResponses() ([]Response, error) { - basereq := BaseRequest(j.Config) - cInputs := make([]string, 0) - rand.Seed(time.Now().UnixNano()) - if len(j.Config.AutoCalibrationStrings) < 1 { - cInputs = append(cInputs, "admin"+RandomString(16)+"/") - cInputs = append(cInputs, ".htaccess"+RandomString(16)) - cInputs = append(cInputs, RandomString(16)+"/") - cInputs = append(cInputs, RandomString(16)) - } else { - cInputs = append(cInputs, j.Config.AutoCalibrationStrings...) - } - - results := make([]Response, 0) - for _, input := range cInputs { - inputs := make(map[string][]byte, len(j.Config.InputProviders)) - for _, v := range j.Config.InputProviders { - inputs[v.Keyword] = []byte(input) - } - - req, err := j.Runner.Prepare(inputs, &basereq) - if err != nil { - j.Output.Error(fmt.Sprintf("Encountered an error while preparing request: %s\n", err)) - j.incError() - log.Printf("%s", err) - return results, err - } - resp, err := j.Runner.Execute(&req) - if err != nil { - return results, err - } - - // Only calibrate on responses that would be matched otherwise - if j.isMatch(resp) { - results = append(results, resp) - } - } - return results, nil -} - // CheckStop stops the job if stopping conditions are met func (j *Job) CheckStop() { if j.Counter > 50 { diff --git a/pkg/ffuf/optionsparser.go b/pkg/ffuf/optionsparser.go index 83f834e..cb3ebb3 100644 --- a/pkg/ffuf/optionsparser.go +++ b/pkg/ffuf/optionsparser.go @@ -44,23 +44,26 @@ type HTTPOptions struct { } type GeneralOptions struct { - AutoCalibration bool - AutoCalibrationStrings []string - Colors bool - ConfigFile string `toml:"-"` - Delay string - Json bool - MaxTime int - MaxTimeJob int - Noninteractive bool - Quiet bool - Rate int - ShowVersion bool `toml:"-"` - StopOn403 bool - StopOnAll bool - StopOnErrors bool - Threads int - Verbose bool + AutoCalibration bool + AutoCalibrationKeyword string + AutoCalibrationPerHost bool + AutoCalibrationStrategy string + AutoCalibrationStrings []string + Colors bool + ConfigFile string `toml:"-"` + Delay string + Json bool + MaxTime int + MaxTimeJob int + Noninteractive bool + Quiet bool + Rate int + ShowVersion bool `toml:"-"` + StopOn403 bool + StopOnAll bool + StopOnErrors bool + Threads int + Verbose bool } type InputOptions struct { @@ -112,6 +115,8 @@ func NewConfigOptions() *ConfigOptions { c.Filter.Time = "" c.Filter.Words = "" c.General.AutoCalibration = false + c.General.AutoCalibrationKeyword = "FUZZ" + c.General.AutoCalibrationStrategy = "basic" c.General.Colors = false c.General.Delay = "" c.General.Json = false @@ -445,6 +450,8 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con conf.RecursionDepth = parseOpts.HTTP.RecursionDepth conf.RecursionStrategy = parseOpts.HTTP.RecursionStrategy conf.AutoCalibration = parseOpts.General.AutoCalibration + conf.AutoCalibrationPerHost = parseOpts.General.AutoCalibrationPerHost + conf.AutoCalibrationStrategy = parseOpts.General.AutoCalibrationStrategy conf.Threads = parseOpts.General.Threads conf.Timeout = parseOpts.HTTP.Timeout conf.MaxTime = parseOpts.General.MaxTime @@ -454,6 +461,11 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con conf.Json = parseOpts.General.Json conf.Http2 = parseOpts.HTTP.Http2 + if conf.AutoCalibrationPerHost { + // AutoCalibrationPerHost implies AutoCalibration + conf.AutoCalibration = true + } + // Handle copy as curl situation where POST method is implied by --data flag. If method is set to anything but GET, NOOP if len(conf.Data) > 0 && conf.Method == "GET" && @@ -557,6 +569,7 @@ func parseRawRequest(parseOpts *ConfigOptions, conf *Config) error { conf.Data = string(b) // Remove newline (typically added by the editor) at the end of the file + //nolint:gosimple // we specifically want to remove just a single newline, not all of them if strings.HasSuffix(conf.Data, "\r\n") { conf.Data = conf.Data[:len(conf.Data)-2] } else if strings.HasSuffix(conf.Data, "\n") { diff --git a/pkg/filter/filter.go b/pkg/filter/filter.go index 17234fd..72e73d6 100644 --- a/pkg/filter/filter.go +++ b/pkg/filter/filter.go @@ -1,14 +1,56 @@ package filter import ( - "flag" "fmt" - "strconv" - "strings" - "github.com/ffuf/ffuf/pkg/ffuf" + "sync" ) +// MatcherManager handles both filters and matchers. +type MatcherManager struct { + IsCalibrated bool + Mutex sync.Mutex + Matchers map[string]ffuf.FilterProvider + Filters map[string]ffuf.FilterProvider + PerDomainFilters map[string]*PerDomainFilter +} + +type PerDomainFilter struct { + IsCalibrated bool + Filters map[string]ffuf.FilterProvider +} + +func NewPerDomainFilter(globfilters map[string]ffuf.FilterProvider) *PerDomainFilter { + return &PerDomainFilter{IsCalibrated: false, Filters: globfilters} +} + +func (p *PerDomainFilter) SetCalibrated(value bool) { + p.IsCalibrated = value +} + +func NewMatcherManager() ffuf.MatcherManager { + return &MatcherManager{ + IsCalibrated: false, + Matchers: make(map[string]ffuf.FilterProvider), + Filters: make(map[string]ffuf.FilterProvider), + PerDomainFilters: make(map[string]*PerDomainFilter), + } +} + +func (f *MatcherManager) SetCalibrated(value bool) { + f.IsCalibrated = value +} + +func (f *MatcherManager) SetCalibratedForHost(host string, value bool) { + if f.PerDomainFilters[host] != nil { + f.PerDomainFilters[host].IsCalibrated = value + } else { + newFilter := NewPerDomainFilter(f.Filters) + newFilter.IsCalibrated = true + f.PerDomainFilters[host] = newFilter + } +} + func NewFilterByName(name string, value string) (ffuf.FilterProvider, error) { if name == "status" { return NewStatusFilter(value) @@ -31,195 +73,102 @@ func NewFilterByName(name string, value string) (ffuf.FilterProvider, error) { return nil, fmt.Errorf("Could not create filter with name %s", name) } -//AddFilter adds a new filter to Config -func AddFilter(conf *ffuf.Config, name string, option string) error { +//AddFilter adds a new filter to MatcherManager +func (f *MatcherManager) AddFilter(name string, option string, replace bool) error { + f.Mutex.Lock() + defer f.Mutex.Unlock() newf, err := NewFilterByName(name, option) if err == nil { // valid filter create or append - if conf.Filters[name] == nil { - conf.Filters[name] = newf + if f.Filters[name] == nil || replace { + f.Filters[name] = newf } else { - newoption := conf.Filters[name].Repr() + "," + option + newoption := f.Filters[name].Repr() + "," + option newerf, err := NewFilterByName(name, newoption) if err == nil { - conf.Filters[name] = newerf + f.Filters[name] = newerf } } } return err } +//AddPerDomainFilter adds a new filter to PerDomainFilter configuration +func (f *MatcherManager) AddPerDomainFilter(domain string, name string, option string) error { + f.Mutex.Lock() + defer f.Mutex.Unlock() + var pdFilters *PerDomainFilter + if filter, ok := f.PerDomainFilters[domain]; ok { + pdFilters = filter + } else { + pdFilters = NewPerDomainFilter(f.Filters) + } + newf, err := NewFilterByName(name, option) + if err == nil { + // valid filter create or append + if pdFilters.Filters[name] == nil { + pdFilters.Filters[name] = newf + } else { + newoption := pdFilters.Filters[name].Repr() + "," + option + newerf, err := NewFilterByName(name, newoption) + if err == nil { + pdFilters.Filters[name] = newerf + } + } + } + f.PerDomainFilters[domain] = pdFilters + return err +} + //RemoveFilter removes a filter of a given type -func RemoveFilter(conf *ffuf.Config, name string) { - delete(conf.Filters, name) +func (f *MatcherManager) RemoveFilter(name string) { + f.Mutex.Lock() + defer f.Mutex.Unlock() + delete(f.Filters, name) } //AddMatcher adds a new matcher to Config -func AddMatcher(conf *ffuf.Config, name string, option string) error { +func (f *MatcherManager) AddMatcher(name string, option string) error { + f.Mutex.Lock() + defer f.Mutex.Unlock() newf, err := NewFilterByName(name, option) if err == nil { - conf.Matchers[name] = newf + // valid filter create or append + if f.Matchers[name] == nil { + f.Matchers[name] = newf + } else { + newoption := f.Matchers[name].Repr() + "," + option + newerf, err := NewFilterByName(name, newoption) + if err == nil { + f.Matchers[name] = newerf + } + } } return err } -//CalibrateIfNeeded runs a self-calibration task for filtering options (if needed) by requesting random resources and acting accordingly -func CalibrateIfNeeded(j *ffuf.Job) error { - var err error - if !j.Config.AutoCalibration { - return nil - } - // Handle the calibration - responses, err := j.CalibrateResponses() - if err != nil { - return err - } - if len(responses) > 0 { - err = calibrateFilters(j, responses) - } - return err +func (f *MatcherManager) GetFilters() map[string]ffuf.FilterProvider { + return f.Filters } -func calibrateFilters(j *ffuf.Job, responses []ffuf.Response) error { - sizeCalib := make([]string, 0) - wordCalib := make([]string, 0) - lineCalib := make([]string, 0) - for _, r := range responses { - if r.ContentLength > 0 { - // Only add if we have an actual size of responses - sizeCalib = append(sizeCalib, strconv.FormatInt(r.ContentLength, 10)) - } - if r.ContentWords > 0 { - // Only add if we have an actual word length of response - wordCalib = append(wordCalib, strconv.FormatInt(r.ContentWords, 10)) - } - if r.ContentLines > 1 { - // Only add if we have an actual word length of response - lineCalib = append(lineCalib, strconv.FormatInt(r.ContentLines, 10)) - } - } - - //Remove duplicates - sizeCalib = ffuf.UniqStringSlice(sizeCalib) - wordCalib = ffuf.UniqStringSlice(wordCalib) - lineCalib = ffuf.UniqStringSlice(lineCalib) - - if len(sizeCalib) > 0 { - err := AddFilter(j.Config, "size", strings.Join(sizeCalib, ",")) - if err != nil { - return err - } - } - if len(wordCalib) > 0 { - err := AddFilter(j.Config, "word", strings.Join(wordCalib, ",")) - if err != nil { - return err - } - } - if len(lineCalib) > 0 { - err := AddFilter(j.Config, "line", strings.Join(lineCalib, ",")) - if err != nil { - return err - } - } - return nil +func (f *MatcherManager) GetMatchers() map[string]ffuf.FilterProvider { + return f.Matchers } -func SetupFilters(parseOpts *ffuf.ConfigOptions, conf *ffuf.Config) error { - errs := ffuf.NewMultierror() - // If any other matcher is set, ignore -mc default value - matcherSet := false - statusSet := false - warningIgnoreBody := false - flag.Visit(func(f *flag.Flag) { - if f.Name == "mc" { - statusSet = true - } - if f.Name == "ms" { - matcherSet = true - warningIgnoreBody = true - } - if f.Name == "ml" { - matcherSet = true - warningIgnoreBody = true - } - if f.Name == "mr" { - matcherSet = true - } - if f.Name == "mt" { - matcherSet = true - } - if f.Name == "mw" { - matcherSet = true - warningIgnoreBody = true - } - }) - if statusSet || !matcherSet { - if err := AddMatcher(conf, "status", parseOpts.Matcher.Status); err != nil { - errs.Add(err) - } +func (f *MatcherManager) FiltersForDomain(domain string) map[string]ffuf.FilterProvider { + if f.PerDomainFilters[domain] == nil { + return f.Filters } - - if parseOpts.Filter.Status != "" { - if err := AddFilter(conf, "status", parseOpts.Filter.Status); err != nil { - errs.Add(err) - } - } - if parseOpts.Filter.Size != "" { - warningIgnoreBody = true - if err := AddFilter(conf, "size", parseOpts.Filter.Size); err != nil { - errs.Add(err) - } - } - if parseOpts.Filter.Regexp != "" { - if err := AddFilter(conf, "regexp", parseOpts.Filter.Regexp); err != nil { - errs.Add(err) - } - } - if parseOpts.Filter.Words != "" { - warningIgnoreBody = true - if err := AddFilter(conf, "word", parseOpts.Filter.Words); err != nil { - errs.Add(err) - } - } - if parseOpts.Filter.Lines != "" { - warningIgnoreBody = true - if err := AddFilter(conf, "line", parseOpts.Filter.Lines); err != nil { - errs.Add(err) - } - } - if parseOpts.Filter.Time != "" { - if err := AddFilter(conf, "time", parseOpts.Filter.Time); err != nil { - errs.Add(err) - } - } - if parseOpts.Matcher.Size != "" { - if err := AddMatcher(conf, "size", parseOpts.Matcher.Size); err != nil { - errs.Add(err) - } - } - if parseOpts.Matcher.Regexp != "" { - if err := AddMatcher(conf, "regexp", parseOpts.Matcher.Regexp); err != nil { - errs.Add(err) - } - } - if parseOpts.Matcher.Words != "" { - if err := AddMatcher(conf, "word", parseOpts.Matcher.Words); err != nil { - errs.Add(err) - } - } - if parseOpts.Matcher.Lines != "" { - if err := AddMatcher(conf, "line", parseOpts.Matcher.Lines); err != nil { - errs.Add(err) - } - } - if parseOpts.Matcher.Time != "" { - if err := AddFilter(conf, "time", parseOpts.Matcher.Time); err != nil { - errs.Add(err) - } - } - if conf.IgnoreBody && warningIgnoreBody { - fmt.Printf("*** Warning: possible undesired combination of -ignore-body and the response options: fl,fs,fw,ml,ms and mw.\n") - } - return errs.ErrorOrNil() + return f.PerDomainFilters[domain].Filters +} + +func (f *MatcherManager) CalibratedForDomain(domain string) bool { + if f.PerDomainFilters[domain] != nil { + return f.PerDomainFilters[domain].IsCalibrated + } + return false +} + +func (f *MatcherManager) Calibrated() bool { + return f.IsCalibrated } diff --git a/pkg/interactive/termhandler.go b/pkg/interactive/termhandler.go index 15a6c3a..5846bf4 100644 --- a/pkg/interactive/termhandler.go +++ b/pkg/interactive/termhandler.go @@ -8,7 +8,6 @@ import ( "time" "github.com/ffuf/ffuf/pkg/ffuf" - "github.com/ffuf/ffuf/pkg/filter" ) type interactive struct { @@ -81,7 +80,7 @@ func (i *interactive) handleInput(in []byte) { } else if len(args) > 2 { i.Job.Output.Error("Too many arguments for \"fc\"") } else { - i.updateFilter("status", args[1]) + i.updateFilter("status", args[1], true) i.Job.Output.Info("New status code filter value set") } case "afc": @@ -99,7 +98,7 @@ func (i *interactive) handleInput(in []byte) { } else if len(args) > 2 { i.Job.Output.Error("Too many arguments for \"fl\"") } else { - i.updateFilter("line", args[1]) + i.updateFilter("line", args[1], true) i.Job.Output.Info("New line count filter value set") } case "afl": @@ -117,7 +116,7 @@ func (i *interactive) handleInput(in []byte) { } else if len(args) > 2 { i.Job.Output.Error("Too many arguments for \"fw\"") } else { - i.updateFilter("word", args[1]) + i.updateFilter("word", args[1], true) i.Job.Output.Info("New word count filter value set") } case "afw": @@ -135,7 +134,7 @@ func (i *interactive) handleInput(in []byte) { } else if len(args) > 2 { i.Job.Output.Error("Too many arguments for \"fs\"") } else { - i.updateFilter("size", args[1]) + i.updateFilter("size", args[1], true) i.Job.Output.Info("New response size filter value set") } case "afs": @@ -153,7 +152,7 @@ func (i *interactive) handleInput(in []byte) { } else if len(args) > 2 { i.Job.Output.Error("Too many arguments for \"ft\"") } else { - i.updateFilter("time", args[1]) + i.updateFilter("time", args[1], true) i.Job.Output.Info("New response time filter value set") } case "aft": @@ -192,19 +191,10 @@ func (i *interactive) handleInput(in []byte) { } } -func (i *interactive) updateFilter(name, value string) { - if value == "none" { - filter.RemoveFilter(i.Job.Config, name) - } else { - newFc, err := filter.NewFilterByName(name, value) - if err != nil { - i.Job.Output.Error(fmt.Sprintf("Error while setting new filter value: %s", err)) - return - } else { - i.Job.Config.Filters[name] = newFc - } - - results := make([]ffuf.Result, 0) +func (i *interactive) refreshResults() { + results := make([]ffuf.Result, 0) + filters := i.Job.Config.MatcherManager.GetFilters() + for _, filter := range filters { for _, res := range i.Job.Output.GetCurrentResults() { fakeResp := &ffuf.Response{ StatusCode: res.StatusCode, @@ -212,22 +202,26 @@ func (i *interactive) updateFilter(name, value string) { ContentWords: res.ContentWords, ContentLength: res.ContentLength, } - filterOut, _ := newFc.Filter(fakeResp) + filterOut, _ := filter.Filter(fakeResp) if !filterOut { results = append(results, res) } } - i.Job.Output.SetCurrentResults(results) } + i.Job.Output.SetCurrentResults(results) +} + +func (i *interactive) updateFilter(name, value string, replace bool) { + if value == "none" { + i.Job.Config.MatcherManager.RemoveFilter(name) + } else { + _ = i.Job.Config.MatcherManager.AddFilter(name, value, replace) + } + i.refreshResults() } func (i *interactive) appendFilter(name, value string) { - if oldFc, found := i.Job.Config.Filters[name]; found { - oldVal := oldFc.Repr() - i.updateFilter(name, strings.Join([]string{oldVal, value}, ",")) - } else { - i.updateFilter(name, value) - } + i.updateFilter(name, value, false) } func (i *interactive) printQueue() { @@ -270,7 +264,7 @@ func (i *interactive) printPrompt() { func (i *interactive) printHelp() { var fc, fl, fs, ft, fw string - for name, filter := range i.Job.Config.Filters { + for name, filter := range i.Job.Config.MatcherManager.GetFilters() { switch name { case "status": fc = "(active: " + filter.Repr() + ")" diff --git a/pkg/output/stdout.go b/pkg/output/stdout.go index 588decb..758f3bd 100644 --- a/pkg/output/stdout.go +++ b/pkg/output/stdout.go @@ -124,11 +124,11 @@ func (s *Stdoutput) Banner() { } // Print matchers - for _, f := range s.config.Matchers { + for _, f := range s.config.MatcherManager.GetMatchers() { printOption([]byte("Matcher"), []byte(f.ReprVerbose())) } // Print filters - for _, f := range s.config.Filters { + for _, f := range s.config.MatcherManager.GetFilters() { printOption([]byte("Filter"), []byte(f.ReprVerbose())) } fmt.Fprintf(os.Stderr, "%s\n\n", BANNER_SEP)