From ac2b447dfd511b50500c0e27e72c97b7a45a3b30 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 7 Jan 2020 18:27:43 +0200 Subject: [PATCH] Write configuration to output JSON (#135) * Config to json output, filters and matchers * optRange marshaling * Add CHANGELOG entry --- CHANGELOG.md | 2 +- main.go | 9 ++-- pkg/ffuf/config.go | 91 +++++++++++++++++++---------------------- pkg/ffuf/optrange.go | 67 ++++++++++++++++++++++++++++++ pkg/filter/filter.go | 4 +- pkg/filter/lines.go | 17 ++++++++ pkg/filter/regex.go | 9 ++++ pkg/filter/size.go | 17 ++++++++ pkg/filter/status.go | 21 ++++++++++ pkg/filter/words.go | 17 ++++++++ pkg/output/file_json.go | 2 + pkg/runner/simple.go | 12 +++++- 12 files changed, 210 insertions(+), 58 deletions(-) create mode 100644 pkg/ffuf/optrange.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 32f8c2f..f4fce26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ - Regexp matching and filtering (-mr/-fr) allow using keywords in patterns - Take 429 responses into account when -sa (stop on all error cases) is used - Remove -k flag support, convert to dummy flag #134 - + - Write configuration to output JSON - v0.12 - New diff --git a/main.go b/main.go index d18528c..c22248a 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" "log" - "net/http" "net/url" "os" "strconv" @@ -326,11 +325,11 @@ func prepareConfig(parseOpts *cliOptions, conf *ffuf.Config) error { // Verify proxy url format if len(parseOpts.proxyURL) > 0 { - pu, err := url.Parse(parseOpts.proxyURL) + _, err := url.Parse(parseOpts.proxyURL) if err != nil { errs.Add(fmt.Errorf("Bad proxy url (-x) format: %s", err)) } else { - conf.ProxyURL = http.ProxyURL(pu) + conf.ProxyURL = parseOpts.proxyURL } } @@ -351,7 +350,9 @@ func prepareConfig(parseOpts *cliOptions, conf *ffuf.Config) error { } // Auto-calibration strings - conf.AutoCalibrationStrings = parseOpts.AutoCalibrationStrings + if len(parseOpts.AutoCalibrationStrings) > 0 { + conf.AutoCalibrationStrings = parseOpts.AutoCalibrationStrings + } // Using -acc implies -ac if len(conf.AutoCalibrationStrings) > 0 { conf.AutoCalibration = true diff --git a/pkg/ffuf/config.go b/pkg/ffuf/config.go index 3a31987..d816f89 100644 --- a/pkg/ffuf/config.go +++ b/pkg/ffuf/config.go @@ -2,60 +2,49 @@ package ffuf import ( "context" - "net/http" - "net/url" ) -//optRange stores either a single float, in which case the value is stored in min and IsRange is false, -//or a range of floats, in which case IsRange is true -type optRange struct { - Min float64 - Max float64 - IsRange bool - HasDelay bool -} - type Config struct { - Headers map[string]string - Extensions []string - DirSearchCompat bool - Method string - Url string - Data string - Quiet bool - Colors bool - InputProviders []InputProviderConfig - CommandKeywords []string - InputNum int - InputMode string - OutputDirectory string - OutputFile string - OutputFormat string - StopOn403 bool - StopOnErrors bool - StopOnAll bool - FollowRedirects bool - AutoCalibration bool - AutoCalibrationStrings []string - Timeout int - ProgressFrequency int - Delay optRange - Filters []FilterProvider - Matchers []FilterProvider - Threads int - Context context.Context - ProxyURL func(*http.Request) (*url.URL, error) - CommandLine string - Verbose bool - MaxTime int - Recursion bool - RecursionDepth int + Headers map[string]string `json:"headers"` + Extensions []string `json:"extensions"` + DirSearchCompat bool `json:"dirsearch_compatibility"` + Method string `json:"method"` + Url string `json:"url"` + Data string `json:"postdata"` + Quiet bool `json:"quiet"` + Colors bool `json:"colors"` + InputProviders []InputProviderConfig `json:"inputproviders"` + CommandKeywords []string `json:"-"` + InputNum int `json:"cmd_inputnum"` + InputMode string `json:"inputmode"` + OutputDirectory string `json:"outputdirectory"` + OutputFile string `json:"outputfile"` + OutputFormat string `json:"outputformat"` + StopOn403 bool `json:"stop_403"` + StopOnErrors bool `json:"stop_errors"` + StopOnAll bool `json:"stop_all"` + FollowRedirects bool `json:"follow_redirects"` + AutoCalibration bool `json:"autocalibration"` + AutoCalibrationStrings []string `json:"autocalibration_strings"` + Timeout int `json:"timeout"` + ProgressFrequency int `json:"-"` + Delay optRange `json:"delay"` + Filters map[string]FilterProvider `json:"filters"` + Matchers map[string]FilterProvider `json:"matchers"` + Threads int `json:"threads"` + Context context.Context `json:"-"` + ProxyURL string `json:"proxyurl"` + CommandLine string `json:"cmdline"` + Verbose bool `json:"verbose"` + MaxTime int `json:"maxtime"` + Recursion bool `json:"recursion"` + RecursionDepth int `json:"recursion_depth"` } type InputProviderConfig struct { - Name string - Keyword string - Value string + Name string `json:"name"` + Keyword string `json:"keyword"` + Value string `json:"value"` } func NewConfig(ctx context.Context) Config { @@ -72,10 +61,12 @@ func NewConfig(ctx context.Context) Config { conf.FollowRedirects = false conf.InputProviders = make([]InputProviderConfig, 0) conf.CommandKeywords = make([]string, 0) + conf.AutoCalibrationStrings = make([]string, 0) conf.InputNum = 0 conf.InputMode = "clusterbomb" - conf.ProxyURL = http.ProxyFromEnvironment - conf.Filters = make([]FilterProvider, 0) + conf.ProxyURL = "" + conf.Filters = make(map[string]FilterProvider) + conf.Matchers = make(map[string]FilterProvider) conf.Delay = optRange{0, 0, false, false} conf.Extensions = make([]string, 0) conf.Timeout = 10 diff --git a/pkg/ffuf/optrange.go b/pkg/ffuf/optrange.go new file mode 100644 index 0000000..3d9fd18 --- /dev/null +++ b/pkg/ffuf/optrange.go @@ -0,0 +1,67 @@ +package ffuf + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" +) + +//optRange stores either a single float, in which case the value is stored in min and IsRange is false, +//or a range of floats, in which case IsRange is true +type optRange struct { + Min float64 + Max float64 + IsRange bool + HasDelay bool +} + +type optRangeJSON struct { + Value string `json:"value"` +} + +func (o *optRange) MarshalJSON() ([]byte, error) { + value := "" + if o.Min == o.Max { + value = fmt.Sprintf("%.2f", o.Min) + } else { + value = fmt.Sprintf("%.2f-%.2f", o.Min, o.Max) + } + return json.Marshal(&optRangeJSON{ + Value: value, + }) +} + +func (o *optRange) UnmarshalJSON(b []byte) error { + var inc optRangeJSON + err := json.Unmarshal(b, &inc) + if err != nil { + return err + } + return o.Initialize(inc.Value) +} + +//Initialize sets up the optRange from string value +func (o *optRange) Initialize(value string) error { + var err, err2 error + d := strings.Split(value, "-") + if len(d) > 2 { + return fmt.Errorf("Delay needs to be either a single float: \"0.1\" or a range of floats, delimited by dash: \"0.1-0.8\"") + } else if len(d) == 2 { + o.IsRange = true + o.HasDelay = true + o.Min, err = strconv.ParseFloat(d[0], 64) + o.Max, err2 = strconv.ParseFloat(d[1], 64) + if err != nil || err2 != nil { + return fmt.Errorf("Delay range min and max values need to be valid floats. For example: 0.1-0.5") + } + } else if len(value) > 0 { + o.IsRange = false + o.HasDelay = true + o.Min, err = strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("Delay needs to be either a single float: \"0.1\" or a range of floats, delimited by dash: \"0.1-0.8\"") + } + } + return nil +} diff --git a/pkg/filter/filter.go b/pkg/filter/filter.go index 6d10dcb..846612b 100644 --- a/pkg/filter/filter.go +++ b/pkg/filter/filter.go @@ -31,7 +31,7 @@ func NewFilterByName(name string, value string) (ffuf.FilterProvider, error) { func AddFilter(conf *ffuf.Config, name string, option string) error { newf, err := NewFilterByName(name, option) if err == nil { - conf.Filters = append(conf.Filters, newf) + conf.Filters[name] = newf } return err } @@ -40,7 +40,7 @@ func AddFilter(conf *ffuf.Config, name string, option string) error { func AddMatcher(conf *ffuf.Config, name string, option string) error { newf, err := NewFilterByName(name, option) if err == nil { - conf.Matchers = append(conf.Matchers, newf) + conf.Matchers[name] = newf } return err } diff --git a/pkg/filter/lines.go b/pkg/filter/lines.go index 0ed684f..ab07fe5 100644 --- a/pkg/filter/lines.go +++ b/pkg/filter/lines.go @@ -1,6 +1,7 @@ package filter import ( + "encoding/json" "fmt" "strconv" "strings" @@ -24,6 +25,22 @@ func NewLineFilter(value string) (ffuf.FilterProvider, error) { return &LineFilter{Value: intranges}, nil } +func (f *LineFilter) MarshalJSON() ([]byte, error) { + value := make([]string, 0) + for _, v := range f.Value { + if v.Min == v.Max { + value = append(value, strconv.FormatInt(v.Min, 10)) + } else { + value = append(value, fmt.Sprintf("%d-%d", v.Min, v.Max)) + } + } + return json.Marshal(&struct { + Value string `json:"value"` + }{ + Value: strings.Join(value, ","), + }) +} + func (f *LineFilter) Filter(response *ffuf.Response) (bool, error) { linesSize := len(strings.Split(string(response.Data), "\n")) for _, iv := range f.Value { diff --git a/pkg/filter/regex.go b/pkg/filter/regex.go index f4c56ba..6c47110 100644 --- a/pkg/filter/regex.go +++ b/pkg/filter/regex.go @@ -1,6 +1,7 @@ package filter import ( + "encoding/json" "fmt" "regexp" "strings" @@ -21,6 +22,14 @@ func NewRegexpFilter(value string) (ffuf.FilterProvider, error) { return &RegexpFilter{Value: re, valueRaw: value}, nil } +func (f *RegexpFilter) MarshalJSON() ([]byte, error) { + return json.Marshal(&struct { + Value string `json:"value"` + }{ + Value: f.valueRaw, + }) +} + func (f *RegexpFilter) Filter(response *ffuf.Response) (bool, error) { matchheaders := "" for k, v := range response.Headers { diff --git a/pkg/filter/size.go b/pkg/filter/size.go index e320778..c6490a4 100644 --- a/pkg/filter/size.go +++ b/pkg/filter/size.go @@ -1,6 +1,7 @@ package filter import ( + "encoding/json" "fmt" "strconv" "strings" @@ -25,6 +26,22 @@ func NewSizeFilter(value string) (ffuf.FilterProvider, error) { return &SizeFilter{Value: intranges}, nil } +func (f *SizeFilter) MarshalJSON() ([]byte, error) { + value := make([]string, 0) + for _, v := range f.Value { + if v.Min == v.Max { + value = append(value, strconv.FormatInt(v.Min, 10)) + } else { + value = append(value, fmt.Sprintf("%d-%d", v.Min, v.Max)) + } + } + return json.Marshal(&struct { + Value string `json:"value"` + }{ + Value: strings.Join(value, ","), + }) +} + func (f *SizeFilter) Filter(response *ffuf.Response) (bool, error) { for _, iv := range f.Value { if iv.Min <= response.ContentLength && response.ContentLength <= iv.Max { diff --git a/pkg/filter/status.go b/pkg/filter/status.go index a7de52e..2e74272 100644 --- a/pkg/filter/status.go +++ b/pkg/filter/status.go @@ -1,6 +1,7 @@ package filter import ( + "encoding/json" "fmt" "strconv" "strings" @@ -30,6 +31,26 @@ func NewStatusFilter(value string) (ffuf.FilterProvider, error) { return &StatusFilter{Value: intranges}, nil } +func (f *StatusFilter) MarshalJSON() ([]byte, error) { + value := make([]string, 0) + for _, v := range f.Value { + if v.Min == 0 && v.Max == 0 { + value = append(value, "all") + } else { + if v.Min == v.Max { + value = append(value, strconv.FormatInt(v.Min, 10)) + } else { + value = append(value, fmt.Sprintf("%d-%d", v.Min, v.Max)) + } + } + } + return json.Marshal(&struct { + Value string `json:"value"` + }{ + Value: strings.Join(value, ","), + }) +} + func (f *StatusFilter) Filter(response *ffuf.Response) (bool, error) { for _, iv := range f.Value { if iv.Min == AllStatuses && iv.Max == AllStatuses { diff --git a/pkg/filter/words.go b/pkg/filter/words.go index 7911b50..ce22a3e 100644 --- a/pkg/filter/words.go +++ b/pkg/filter/words.go @@ -1,6 +1,7 @@ package filter import ( + "encoding/json" "fmt" "strconv" "strings" @@ -24,6 +25,22 @@ func NewWordFilter(value string) (ffuf.FilterProvider, error) { return &WordFilter{Value: intranges}, nil } +func (f *WordFilter) MarshalJSON() ([]byte, error) { + value := make([]string, 0) + for _, v := range f.Value { + if v.Min == v.Max { + value = append(value, strconv.FormatInt(v.Min, 10)) + } else { + value = append(value, fmt.Sprintf("%d-%d", v.Min, v.Max)) + } + } + return json.Marshal(&struct { + Value string `json:"value"` + }{ + Value: strings.Join(value, ","), + }) +} + func (f *WordFilter) Filter(response *ffuf.Response) (bool, error) { wordsSize := len(strings.Split(string(response.Data), " ")) for _, iv := range f.Value { diff --git a/pkg/output/file_json.go b/pkg/output/file_json.go index 40f757d..508aa0c 100644 --- a/pkg/output/file_json.go +++ b/pkg/output/file_json.go @@ -30,6 +30,7 @@ type jsonFileOutput struct { CommandLine string `json:"commandline"` Time string `json:"time"` Results []JsonResult `json:"results"` + Config *ffuf.Config `json:"config"` } func writeEJSON(config *ffuf.Config, res []Result) error { @@ -75,6 +76,7 @@ func writeJSON(config *ffuf.Config, res []Result) error { CommandLine: config.CommandLine, Time: t.Format(time.RFC3339), Results: jsonRes, + Config: config, } outBytes, err := json.Marshal(outJSON) if err != nil { diff --git a/pkg/runner/simple.go b/pkg/runner/simple.go index 80167c6..0649808 100644 --- a/pkg/runner/simple.go +++ b/pkg/runner/simple.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "strconv" "strings" "time" @@ -24,12 +25,21 @@ type SimpleRunner struct { func NewSimpleRunner(conf *ffuf.Config) ffuf.RunnerProvider { var simplerunner SimpleRunner + proxyURL := http.ProxyFromEnvironment + + if len(conf.ProxyURL) > 0 { + pu, err := url.Parse(conf.ProxyURL) + if err == nil { + proxyURL = http.ProxyURL(pu) + } + } + simplerunner.config = conf simplerunner.client = &http.Client{ CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, Timeout: time.Duration(time.Duration(conf.Timeout) * time.Second), Transport: &http.Transport{ - Proxy: conf.ProxyURL, + Proxy: proxyURL, MaxIdleConns: 1000, MaxIdleConnsPerHost: 500, MaxConnsPerHost: 500,