Added lines count to filter/matcher and stdout + csv + json (#71)

* Added HTML and Markdown output support

* Add HTML color code in HTML template

* Added lines count

* Added content lines to json + csv

* Added changelog entry

* Fixed copy paste mistake

* Changed the html report to be grepable :)

* Grepable output fixed

* Fixed lines count

README.md

@@ -130,6 +130,8 @@ To define the test case for ffuf, use the keyword `FUZZ` anywhere in the URL (`-
     	Filter HTTP response size. Comma separated list of sizes and ranges
   -fw string
     	Filter by amount of words in response. Comma separated list of word counts and ranges
+  -fl string
+    	Filter by amount of lines in response. Comma separated list of line counts and ranges
   -input-cmd string
     	Command producing the input. --input-num is required when using this input method. Overrides -w.
   -input-num int
@@ -143,6 +145,8 @@ To define the test case for ffuf, use the keyword `FUZZ` anywhere in the URL (`-
     	Match HTTP response size
   -mw string
     	Match amount of words in response
+  -ml string
+    	Match amount of lines in response
   -o string
     	Write output to file
   -of string
@@ -186,11 +190,14 @@ The only dependency of ffuf is Go 1.11. No dependencies outside of Go standard l
 - master
 
   - New
+
     - New CLI flag: -l, shows target location of redirect responses
     - New CLI flac: -acc, custom auto-calibration strings
     - New CLI flag: -debug-log, writes the debug logging to the specified file.
+    - New CLI flags -ml and -fl, filters/matches line count in response
 
   - Changed
+
     - New CLI flag: -i, dummy flag that does nothing. for compatibility with copy as curl.
     - New CLI flag: -b/--cookie, cookie data for compatibility with copy as curl.
     - New Output format are available: HTML and Markdown table.

main.go

@@ -26,10 +26,12 @@ type cliOptions struct {
 	filterSize             string
 	filterRegexp           string
 	filterWords            string
+	filterLines            string
 	matcherStatus          string
 	matcherSize            string
 	matcherRegexp          string
 	matcherWords           string
+	matcherLines           string
 	proxyURL               string
 	outputFormat           string
 	headers                multiStringFlag
@@ -67,6 +69,7 @@ func main() {
 	flag.StringVar(&opts.filterSize, "fs", "", "Filter HTTP response size. Comma separated list of sizes and ranges")
 	flag.StringVar(&opts.filterRegexp, "fr", "", "Filter regexp")
 	flag.StringVar(&opts.filterWords, "fw", "", "Filter by amount of words in response. Comma separated list of word counts and ranges")
+	flag.StringVar(&opts.filterLines, "fl", "", "Filter by amount of lines in response. Comma separated list of line counts and ranges")
 	flag.StringVar(&conf.Data, "d", "", "POST data")
 	flag.StringVar(&conf.Data, "data", "", "POST data (alias of -d)")
 	flag.StringVar(&conf.Data, "data-ascii", "", "POST data (alias of -d)")
@@ -82,6 +85,7 @@ func main() {
 	flag.StringVar(&opts.matcherSize, "ms", "", "Match HTTP response size")
 	flag.StringVar(&opts.matcherRegexp, "mr", "", "Match regexp")
 	flag.StringVar(&opts.matcherWords, "mw", "", "Match amount of words in response")
+	flag.StringVar(&opts.matcherLines, "ml", "", "Match amount of lines in response")
 	flag.StringVar(&opts.proxyURL, "x", "", "HTTP Proxy URL")
 	flag.StringVar(&conf.Method, "X", "GET", "HTTP method to use")
 	flag.StringVar(&conf.OutputFile, "o", "", "Write output to file")
@@ -189,6 +193,11 @@ func prepareFilters(parseOpts *cliOptions, conf *ffuf.Config) error {
 			errs.Add(err)
 		}
 	}
+	if parseOpts.filterLines != "" {
+		if err := filter.AddFilter(conf, "line", parseOpts.filterLines); err != nil {
+			errs.Add(err)
+		}
+	}
 	if parseOpts.matcherStatus != "" {
 		if err := filter.AddMatcher(conf, "status", parseOpts.matcherStatus); err != nil {
 			errs.Add(err)
@@ -209,6 +218,11 @@ func prepareFilters(parseOpts *cliOptions, conf *ffuf.Config) error {
 			errs.Add(err)
 		}
 	}
+	if parseOpts.matcherLines != "" {
+		if err := filter.AddMatcher(conf, "line", parseOpts.matcherLines); err != nil {
+			errs.Add(err)
+		}
+	}
 	return errs.ErrorOrNil()
 }
 

pkg/ffuf/response.go

@@ -11,6 +11,7 @@ type Response struct {
 	Data          []byte
 	ContentLength int64
 	ContentWords  int64
+	ContentLines  int64
 	Cancelled     bool
 	Request       *Request
 }

pkg/filter/filter.go

@@ -18,6 +18,9 @@ func NewFilterByName(name string, value string) (ffuf.FilterProvider, error) {
 	if name == "word" {
 		return NewWordFilter(value)
 	}
+	if name == "line" {
+		return NewLineFilter(value)
+	}
 	if name == "regexp" {
 		return NewRegexpFilter(value)
 	}
@@ -61,6 +64,7 @@ func CalibrateIfNeeded(j *ffuf.Job) error {
 func calibrateFilters(j *ffuf.Job, responses []ffuf.Response) {
 	sizeCalib := make([]string, 0)
 	wordCalib := make([]string, 0)
+	lineCalib := make([]string, 0)
 	for _, r := range responses {
 		if r.ContentLength > 0 {
 			// Only add if we have an actual size of responses
@@ -70,11 +74,16 @@ func calibrateFilters(j *ffuf.Job, responses []ffuf.Response) {
 			// Only add if we have an actual word length of response
 			wordCalib = append(wordCalib, strconv.FormatInt(r.ContentWords, 10))
 		}
+		if r.ContentLines > 1 {
+			// Only add if we have an actual word length of response
+			lineCalib = append(lineCalib, strconv.FormatInt(r.ContentLines, 10))
+		}
 	}
 
 	//Remove duplicates
 	sizeCalib = ffuf.UniqStringSlice(sizeCalib)
 	wordCalib = ffuf.UniqStringSlice(wordCalib)
+	lineCalib = ffuf.UniqStringSlice(lineCalib)
 
 	if len(sizeCalib) > 0 {
 		AddFilter(j.Config, "size", strings.Join(sizeCalib, ","))
@@ -82,4 +91,7 @@ func calibrateFilters(j *ffuf.Job, responses []ffuf.Response) {
 	if len(wordCalib) > 0 {
 		AddFilter(j.Config, "word", strings.Join(wordCalib, ","))
 	}
+	if len(lineCalib) > 0 {
+		AddFilter(j.Config, "line", strings.Join(lineCalib, ","))
+	}
 }

pkg/filter/filter_test.go

@@ -20,6 +20,11 @@ func TestNewFilterByName(t *testing.T) {
 		t.Errorf("Was expecting wordfilter")
 	}
 
+	lf, _ := NewFilterByName("line", "200")
+	if _, ok := lf.(*LineFilter); !ok {
+		t.Errorf("Was expecting linefilter")
+	}
+
 	ref, _ := NewFilterByName("regexp", "200")
 	if _, ok := ref.(*RegexpFilter); !ok {
 		t.Errorf("Was expecting regexpfilter")

pkg/filter/lines.go

@@ -0,0 +1,47 @@
+package filter
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/ffuf/ffuf/pkg/ffuf"
+)
+
+type LineFilter struct {
+	Value []ffuf.ValueRange
+}
+
+func NewLineFilter(value string) (ffuf.FilterProvider, error) {
+	var intranges []ffuf.ValueRange
+	for _, sv := range strings.Split(value, ",") {
+		vr, err := ffuf.ValueRangeFromString(sv)
+		if err != nil {
+			return &LineFilter{}, fmt.Errorf("Line filter or matcher (-fl / -ml): invalid value: %s", sv)
+		}
+		intranges = append(intranges, vr)
+	}
+	return &LineFilter{Value: intranges}, nil
+}
+
+func (f *LineFilter) Filter(response *ffuf.Response) (bool, error) {
+	linesSize := len(strings.Split(string(response.Data), "\n"))
+	for _, iv := range f.Value {
+		if iv.Min <= int64(linesSize) && int64(linesSize) <= iv.Max {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (f *LineFilter) Repr() string {
+	var strval []string
+	for _, iv := range f.Value {
+		if iv.Min == iv.Max {
+			strval = append(strval, strconv.Itoa(int(iv.Min)))
+		} else {
+			strval = append(strval, strconv.Itoa(int(iv.Min))+"-"+strconv.Itoa(int(iv.Max)))
+		}
+	}
+	return fmt.Sprintf("Response lines: %s", strings.Join(strval, ","))
+}

pkg/filter/lines_test.go

@@ -0,0 +1,52 @@
+package filter
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/ffuf/ffuf/pkg/ffuf"
+)
+
+func TestNewLineFilter(t *testing.T) {
+	f, _ := NewLineFilter("200,301,400-410,500")
+	linesRepr := f.Repr()
+	if strings.Index(linesRepr, "200,301,400-410,500") == -1 {
+		t.Errorf("Word filter was expected to have 4 values")
+	}
+}
+
+func TestNewLineFilterError(t *testing.T) {
+	_, err := NewLineFilter("invalid")
+	if err == nil {
+		t.Errorf("Was expecting an error from errenous input data")
+	}
+}
+
+func TestLineFiltering(t *testing.T) {
+	f, _ := NewLineFilter("200,301,402-450,500")
+	for i, test := range []struct {
+		input  int64
+		output bool
+	}{
+		{200, true},
+		{301, true},
+		{500, true},
+		{4, false},
+		{444, true},
+		{302, false},
+		{401, false},
+		{402, true},
+		{450, true},
+		{451, false},
+	} {
+		var data []string
+		for i := int64(0); i < test.input; i++ {
+			data = append(data, "A")
+		}
+		resp := ffuf.Response{Data: []byte(strings.Join(data, " "))}
+		filterReturn, _ := f.Filter(&resp)
+		if filterReturn != test.output {
+			t.Errorf("Filter test %d: Was expecing filter return value of %t but got %t", i, test.output, filterReturn)
+		}
+	}
+}

pkg/output/file_csv.go

@@ -9,7 +9,7 @@ import (
 	"github.com/ffuf/ffuf/pkg/ffuf"
 )
 
-var header = []string{"input", "position", "status_code", "content_length", "content_words"}
+var header = []string{"input", "position", "status_code", "content_length", "content_words", "content_lines"}
 
 func writeCSV(config *ffuf.Config, res []Result, encode bool) error {
 	f, err := os.Create(config.OutputFile)
@@ -48,5 +48,6 @@ func toCSV(r Result) []string {
 		strconv.FormatInt(r.StatusCode, 10),
 		strconv.FormatInt(r.ContentLength, 10),
 		strconv.FormatInt(r.ContentWords, 10),
+		strconv.FormatInt(r.ContentLines, 10),
 	}
 }

pkg/output/file_html.go

@@ -58,7 +58,7 @@ const (
    <table>
         <thead>
         <div style="display:none">
-|result_raw|StatusCode|Input|Position|ContentLength|ContentWords| 
+|result_raw|StatusCode|Input|Position|ContentLength|ContentWords|ContentLines|
         </div>
           <tr>
               <th>Status</th>
@@ -66,15 +66,16 @@ const (
               <th>Position</th>
               <th>Length</th>
               <th>Words</th>
+              <th>Lines</th>
           </tr>
         </thead>
 
         <tbody>
             {{range .Results}}
                 <div style="display:none">
-|result_raw|{{ .StatusCode }}|{{ .Input }}|{{ .Position }}|{{ .ContentLength }}|{{ .ContentWords }}| 
+|result_raw|{{ .StatusCode }}|{{ .Input }}|{{ .Position }}|{{ .ContentLength }}|{{ .ContentWords }}|{{ .ContentLines }}|
                 </div>
-                <tr class="result-{{ .StatusCode }}" style="background-color: {{.HTMLColor}};"><td><font color="black" class="status-code">{{ .StatusCode }}</font></td><td>{{ .Input }}</td><td>{{ .Position }}</td><td>{{ .ContentLength }}</td><td>{{ .ContentWords }}</td></tr>
+                <tr class="result-{{ .StatusCode }}" style="background-color: {{.HTMLColor}};"><td><font color="black" class="status-code">{{ .StatusCode }}</font></td><td>{{ .Input }}</td><td>{{ .Position }}</td><td>{{ .ContentLength }}</td><td>{{ .ContentWords }}</td><td>{{ .ContentLines }}</td></tr>
             {{end}}
         </tbody>
       </table>

pkg/output/file_md.go

@@ -20,9 +20,9 @@ const (
   Command line : ` + "`{{.CommandLine}}`" + `
   Time: ` + "{{ .Time }}" + `
 
-  | Input | Position | Status Code | Content Length | Content Words |
+  | Input | Position | Status Code | Content Length | Content Words | Content Lines |
-  | :---- | :------- | :---------- | :------------- | :------------ |
+  | :---- | :------- | :---------- | :------------- | :------------ | :------------ |
-  {{range .Results}}| {{ .Input }} | {{ .Position }} | {{ .StatusCode }} | {{ .ContentLength }} | {{ .ContentWords }} |
+  {{range .Results}}| {{ .Input }} | {{ .Position }} | {{ .StatusCode }} | {{ .ContentLength }} | {{ .ContentWords }} | {{ .ContentLines }} |
   {{end}}
 	` // The template format is not pretty but follows the markdown guide
 )

pkg/output/stdout.go

@@ -32,6 +32,7 @@ type Result struct {
 	StatusCode    int64  `json:"status"`
 	ContentLength int64  `json:"length"`
 	ContentWords  int64  `json:"words"`
+	ContentLines  int64  `json:"lines"`
 	HTMLColor     string `json:"html_color"`
 }
 
@@ -138,6 +139,7 @@ func (s *Stdoutput) Result(resp ffuf.Response) {
 			StatusCode:    resp.StatusCode,
 			ContentLength: resp.ContentLength,
 			ContentWords:  resp.ContentWords,
+			ContentLines:  resp.ContentLines,
 		}
 		s.Results = append(s.Results, sResult)
 	}
@@ -164,9 +166,9 @@ func (s *Stdoutput) resultNormal(resp ffuf.Response) {
 	var responseString string
 	if len(s.config.InputCommand) > 0 {
 		// If we're using external command for input, display the position instead of input
-		responseString = fmt.Sprintf("%s%-23s [Status: %s, Size: %d, Words: %d%s]", TERMINAL_CLEAR_LINE, strconv.Itoa(resp.Request.Position), s.colorizeStatus(resp.StatusCode), resp.ContentLength, resp.ContentWords, s.addRedirectLocation(resp))
+		responseString = fmt.Sprintf("%s%-23s [Status: %s, Size: %d, Words: %d, Lines: %d]", TERMINAL_CLEAR_LINE, strconv.Itoa(resp.Request.Position), s.colorizeStatus(resp.StatusCode), resp.ContentLength, resp.ContentWords, resp.ContentLines)
 	} else {
-		responseString = fmt.Sprintf("%s%-23s [Status: %s, Size: %d, Words: %d%s]", TERMINAL_CLEAR_LINE, resp.Request.Input, s.colorizeStatus(resp.StatusCode), resp.ContentLength, resp.ContentWords, s.addRedirectLocation(resp))
+		responseString = fmt.Sprintf("%s%-23s [Status: %s, Size: %d, Words: %d, Lines: %d]", TERMINAL_CLEAR_LINE, resp.Request.Input, s.colorizeStatus(resp.StatusCode), resp.ContentLength, resp.ContentWords, resp.ContentLines)
 	}
 	fmt.Println(responseString)
 }

pkg/runner/simple.go

@@ -106,7 +106,9 @@ func (r *SimpleRunner) Execute(req *ffuf.Request) (ffuf.Response, error) {
 	}
 
 	wordsSize := len(strings.Split(string(resp.Data), " "))
+	linesSize := len(strings.Split(string(resp.Data), "\n"))
 	resp.ContentWords = int64(wordsSize)
+	resp.ContentLines = int64(linesSize)
 
 	return resp, nil
 }