Do autocalibration for full path (#22)

* Do autocalibration for full path

* Linter fixes

pkg/ffuf/autocalibration.go

@@ -55,14 +55,18 @@ func (j *Job) calibrationRequest(inputs map[string][]byte) (Response, error) {
 }
 
 //CalibrateForHost runs autocalibration for a specific host
-func (j *Job) CalibrateForHost(host string, input map[string][]byte) error {
+func (j *Job) CalibrateForHost(host string, baseinput map[string][]byte) error {
 	if j.Config.MatcherManager.CalibratedForDomain(host) {
 		return nil
 	}
-	if input[j.Config.AutoCalibrationKeyword] == nil {
+	if baseinput[j.Config.AutoCalibrationKeyword] == nil {
 		return fmt.Errorf("Autocalibration keyword \"%s\" not found in the request.", j.Config.AutoCalibrationKeyword)
 	}
 	cStrings := j.autoCalibrationStrings()
+	input := make(map[string][]byte)
+	for k, v := range baseinput {
+		input[k] = v
+	}
 	for _, v := range cStrings {
 		responses := make([]Response, 0)
 		for _, cs := range v {
@@ -136,14 +140,14 @@ func (j *Job) calibrateFilters(responses []Response, perHost bool) error {
 		if sizeMatch {
 			if perHost {
 				// Check if already filtered
-				for _, f := range j.Config.MatcherManager.FiltersForDomain(responses[0].Request.Host) {
+				for _, f := range j.Config.MatcherManager.FiltersForDomain(HostURLFromRequest(*responses[0].Request)) {
 					match, _ := f.Filter(&responses[0])
 					if match {
 						// Already filtered
 						return nil
 					}
 				}
-				_ = j.Config.MatcherManager.AddPerDomainFilter(responses[0].Request.Host, "size", strconv.FormatInt(baselineSize, 10))
+				_ = j.Config.MatcherManager.AddPerDomainFilter(HostURLFromRequest(*responses[0].Request), "size", strconv.FormatInt(baselineSize, 10))
 				return nil
 			} else {
 				// Check if already filtered
@@ -170,14 +174,14 @@ func (j *Job) calibrateFilters(responses []Response, perHost bool) error {
 		if wordsMatch {
 			if perHost {
 				// Check if already filtered
-				for _, f := range j.Config.MatcherManager.FiltersForDomain(responses[0].Request.Host) {
+				for _, f := range j.Config.MatcherManager.FiltersForDomain(HostURLFromRequest(*responses[0].Request)) {
 					match, _ := f.Filter(&responses[0])
 					if match {
 						// Already filtered
 						return nil
 					}
 				}
-				_ = j.Config.MatcherManager.AddPerDomainFilter(responses[0].Request.Host, "word", strconv.FormatInt(baselineWords, 10))
+				_ = j.Config.MatcherManager.AddPerDomainFilter(HostURLFromRequest(*responses[0].Request), "word", strconv.FormatInt(baselineWords, 10))
 				return nil
 			} else {
 				// Check if already filtered
@@ -204,14 +208,14 @@ func (j *Job) calibrateFilters(responses []Response, perHost bool) error {
 		if linesMatch {
 			if perHost {
 				// Check if already filtered
-				for _, f := range j.Config.MatcherManager.FiltersForDomain(responses[0].Request.Host) {
+				for _, f := range j.Config.MatcherManager.FiltersForDomain(HostURLFromRequest(*responses[0].Request)) {
 					match, _ := f.Filter(&responses[0])
 					if match {
 						// Already filtered
 						return nil
 					}
 				}
-				_ = j.Config.MatcherManager.AddPerDomainFilter(responses[0].Request.Host, "line", strconv.FormatInt(baselineLines, 10))
+				_ = j.Config.MatcherManager.AddPerDomainFilter(HostURLFromRequest(*responses[0].Request), "line", strconv.FormatInt(baselineLines, 10))
 				return nil
 			} else {
 				// Check if already filtered

pkg/ffuf/job.go

@@ -329,7 +329,7 @@ func (j *Job) isMatch(resp Response) bool {
 	var matchers map[string]FilterProvider
 	var filters map[string]FilterProvider
 	if j.Config.AutoCalibrationPerHost {
-		filters = j.Config.MatcherManager.FiltersForDomain(resp.Request.Host)
+		filters = j.Config.MatcherManager.FiltersForDomain(HostURLFromRequest(*resp.Request))
 	} else {
 		filters = j.Config.MatcherManager.GetFilters()
 	}
@@ -415,7 +415,7 @@ func (j *Job) runTask(input map[string][]byte, position int, retried bool) {
 	j.pauseWg.Wait()
 
 	// Handle autocalibration, must be done after the actual request to ensure sane value in req.Host
-	_ = j.CalibrateIfNeeded(req.Host, input)
+	_ = j.CalibrateIfNeeded(HostURLFromRequest(req), input)
 
 	if j.isMatch(resp) {
 		// Re-send request through replay-proxy if needed

pkg/ffuf/util.go

@@ -3,6 +3,7 @@ package ffuf
 import (
 	"fmt"
 	"math/rand"
+	"net/url"
 	"os"
 	"strings"
 )
@@ -66,6 +67,15 @@ func RequestContainsKeyword(req Request, kw string) bool {
 	return false
 }
 
+//HostURLFromRequest gets a host + path without the filename or last part of the URL path
+func HostURLFromRequest(req Request) string {
+	u, _ := url.Parse(req.Url)
+	u.Host = req.Host
+	pathparts := strings.Split(u.Path, "/")
+	trimpath := strings.TrimSpace(strings.Join(pathparts[:len(pathparts)-1], "/"))
+	return u.Host + trimpath
+}
+
 //Version returns the ffuf version string
 func Version() string {
 	return fmt.Sprintf("%s%s", VERSION, VERSION_APPENDIX)