Make defining User-agent header case insensitive #171 (#173)

* Takes the User-Agent header from a list.

* typo

* base

* Make defining User-agent header case insensitive #171

* -whitespaces

* Make canonical http headers and set default User-Agent only once.

* clean-up

* formatting, canonical customer headers, docs updated

* cleanup

* fmt

* Checking userdefined headers for excluding in canonicalization.

* resolving one conflict

* moved logic back and less resolve conflicts

Co-authored-by: Joona Hoikkala <joohoi@users.noreply.github.com>

CHANGELOG.md

@@ -4,7 +4,8 @@
   - New
 
   - Changed
-    - Write POST request data properly to file when ran with `-od`
+    - Write POST request data properly to file when ran with `-od`.
+    - Fixed a bug by using header canonicaliztion related to HTTP headers being case insensitive.
     - Properly handle relative redirect urls with `-recursion`
     - Calculate req/sec correctly for when using recursion
     - When `-request` is used, allow the user to override URL using `-u`

CONTRIBUTORS.md

@@ -1,5 +1,6 @@
 # Contributors
 
+* [bjhulst](https://github.com/bjhulst)
 * [ccsplit](https://github.com/ccsplit)
 * [codingo](https://github.com/codingo)
 * [delic](https://github.com/delic)

main.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net/textproto"
 	"net/url"
 	"os"
 	"strconv"
@@ -341,15 +342,37 @@ func prepareConfig(parseOpts *cliOptions, conf *ffuf.Config) error {
 		conf.Url = parseOpts.URL
 	}
 
-	//Prepare headers
+	//Prepare headers and make canonical
 	for _, v := range parseOpts.headers {
 		hs := strings.SplitN(v, ":", 2)
 		if len(hs) == 2 {
+			// trim and make canonical
+			// except if used in custom defined header
+			var CanonicalNeeded bool = true
+			for _, a := range conf.CommandKeywords {
+				if a == hs[0] {
+					CanonicalNeeded = false
+				}
+			}
+			// check if part of InputProviders
+			if CanonicalNeeded {
+				for _, b := range conf.InputProviders {
+					if b.Keyword == hs[0] {
+						CanonicalNeeded = false
+					}
+				}
+			}
+			if CanonicalNeeded {
+				var CanonicalHeader string = textproto.CanonicalMIMEHeaderKey(strings.TrimSpace(hs[0]))
+				conf.Headers[CanonicalHeader] = strings.TrimSpace(hs[1])
+			} else {
 				conf.Headers[strings.TrimSpace(hs[0])] = strings.TrimSpace(hs[1])
+			}
 		} else {
 			errs.Add(fmt.Errorf("Header defined by -H needs to have a value. \":\" should be used as a separator"))
 		}
 	}
+
 	//Prepare delay
 	d := strings.Split(parseOpts.delay, "-")
 	if len(d) > 2 {

pkg/runner/simple.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
+	"net/textproto"
 	"net/url"
 	"strconv"
 	"strings"
@@ -73,7 +74,8 @@ func (r *SimpleRunner) Prepare(input map[string][]byte) (ffuf.Request, error) {
 		req.Method = strings.Replace(req.Method, keyword, string(inputitem), -1)
 		headers := make(map[string]string, 0)
 		for h, v := range req.Headers {
-			headers[strings.Replace(h, keyword, string(inputitem), -1)] = strings.Replace(v, keyword, string(inputitem), -1)
+			var CanonicalHeader string = textproto.CanonicalMIMEHeaderKey(strings.Replace(h, keyword, string(inputitem), -1))
+			headers[CanonicalHeader] = strings.Replace(v, keyword, string(inputitem), -1)
 		}
 		req.Headers = headers
 		req.Url = strings.Replace(req.Url, keyword, string(inputitem), -1)
@@ -93,10 +95,12 @@ func (r *SimpleRunner) Execute(req *ffuf.Request) (ffuf.Response, error) {
 	if err != nil {
 		return ffuf.Response{}, err
 	}
-	// Add user agent string if not defined
+
+	// set default User-Agent header if not present
 	if _, ok := req.Headers["User-Agent"]; !ok {
 		req.Headers["User-Agent"] = fmt.Sprintf("%s v%s", "Fuzz Faster U Fool", ffuf.VERSION)
 	}
+
 	// Handle Go http.Request special cases
 	if _, ok := req.Headers["Host"]; ok {
 		httpreq.Host = req.Headers["Host"]