Pencode (#717)

* Pencode integration

* Added CHANGELOG entry

* Make linter happy

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Changelog
 - master
   - New
+    - Integration with `github.com/ffuf/pencode` library, added `-enc` cli flag to do various in-fly encodings for input data
   - Changed
     - Explicitly allow TLS1.0 
     - Fix markdown output file format

go.mod

@@ -5,6 +5,7 @@ go 1.17
 require (
 	github.com/PuerkitoBio/goquery v1.8.0
 	github.com/adrg/xdg v0.4.0
+	github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693
 	github.com/pelletier/go-toml v1.9.5
 )
 

go.sum

@@ -7,6 +7,8 @@ github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEq
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693 h1:fdlgw33oLPzRpoHa4ppDFX5EcmzHHychPrO5xXmzxqc=
+github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693/go.mod h1:Qmgn2URTRtZ5wMntUke1+/G7z8rofTFHG1EvN3addNY=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

help.go

@@ -89,7 +89,7 @@ func Usage() {
 		Description:   "Options for input data for fuzzing. Wordlists and input generators.",
 		Flags:         make([]UsageFlag, 0),
 		Hidden:        false,
-		ExpectedFlags: []string{"D", "ic", "input-cmd", "input-num", "input-shell", "mode", "request", "request-proto", "e", "w"},
+		ExpectedFlags: []string{"D", "enc", "ic", "input-cmd", "input-num", "input-shell", "mode", "request", "request-proto", "e", "w"},
 	}
 	u_output := UsageSection{
 		Name:          "OUTPUT OPTIONS",

main.go

@@ -51,13 +51,14 @@ func (m *wordlistFlag) Set(value string) error {
 func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	var ignored bool
 	var cookies, autocalibrationstrings, headers, inputcommands multiStringFlag
-	var wordlists wordlistFlag
+	var wordlists, encoders wordlistFlag
 
 	cookies = opts.HTTP.Cookies
 	autocalibrationstrings = opts.General.AutoCalibrationStrings
 	headers = opts.HTTP.Headers
 	inputcommands = opts.Input.Inputcommands
 	wordlists = opts.Input.Wordlists
+	encoders = opts.Input.Encoders
 
 	flag.BoolVar(&ignored, "compressed", true, "Dummy flag for copy as curl functionality (ignored)")
 	flag.BoolVar(&ignored, "i", true, "Dummy flag for copy as curl functionality (ignored)")
@@ -133,6 +134,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	flag.Var(&headers, "H", "Header `\"Name: Value\"`, separated by colon. Multiple -H flags are accepted.")
 	flag.Var(&inputcommands, "input-cmd", "Command producing the input. --input-num is required when using this input method. Overrides -w.")
 	flag.Var(&wordlists, "w", "Wordlist file path and (optional) keyword separated by colon. eg. '/path/to/wordlist:KEYWORD'")
+	flag.Var(&encoders, "enc", "Encoders for keywords, eg. 'FUZZ:urlencode b64encode'")
 	flag.Usage = Usage
 	flag.Parse()
 
@@ -141,6 +143,7 @@ func ParseFlags(opts *ffuf.ConfigOptions) *ffuf.ConfigOptions {
 	opts.HTTP.Headers = headers
 	opts.Input.Inputcommands = inputcommands
 	opts.Input.Wordlists = wordlists
+	opts.Input.Encoders = encoders
 	return opts
 }
 

pkg/ffuf/config.go

@@ -20,6 +20,7 @@ type Config struct {
 	Debuglog                string                `json:"debuglog"`
 	Delay                   optRange              `json:"delay"`
 	DirSearchCompat         bool                  `json:"dirsearch_compatibility"`
+	Encoders                []string              `json:"encoders"`
 	Extensions              []string              `json:"extensions"`
 	FilterMode              string                `json:"fmode"`
 	FollowRedirects         bool                  `json:"follow_redirects"`
@@ -69,6 +70,7 @@ type InputProviderConfig struct {
 	Name     string `json:"name"`
 	Keyword  string `json:"keyword"`
 	Value    string `json:"value"`
+	Encoders string `json:"encoders"`
 	Template string `json:"template"` // the templating string used for sniper mode (usually "§")
 }
 
@@ -84,6 +86,7 @@ func NewConfig(ctx context.Context, cancel context.CancelFunc) Config {
 	conf.Debuglog = ""
 	conf.Delay = optRange{0, 0, false, false}
 	conf.DirSearchCompat = false
+	conf.Encoders = make([]string, 0)
 	conf.Extensions = make([]string, 0)
 	conf.FilterMode = "or"
 	conf.FollowRedirects = false

pkg/ffuf/optionsparser.go

@@ -71,6 +71,7 @@ type GeneralOptions struct {
 
 type InputOptions struct {
 	DirSearchCompat        bool     `json:"dirsearch_compat"`
+	Encoders               []string `json:"encoders"`
 	Extensions             string   `json:"extensions"`
 	IgnoreWordlistComments bool     `json:"ignore_wordlist_comments"`
 	InputMode              string   `json:"input_mode"`
@@ -154,6 +155,7 @@ func NewConfigOptions() *ConfigOptions {
 	c.HTTP.URL = ""
 	c.HTTP.Http2 = false
 	c.Input.DirSearchCompat = false
+	c.Input.Encoders = []string{}
 	c.Input.Extensions = ""
 	c.Input.IgnoreWordlistComments = false
 	c.Input.InputMode = "clusterbomb"
@@ -225,7 +227,14 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
 			errs.Add(fmt.Errorf("sniper mode only supports one input command"))
 		}
 	}
-
+	tmpEncoders := make(map[string]string)
+	for _, e := range parseOpts.Input.Encoders {
+		if strings.Contains(e, ":") {
+			key := strings.Split(e, ":")[0]
+			val := strings.Split(e, ":")[1]
+			tmpEncoders[key] = val
+		}
+	}
 	tmpWordlists := make([]string, 0)
 	for _, v := range parseOpts.Input.Wordlists {
 		var wl []string
@@ -265,19 +274,31 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
 			if conf.InputMode == "sniper" {
 				errs.Add(fmt.Errorf("sniper mode does not support wordlist keywords"))
 			} else {
-				conf.InputProviders = append(conf.InputProviders, InputProviderConfig{
+				newp := InputProviderConfig{
 					Name:    "wordlist",
 					Value:   wl[0],
 					Keyword: wl[1],
-				})
+				}
+				// Add encoders if set
+				enc, ok := tmpEncoders[wl[1]]
+				if ok {
+					newp.Encoders = enc
+				}
+				conf.InputProviders = append(conf.InputProviders, newp)
 			}
 		} else {
-			conf.InputProviders = append(conf.InputProviders, InputProviderConfig{
+			newp := InputProviderConfig{
 				Name:     "wordlist",
 				Value:    wl[0],
 				Keyword:  "FUZZ",
 				Template: template,
-			})
+			}
+			// Add encoders if set
+			enc, ok := tmpEncoders["FUZZ"]
+			if ok {
+				newp.Encoders = enc
+			}
+			conf.InputProviders = append(conf.InputProviders, newp)
 		}
 		tmpWordlists = append(tmpWordlists, strings.Join(wl, ":"))
 	}
@@ -289,20 +310,30 @@ func ConfigFromOptions(parseOpts *ConfigOptions, ctx context.Context, cancel con
 			if conf.InputMode == "sniper" {
 				errs.Add(fmt.Errorf("sniper mode does not support command keywords"))
 			} else {
-				conf.InputProviders = append(conf.InputProviders, InputProviderConfig{
+				newp := InputProviderConfig{
 					Name:    "command",
 					Value:   ic[0],
 					Keyword: ic[1],
-				})
+				}
+				enc, ok := tmpEncoders[ic[1]]
+				if ok {
+					newp.Encoders = enc
+				}
+				conf.InputProviders = append(conf.InputProviders, newp)
 				conf.CommandKeywords = append(conf.CommandKeywords, ic[0])
 			}
 		} else {
-			conf.InputProviders = append(conf.InputProviders, InputProviderConfig{
+			newp := InputProviderConfig{
 				Name:     "command",
 				Value:    ic[0],
 				Keyword:  "FUZZ",
 				Template: template,
-			})
+			}
+			enc, ok := tmpEncoders["FUZZ"]
+			if ok {
+				newp.Encoders = enc
+			}
+			conf.InputProviders = append(conf.InputProviders, newp)
 			conf.CommandKeywords = append(conf.CommandKeywords, "FUZZ")
 		}
 	}

pkg/input/input.go

@@ -2,12 +2,15 @@ package input
 
 import (
 	"fmt"
-
 	"github.com/ffuf/ffuf/v2/pkg/ffuf"
+	"strings"
+
+	"github.com/ffuf/pencode/pkg/pencode"
 )
 
 type MainInputProvider struct {
 	Providers   []ffuf.InternalInputProvider
+	Encoders    map[string]*pencode.Chain
 	Config      *ffuf.Config
 	position    int
 	msbIterator int
@@ -25,7 +28,7 @@ func NewInputProvider(conf *ffuf.Config) (ffuf.InputProvider, ffuf.Multierror) {
 		errs.Add(fmt.Errorf("Input mode (-mode) %s not recognized", conf.InputMode))
 		return &MainInputProvider{}, errs
 	}
-	mainip := MainInputProvider{Config: conf, msbIterator: 0}
+	mainip := MainInputProvider{Config: conf, msbIterator: 0, Encoders: make(map[string]*pencode.Chain)}
 	// Initialize the correct inputprovider
 	for _, v := range conf.InputProviders {
 		err := mainip.AddProvider(v)
@@ -48,6 +51,14 @@ func (i *MainInputProvider) AddProvider(provider ffuf.InputProviderConfig) error
 		}
 		i.Providers = append(i.Providers, newwl)
 	}
+	if len(provider.Encoders) > 0 {
+		chain := pencode.NewChain()
+		err := chain.Initialize(strings.Split(strings.TrimSpace(provider.Encoders), " "))
+		if err != nil {
+			return err
+		}
+		i.Encoders[provider.Keyword] = chain
+	}
 	return nil
 }
 
@@ -103,6 +114,18 @@ func (i *MainInputProvider) Value() map[string][]byte {
 	if i.Config.InputMode == "pitchfork" {
 		retval = i.pitchforkValue()
 	}
+	if len(i.Encoders) > 0 {
+		for key, val := range retval {
+			chain, ok := i.Encoders[key]
+			if ok {
+				tmpVal, err := chain.Encode([]byte(val))
+				if err != nil {
+					fmt.Printf("ERROR: %s\n", err)
+				}
+				retval[key] = tmpVal
+			}
+		}
+	}
 	return retval
 }