send -H cookies once on same-host redirects

common/httpx/httpx.go

@@ -80,8 +80,10 @@ func New(options *Options) (*HTTPX, error) {
 	if httpx.Options.FollowRedirects {
 		// Follow redirects up to a maximum number
 		redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
-			// add custom cookies if necessary
-			httpx.setCustomCookies(redirectedRequest)
+			// re-apply secret-file credentials for the new URL and add
+			// custom cookies if the host allows it
+			applyRedirectAuth(redirectedRequest)
+			httpx.setCustomCookies(redirectedRequest, previousRequests)
 
 			if len(previousRequests) >= options.MaxRedirects {
 				// https://github.com/golang/go/issues/10069
@@ -99,8 +101,10 @@ func New(options *Options) (*HTTPX, error) {
 	if httpx.Options.FollowHostRedirects {
 		// Only follow redirects on the same host up to a maximum number
 		redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
-			// add custom cookies if necessary
-			httpx.setCustomCookies(redirectedRequest)
+			// re-apply secret-file credentials for the new URL and add
+			// custom cookies if the host allows it
+			applyRedirectAuth(redirectedRequest)
+			httpx.setCustomCookies(redirectedRequest, previousRequests)
 
 			// Check if we get a redirect to a different host
 			var newHost = redirectedRequest.URL.Hostname()
@@ -580,11 +584,53 @@ func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string][]
 	}
 }
 
-func (httpx *HTTPX) setCustomCookies(req *http.Request) {
-	if httpx.Options.hasCustomCookies() {
-		for _, cookie := range httpx.Options.customCookies {
-			req.AddCookie(cookie)
+// setCustomCookies adds the -H cookies to a redirect hop. Like net/http for
+// the Cookie header, it does so only when the new host is the first host or
+// one of its subdomains, and it skips cookies the request already has.
+func (httpx *HTTPX) setCustomCookies(req *http.Request, via []*http.Request) {
+	if !httpx.Options.hasCustomCookies() || len(via) == 0 {
+		return
+	}
+	if !isDomainOrSubdomain(req.URL.Hostname(), via[0].URL.Hostname()) {
+		return
+	}
+	present := make(map[string]struct{})
+	for _, cookie := range req.Cookies() {
+		present[cookie.Name] = struct{}{}
+	}
+	for _, cookie := range httpx.Options.customCookies {
+		if _, ok := present[cookie.Name]; ok {
+			continue
 		}
+		req.AddCookie(cookie)
+	}
+}
+
+// isDomainOrSubdomain reports whether sub is parent or a subdomain of it.
+func isDomainOrSubdomain(sub, parent string) bool {
+	sub, parent = strings.ToLower(sub), strings.ToLower(parent)
+	if sub == parent {
+		return true
+	}
+	// an IP address has no subdomains
+	if strings.ContainsAny(sub, ":%") || net.ParseIP(sub) != nil {
+		return false
+	}
+	return strings.HasSuffix(sub, "."+parent)
+}
+
+type redirectAuthKey struct{}
+
+// WithRedirectAuth returns a context that makes the client call apply on
+// every redirect hop of a request made with it. apply must remove the
+// credentials of the first URL and add the ones that match the new URL.
+func WithRedirectAuth(ctx context.Context, apply func(*http.Request)) context.Context {
+	return context.WithValue(ctx, redirectAuthKey{}, apply)
+}
+
+func applyRedirectAuth(req *http.Request) {
+	if apply, ok := req.Context().Value(redirectAuthKey{}).(func(*http.Request)); ok && apply != nil {
+		apply(req)
 	}
 }
 

runner/redirect_credentials_test.go

@@ -0,0 +1,136 @@
+package runner
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/projectdiscovery/goflags"
+	"github.com/projectdiscovery/httpx/common/customheader"
+	"github.com/stretchr/testify/require"
+)
+
+type headerLog struct {
+	mu      sync.Mutex
+	headers map[string]http.Header
+}
+
+func (l *headerLog) record(path string, h http.Header) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.headers == nil {
+		l.headers = make(map[string]http.Header)
+	}
+	l.headers[path] = h.Clone()
+}
+
+func (l *headerLog) get(path string) http.Header {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.headers[path]
+}
+
+// TestRedirectCredentialsStayOnHost checks that -fr does not send -H cookies
+// or secret-file headers to another host, and keeps them on the same host.
+func TestRedirectCredentialsStayOnHost(t *testing.T) {
+	log := &headerLog{}
+
+	// [::1] is another host than the 127.0.0.1 target and needs no name resolution
+	ln, err := net.Listen("tcp6", "[::1]:0")
+	if err != nil {
+		t.Skipf("ipv6 loopback is not available: %v", err)
+	}
+	third := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		log.record("third"+r.URL.Path, r.Header)
+		_, _ = io.WriteString(w, "third")
+	}))
+	_ = third.Listener.Close()
+	third.Listener = ln
+	third.Start()
+	defer third.Close()
+	thirdURL := third.URL
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/away", func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, thirdURL+"/landing", http.StatusFound)
+	})
+	mux.HandleFunc("/stay", func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "/final", http.StatusFound)
+	})
+	mux.HandleFunc("/final", func(w http.ResponseWriter, r *http.Request) {
+		log.record("target/final", r.Header)
+		_, _ = io.WriteString(w, "final")
+	})
+	target := httptest.NewServer(mux)
+	defer target.Close()
+
+	targetHost := strings.TrimPrefix(target.URL, "http://")
+	secrets := filepath.Join(t.TempDir(), "secrets.yaml")
+	require.NoError(t, os.WriteFile(secrets, []byte(fmt.Sprintf(`id: test
+info:
+  name: test
+static:
+  - type: Header
+    domains:
+      - %s
+    headers:
+      - key: X-Api-Key
+        value: SECRET-SF-KEY
+  - type: Cookie
+    domains:
+      - %s
+    cookies:
+      - key: sfsess
+        value: SECRET-SF-COOKIE
+`, targetHost, targetHost)), 0o600))
+
+	var mu sync.Mutex
+	var results []Result
+	options := &Options{
+		Methods:         "GET",
+		InputTargetHost: goflags.StringSlice{target.URL + "/away", target.URL + "/stay"},
+		FollowRedirects: true,
+		MaxRedirects:    10,
+		SecretFile:      secrets,
+		CustomHeaders:   customheader.CustomHeaders{"Cookie: sess=SECRET-H-COOKIE"},
+		Threads:         1,
+		Timeout:         5,
+		OnResult: func(r Result) {
+			mu.Lock()
+			results = append(results, r)
+			mu.Unlock()
+		},
+	}
+	require.NoError(t, options.ValidateOptions())
+	r, err := New(options)
+	require.NoError(t, err)
+	r.RunEnumeration()
+	r.Close()
+
+	for _, res := range results {
+		require.NoError(t, res.Err, res.Input)
+	}
+
+	got := log.get("third/landing")
+	require.NotNil(t, got, "redirect to the other host was not followed")
+	require.Empty(t, got.Get("Cookie"), "-H cookie sent to another host")
+	require.Empty(t, got.Get("X-Api-Key"), "secret-file header sent to a host it is not scoped to")
+
+	got = log.get("target/final")
+	require.NotNil(t, got, "same-host redirect was not followed")
+	require.Equal(t, "SECRET-SF-KEY", got.Get("X-Api-Key"))
+	cookies := (&http.Request{Header: got}).Cookies()
+	counts := map[string]int{}
+	for _, c := range cookies {
+		counts[c.Name]++
+	}
+	require.Equal(t, 1, counts["sess"], "-H cookie on a same-host redirect: %v", got.Values("Cookie"))
+	require.Equal(t, 1, counts["sfsess"], "secret-file cookie on a same-host redirect: %v", got.Values("Cookie"))
+}

runner/runner.go

@@ -1954,9 +1954,13 @@ retry:
 	// Apply auth strategies if auth provider is configured
 	if r.authProvider != nil {
 		if strategies := r.authProvider.LookupURLX(URL); len(strategies) > 0 {
+			before := req.Header.Clone()
 			for _, strategy := range strategies {
 				strategy.ApplyOnRR(req)
 			}
+			// net/http copies these headers to every redirect hop, even
+			// to other hosts: scope them to the URLs the secrets name
+			req.WithContext(httpx.WithRedirectAuth(req.Context(), r.redirectAuth(changedHeaders(before, req.Header))))
 		}
 	}
 
@@ -2809,6 +2813,31 @@ retry:
 	return result
 }
 
+// redirectAuth returns a redirect hook that removes the headers the secret
+// file set for the first URL and applies the secrets that match the new URL.
+func (r *Runner) redirectAuth(headers []string) func(*http.Request) {
+	return func(next *http.Request) {
+		for _, name := range headers {
+			next.Header.Del(name)
+		}
+		for _, strategy := range r.authProvider.LookupURL(next.URL) {
+			strategy.Apply(next)
+		}
+	}
+}
+
+// changedHeaders returns the names of the headers that differ between
+// before and after.
+func changedHeaders(before, after http.Header) []string {
+	var names []string
+	for name, values := range after {
+		if !slices.Equal(before.Values(name), values) {
+			names = append(names, name)
+		}
+	}
+	return names
+}
+
 func (r *Runner) skip(URL *urlutil.URL, target httpx.Target, origInput string) (bool, Result) {
 	if r.skipCDNPort(URL.Hostname(), target.CustomIP, URL.Port()) {
 		gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port())