From d4e2490d175457e4ba49e664a1d5470ff89d04b3 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 16 Jun 2026 21:43:40 +0200 Subject: [PATCH] reqx --- common/authprovider/authx/basic_auth.go | 7 - common/authprovider/authx/bearer_auth.go | 7 - common/authprovider/authx/cookies_auth.go | 15 +- common/authprovider/authx/headers_auth.go | 11 -- common/authprovider/authx/query_auth.go | 11 -- common/authprovider/authx/strategy.go | 4 - common/authprovider/authx/strategy_test.go | 24 ++- common/httputilz/httputilz.go | 6 +- common/httpx/dump.go | 69 +++++++ common/httpx/http2.go | 3 +- common/httpx/httpx.go | 192 ++++++++++++-------- common/httpx/httpx_test.go | 36 ++-- common/httpx/reqx.go | 200 +++++++++++++++++++++ common/httpx/trace.go | 112 ++++++++++++ common/httpx/virtualhost.go | 4 +- common/httpx/ztls.go | 59 ++++++ go.mod | 15 +- go.sum | 20 ++- internal/pdcp/writer.go | 37 ++-- runner/runner.go | 44 ++--- runner/types.go | 3 +- 21 files changed, 650 insertions(+), 229 deletions(-) create mode 100644 common/httpx/dump.go create mode 100644 common/httpx/reqx.go create mode 100644 common/httpx/trace.go create mode 100644 common/httpx/ztls.go diff --git a/common/authprovider/authx/basic_auth.go b/common/authprovider/authx/basic_auth.go index b75790662..8c5a67ebb 100644 --- a/common/authprovider/authx/basic_auth.go +++ b/common/authprovider/authx/basic_auth.go @@ -2,8 +2,6 @@ package authx import ( "net/http" - - "github.com/projectdiscovery/retryablehttp-go" ) var ( @@ -24,8 +22,3 @@ func NewBasicAuthStrategy(data *Secret) *BasicAuthStrategy { func (s *BasicAuthStrategy) Apply(req *http.Request) { req.SetBasicAuth(s.Data.Username, s.Data.Password) } - -// ApplyOnRR applies the basic auth strategy to the retryable request -func (s *BasicAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { - req.SetBasicAuth(s.Data.Username, s.Data.Password) -} diff --git a/common/authprovider/authx/bearer_auth.go b/common/authprovider/authx/bearer_auth.go index edf6f439b..e23c081ab 100644 --- a/common/authprovider/authx/bearer_auth.go +++ b/common/authprovider/authx/bearer_auth.go @@ -2,8 +2,6 @@ package authx import ( "net/http" - - "github.com/projectdiscovery/retryablehttp-go" ) var ( @@ -24,8 +22,3 @@ func NewBearerTokenAuthStrategy(data *Secret) *BearerTokenAuthStrategy { func (s *BearerTokenAuthStrategy) Apply(req *http.Request) { req.Header.Set("Authorization", "Bearer "+s.Data.Token) } - -// ApplyOnRR applies the bearer token auth strategy to the retryable request -func (s *BearerTokenAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { - req.Header.Set("Authorization", "Bearer "+s.Data.Token) -} diff --git a/common/authprovider/authx/cookies_auth.go b/common/authprovider/authx/cookies_auth.go index 0d7872b28..2ef04497c 100644 --- a/common/authprovider/authx/cookies_auth.go +++ b/common/authprovider/authx/cookies_auth.go @@ -2,8 +2,6 @@ package authx import ( "net/http" - - "github.com/projectdiscovery/retryablehttp-go" ) var ( @@ -20,18 +18,9 @@ func NewCookiesAuthStrategy(data *Secret) *CookiesAuthStrategy { return &CookiesAuthStrategy{Data: data} } -// Apply applies the cookies auth strategy to the request +// Apply applies the cookies auth strategy to the request, replacing any +// existing cookies that share a name with the configured cookies. func (s *CookiesAuthStrategy) Apply(req *http.Request) { - for _, cookie := range s.Data.Cookies { - req.AddCookie(&http.Cookie{ - Name: cookie.Key, - Value: cookie.Value, - }) - } -} - -// ApplyOnRR applies the cookies auth strategy to the retryable request -func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { // Build a set of cookie names to replace newCookieNames := make(map[string]struct{}, len(s.Data.Cookies)) for _, cookie := range s.Data.Cookies { diff --git a/common/authprovider/authx/headers_auth.go b/common/authprovider/authx/headers_auth.go index d474f75bd..3e2a129fe 100644 --- a/common/authprovider/authx/headers_auth.go +++ b/common/authprovider/authx/headers_auth.go @@ -2,8 +2,6 @@ package authx import ( "net/http" - - "github.com/projectdiscovery/retryablehttp-go" ) var ( @@ -28,12 +26,3 @@ func (s *HeadersAuthStrategy) Apply(req *http.Request) { req.Header[header.Key] = []string{header.Value} } } - -// ApplyOnRR applies the headers auth strategy to the retryable request -// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken) -// This is useful for APIs that require case-sensitive header names -func (s *HeadersAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { - for _, header := range s.Data.Headers { - req.Header[header.Key] = []string{header.Value} - } -} diff --git a/common/authprovider/authx/query_auth.go b/common/authprovider/authx/query_auth.go index 796d8b1fe..656eb5e09 100644 --- a/common/authprovider/authx/query_auth.go +++ b/common/authprovider/authx/query_auth.go @@ -3,7 +3,6 @@ package authx import ( "net/http" - "github.com/projectdiscovery/retryablehttp-go" urlutil "github.com/projectdiscovery/utils/url" ) @@ -30,13 +29,3 @@ func (s *QueryAuthStrategy) Apply(req *http.Request) { } req.URL.RawQuery = q.Encode() } - -// ApplyOnRR applies the query auth strategy to the retryable request -func (s *QueryAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { - q := urlutil.NewOrderedParams() - q.Decode(req.Request.URL.RawQuery) - for _, p := range s.Data.Params { - q.Add(p.Key, p.Value) - } - req.Request.URL.RawQuery = q.Encode() -} diff --git a/common/authprovider/authx/strategy.go b/common/authprovider/authx/strategy.go index 35e42e357..3ba377f32 100644 --- a/common/authprovider/authx/strategy.go +++ b/common/authprovider/authx/strategy.go @@ -2,8 +2,6 @@ package authx import ( "net/http" - - "github.com/projectdiscovery/retryablehttp-go" ) // AuthStrategy is an interface for auth strategies @@ -11,6 +9,4 @@ import ( type AuthStrategy interface { // Apply applies the strategy to the request Apply(*http.Request) - // ApplyOnRR applies the strategy to the retryable request - ApplyOnRR(*retryablehttp.Request) } diff --git a/common/authprovider/authx/strategy_test.go b/common/authprovider/authx/strategy_test.go index 01a8a9f51..04ce45018 100644 --- a/common/authprovider/authx/strategy_test.go +++ b/common/authprovider/authx/strategy_test.go @@ -3,8 +3,6 @@ package authx import ( "net/http" "testing" - - "github.com/projectdiscovery/retryablehttp-go" ) func TestBasicAuthStrategy(t *testing.T) { @@ -31,8 +29,8 @@ func TestBasicAuthStrategy(t *testing.T) { }) t.Run("ApplyOnRR", func(t *testing.T) { - req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) - strategy.ApplyOnRR(req) + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) user, pass, ok := req.BasicAuth() if !ok { @@ -65,8 +63,8 @@ func TestBearerTokenAuthStrategy(t *testing.T) { }) t.Run("ApplyOnRR", func(t *testing.T) { - req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) - strategy.ApplyOnRR(req) + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) auth := req.Header.Get("Authorization") expected := "Bearer mytoken123" @@ -101,8 +99,8 @@ func TestHeadersAuthStrategy(t *testing.T) { }) t.Run("ApplyOnRR", func(t *testing.T) { - req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) - strategy.ApplyOnRR(req) + req, _ := http.NewRequest("GET", "http://example.com", nil) + strategy.Apply(req) // Use direct map access since headers preserve exact casing //nolint @@ -146,13 +144,13 @@ func TestCookiesAuthStrategy(t *testing.T) { }) t.Run("ApplyOnRR replaces existing cookies", func(t *testing.T) { - req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil) + req, _ := http.NewRequest("GET", "http://example.com", nil) // Add existing cookie that should be replaced req.AddCookie(&http.Cookie{Name: "session", Value: "old_value"}) // Add existing cookie that should be kept req.AddCookie(&http.Cookie{Name: "other", Value: "keep_me"}) - strategy.ApplyOnRR(req) + strategy.Apply(req) cookies := req.Cookies() found := make(map[string]string) @@ -200,10 +198,10 @@ func TestQueryAuthStrategy(t *testing.T) { }) t.Run("ApplyOnRR", func(t *testing.T) { - req, _ := retryablehttp.NewRequest("GET", "http://example.com/path?existing=value", nil) - strategy.ApplyOnRR(req) + req, _ := http.NewRequest("GET", "http://example.com/path?existing=value", nil) + strategy.Apply(req) - query := req.Request.URL.Query() + query := req.URL.Query() if got := query.Get("api_key"); got != "secret123" { t.Errorf("api_key = %v, want secret123", got) } diff --git a/common/httputilz/httputilz.go b/common/httputilz/httputilz.go index 492fba691..efa782c35 100644 --- a/common/httputilz/httputilz.go +++ b/common/httputilz/httputilz.go @@ -4,10 +4,10 @@ import ( "bufio" "fmt" "io" + "net/http" "net/http/httputil" "strings" - "github.com/projectdiscovery/retryablehttp-go" urlutil "github.com/projectdiscovery/utils/url" ) @@ -17,8 +17,8 @@ const ( ) // DumpRequest to string -func DumpRequest(req *retryablehttp.Request) (string, error) { - dump, err := httputil.DumpRequestOut(req.Request, true) +func DumpRequest(req *http.Request) (string, error) { + dump, err := httputil.DumpRequestOut(req, true) return string(dump), err } diff --git a/common/httpx/dump.go b/common/httpx/dump.go new file mode 100644 index 000000000..4195d8c53 --- /dev/null +++ b/common/httpx/dump.go @@ -0,0 +1,69 @@ +package httpx + +import ( + "io" + "net/http" + "strings" + + urlutil "github.com/projectdiscovery/utils/url" +) + +// rawNewLine is the wire-level line terminator used for raw request dumps. +const rawNewLine = "\r\n" + +// DumpRequestRaw renders the wire-level representation of an unsafe request, +// mirroring the previous rawhttp.DumpRequestRaw output: the request line, the +// provided headers verbatim (adding Host from the URL only when absent) and the +// body, separated by CRLFs. Content-Length is intentionally not synthesized, +// matching rawhttp's dump behavior. +func DumpRequestRaw(method, rawURL, uriPath string, headers http.Header, body io.Reader) ([]byte, error) { + u, err := urlutil.ParseURL(rawURL, true) + if err != nil { + return nil, err + } + + h := headers.Clone() + if h == nil { + h = http.Header{} + } + if _, hasHost := h["Host"]; !hasHost { + h["Host"] = []string{u.Host} + } + + path := u.Path + if path == "" { + path = "/" + } + if !u.Params.IsEmpty() { + path += "?" + u.Params.Encode() + } + // override with the custom URI path if specified + if uriPath != "" { + path = uriPath + } + + var b strings.Builder + b.WriteString(method + " " + path + " HTTP/1.1" + rawNewLine) + + for key, values := range h { + for _, value := range values { + if value != "" { + b.WriteString(key + ": " + value + rawNewLine) + } else { + b.WriteString(key + rawNewLine) + } + } + } + + b.WriteString(rawNewLine) + + if body != nil { + bodyBytes, err := io.ReadAll(body) + if err != nil { + return nil, err + } + b.Write(bodyBytes) + } + + return []byte(b.String()), nil +} diff --git a/common/httpx/http2.go b/common/httpx/http2.go index 9d1a70528..f85c46efa 100644 --- a/common/httpx/http2.go +++ b/common/httpx/http2.go @@ -7,7 +7,6 @@ import ( "net/http" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/retryablehttp-go" ) const ( @@ -25,7 +24,7 @@ const ( func (h *HTTPX) SupportHTTP2(protocol, method, targetURL string) bool { // http => supports HTTP1.1 => HTTP/2 (H2C) if protocol == HTTP { - req, err := retryablehttp.NewRequest(method, targetURL, nil) + req, err := http.NewRequestWithContext(context.Background(), method, targetURL, nil) if err != nil { return false } diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 27f4d63bc..7b0c7040c 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -2,14 +2,11 @@ package httpx import ( "context" - "crypto/tls" "fmt" "io" - "net" "net/http" "net/textproto" "net/url" - "os" "strconv" "strings" "time" @@ -17,23 +14,20 @@ import ( "github.com/microcosm-cc/bluemonday" "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fastdialer/fastdialer" - "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" "github.com/projectdiscovery/httpx/common/httputilz" "github.com/projectdiscovery/networkpolicy" - "github.com/projectdiscovery/rawhttp" - retryablehttp "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/useragent" "github.com/projectdiscovery/utils/generic" pdhttputil "github.com/projectdiscovery/utils/http" stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" - "golang.org/x/net/http2" ) // HTTPX represent an instance of the library client type HTTPX struct { - client *retryablehttp.Client + client *http.Client client2 *http.Client + clientRaw http.RoundTripper Filters []Filter Options *Options htmlPolicy *bluemonday.Policy @@ -74,10 +68,6 @@ func New(options *Options) (*HTTPX, error) { httpx.Options.parseCustomCookies() - var retryablehttpOptions = retryablehttp.DefaultOptionsSpraying - retryablehttpOptions.Timeout = httpx.Options.Timeout - retryablehttpOptions.RetryMax = httpx.Options.RetryMax - retryablehttpOptions.Trace = options.Trace handleHSTS := func(req *http.Request) { if req.Response.Header.Get("Strict-Transport-Security") == "" { return @@ -138,73 +128,44 @@ func New(options *Options) (*HTTPX, error) { return nil } } - transport := &http.Transport{ - DialContext: httpx.Dialer.Dial, - DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - if options.TlsImpersonate { - return httpx.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}, impersonate.Random, nil) - } - return httpx.Dialer.DialTLS(ctx, network, addr) - }, - MaxIdleConnsPerHost: -1, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - MinVersion: tls.VersionTLS10, - }, - DisableKeepAlives: true, - } - - if httpx.Options.Protocol == HTTP11 { - // disable http2 - _ = os.Setenv("GODEBUG", "http2client=0") - transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} - } - - if httpx.Options.SniName != "" { - transport.TLSClientConfig.ServerName = httpx.Options.SniName - } - if httpx.Options.HTTPProxy != "" { httpx.Options.Proxy = httpx.Options.HTTPProxy } else if httpx.Options.SocksProxy != "" { httpx.Options.Proxy = httpx.Options.SocksProxy } - if httpx.Options.Proxy != "" { - proxyURL, parseErr := url.Parse(httpx.Options.Proxy) - if parseErr != nil { + if _, parseErr := url.Parse(httpx.Options.Proxy); parseErr != nil { return nil, parseErr } - transport.Proxy = http.ProxyURL(proxyURL) - } else { - transport.Proxy = http.ProxyFromEnvironment } - httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{ - Transport: transport, + // reqx backs every request path except ZTLS: reqx has no zcrypto/ztls + // handshake yet, so -ztls keeps using fastdialer's lenient DialTLS through + // the net/http transport. The dedicated HTTP/2 probe client and the unsafe + // (raw) client always use reqx. Retries are layered on the safe client via a + // retry round tripper (replacing retryablehttp). + var safeTransport http.RoundTripper + if httpx.Options.ZTLS { + safeTransport, err = newZTLSTransport(httpx.Dialer, httpx.Options) + if err != nil { + return nil, err + } + } else { + safeTransport = newReqxTransport(httpx.Dialer, httpx.Options) + } + httpx.client = &http.Client{ + Transport: newRetryRoundTripper(safeTransport, httpx.Options.RetryMax), Timeout: httpx.Options.Timeout, CheckRedirect: redirectFunc, - }, retryablehttpOptions) - - if httpx.Options.Protocol == HTTP11 { - httpx.client.HTTPClient2 = httpx.client.HTTPClient } - transport2 := &http2.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - MinVersion: tls.VersionTLS10, - }, - AllowHTTP: true, - } - if httpx.Options.SniName != "" { - transport2.TLSClientConfig.ServerName = httpx.Options.SniName - } httpx.client2 = &http.Client{ - Transport: transport2, + Transport: newReqxHTTP2Transport(httpx.Dialer, httpx.Options), Timeout: httpx.Options.Timeout, } + httpx.clientRaw = newReqxRawTransport(httpx.Dialer, httpx.Options) + httpx.htmlPolicy = bluemonday.NewPolicy() httpx.CustomHeaders = httpx.Options.CustomHeaders @@ -220,7 +181,7 @@ func New(options *Options) (*HTTPX, error) { } // Do http request -func (h *HTTPX) Do(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*Response, error) { +func (h *HTTPX) Do(req *http.Request, unsafeOptions UnsafeOptions) (*Response, error) { timeStart := time.Now() var gzipRetry bool @@ -230,6 +191,13 @@ get_response: return nil, err } + // Some transports (e.g. reqx TLS impersonation) may not populate + // Response.Request; ensure it is set so downstream redirect-chain + // extraction and request dumps don't dereference a nil request. + if httpresp != nil && httpresp.Request == nil { + httpresp.Request = req + } + var shouldIgnoreErrors, shouldIgnoreBodyErrors bool if h.Options.Unsafe && req.Method == http.MethodHead && err != nil && !stringsutil.ContainsAny(err.Error(), "i/o timeout") { @@ -364,26 +332,80 @@ type UnsafeOptions struct { } // getResponse returns response from safe / unsafe request -func (h *HTTPX) getResponse(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (resp *http.Response, err error) { +func (h *HTTPX) getResponse(req *http.Request, unsafeOptions UnsafeOptions) (resp *http.Response, err error) { if h.Options.Unsafe { return h.doUnsafeWithOptions(req, unsafeOptions) } return h.client.Do(req) } -// doUnsafe does an unsafe http request -func (h *HTTPX) doUnsafeWithOptions(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) { - method := req.Method - headers := req.Header - targetURL := req.String() +// rawUnsafeMaxRedirects mirrors rawhttp.DefaultOptions.MaxRedirects. +const rawUnsafeMaxRedirects = 10 + +// isRawRedirect mirrors rawhttp's client Status.IsRedirect: any 3xx except 304. +func isRawRedirect(code int) bool { + if code == http.StatusNotModified { + return false + } + return code >= http.StatusMultipleChoices && code < http.StatusBadRequest +} + +// doUnsafeWithOptions performs an unsafe (wire-level) request via the reqx raw +// engine. reqx's raw mode sends the request line verbatim (no URL +// normalization) while retaining rawhttp-equivalent automatic Host and +// Content-Length behavior. +// +// It follows redirects the same way as the legacy rawhttp path: preserving the +// method, headers and custom URI path across hops, resolving root-relative +// Location values against the current scheme/host, up to rawUnsafeMaxRedirects. +func (h *HTTPX) doUnsafeWithOptions(req *http.Request, unsafeOptions UnsafeOptions) (*http.Response, error) { + header := req.Header.Clone() + host := req.Host + target := req.URL.String() body := req.Body - options := rawhttp.DefaultOptions - options.Timeout = h.Options.Timeout - return rawhttp.DoRawWithOptions(method, targetURL, unsafeOptions.URIPath, headers, body, options) + + current := 0 + for { + hreq, err := http.NewRequestWithContext(req.Context(), req.Method, target, body) + if err != nil { + return nil, err + } + hreq.Header = header.Clone() + if host != "" { + hreq.Host = host + } + // send the URI path exactly as provided (no normalization) via Opaque + if unsafeOptions.URIPath != "" { + hreq.URL.Opaque = unsafeOptions.URIPath + } + + resp, err := h.clientRaw.RoundTrip(hreq) + if err != nil { + return nil, err + } + + if !isRawRedirect(resp.StatusCode) || current > rawUnsafeMaxRedirects { + return resp, nil + } + loc := resp.Header.Get("Location") + if loc == "" { + return resp, nil + } + // drain and close the intermediate response body + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if strings.HasPrefix(loc, "/") { + loc = fmt.Sprintf("%s://%s%s", hreq.URL.Scheme, hreq.URL.Host, loc) + } + target = loc + // the body reader is consumed after the first write (matches rawhttp) + body = nil + current++ + } } // Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true -func (h *HTTPX) Verify(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (bool, error) { +func (h *HTTPX) Verify(req *http.Request, unsafeOptions UnsafeOptions) (bool, error) { resp, err := h.Do(req, unsafeOptions) if err != nil { return false, err @@ -409,21 +431,32 @@ func (h *HTTPX) AddFilter(f Filter) { } // NewRequest from url -func (h *HTTPX) NewRequest(method, targetURL string) (req *retryablehttp.Request, err error) { +func (h *HTTPX) NewRequest(method, targetURL string) (req *http.Request, err error) { return h.NewRequestWithContext(context.Background(), method, targetURL) } // NewRequest from url -func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL string) (req *retryablehttp.Request, err error) { +func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL string) (req *http.Request, err error) { urlx, err := urlutil.ParseURL(targetURL, h.Options.Unsafe) if err != nil { return nil, err } - req, err = retryablehttp.NewRequestFromURLWithContext(ctx, method, urlx, nil) + // we provide a url without path to http.NewRequest and then replace the URL + // instance directly: http.NewRequest internally parses with url.Parse which + // would otherwise drop the patches urlutil.URL applies in unsafe mode + // (e.g. https://scanme.sh/%invalid). Only u.Host is read by net/http; the + // rest of the request URL is carried by the url.URL instance we assign. + req, err = http.NewRequestWithContext(ctx, method, "https://"+urlx.Host, nil) if err != nil { return nil, err } + urlx.Update() + req.URL = urlx.URL + if req.URL.Host != "" && req.URL.Scheme == "" { + req.URL.Scheme = "https" + } + // Skip if unsafe is used if !h.Options.Unsafe { // set default user agent @@ -431,11 +464,16 @@ func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL str // set default encoding to accept utf8 req.Header.Add("Accept-Charset", "utf-8") } + + // attach httptrace collection when tracing is enabled + if h.Options.Trace { + req = withTrace(req) + } return } // SetCustomHeaders on the provided request -func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string][]string) { +func (h *HTTPX) SetCustomHeaders(r *http.Request, headers map[string][]string) { // Coalesce values by canonical header key first. net/http canonicalizes keys // on Del/Add, so case-variant duplicates (e.g. "X-Test" and "x-test") would // otherwise have the second key's Del wipe the values added for the first. @@ -467,7 +505,7 @@ func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string][] r.Header.Set("User-Agent", userAgent.Raw) //nolint } if h.Options.AutoReferer && r.Header.Get("Referer") == "" { - r.Header.Set("Referer", r.String()) + r.Header.Set("Referer", r.URL.String()) } } diff --git a/common/httpx/httpx_test.go b/common/httpx/httpx_test.go index 0dd9cbbca..6810a9360 100644 --- a/common/httpx/httpx_test.go +++ b/common/httpx/httpx_test.go @@ -4,7 +4,6 @@ import ( "net/http" "testing" - "github.com/projectdiscovery/retryablehttp-go" "github.com/stretchr/testify/require" ) @@ -13,7 +12,7 @@ func TestDo(t *testing.T) { require.Nil(t, err) t.Run("content-length in header", func(t *testing.T) { - req, err := retryablehttp.NewRequest(http.MethodGet, "https://scanme.sh", nil) + req, err := http.NewRequest(http.MethodGet, "https://scanme.sh", nil) require.Nil(t, err) resp, err := ht.Do(req, UnsafeOptions{}) require.Nil(t, err) @@ -21,7 +20,7 @@ func TestDo(t *testing.T) { }) t.Run("content-length with binary body", func(t *testing.T) { - req, err := retryablehttp.NewRequest(http.MethodGet, "https://www.w3schools.com/images/favicon.ico", nil) + req, err := http.NewRequest(http.MethodGet, "https://www.w3schools.com/images/favicon.ico", nil) require.Nil(t, err) resp, err := ht.Do(req, UnsafeOptions{}) require.Nil(t, err) @@ -33,21 +32,21 @@ func TestSetCustomHeaders(t *testing.T) { h := &HTTPX{Options: &Options{}} t.Run("duplicate values preserved in order", func(t *testing.T) { - req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) require.NoError(t, err) h.SetCustomHeaders(req, map[string][]string{"X-Test": {"one", "two"}}) require.Equal(t, []string{"one", "two"}, req.Header.Values("X-Test")) }) t.Run("case-variant duplicates are coalesced", func(t *testing.T) { - req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) require.NoError(t, err) h.SetCustomHeaders(req, map[string][]string{"X-Test": {"one"}, "x-test": {"two"}}) require.ElementsMatch(t, []string{"one", "two"}, req.Header.Values("X-Test")) }) t.Run("custom header replaces existing value", func(t *testing.T) { - req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) require.NoError(t, err) req.Header.Set("User-Agent", "default-agent") h.SetCustomHeaders(req, map[string][]string{"User-Agent": {"custom-agent"}}) @@ -55,7 +54,7 @@ func TestSetCustomHeaders(t *testing.T) { }) t.Run("host header sets request host", func(t *testing.T) { - req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) require.NoError(t, err) h.SetCustomHeaders(req, map[string][]string{"Host": {"custom.host"}}) require.Equal(t, "custom.host", req.Host) @@ -63,7 +62,7 @@ func TestSetCustomHeaders(t *testing.T) { }) t.Run("multiple distinct headers preserved", func(t *testing.T) { - req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) require.NoError(t, err) h.SetCustomHeaders(req, map[string][]string{"X-One": {"1"}, "X-Two": {"2"}}) require.Equal(t, []string{"1"}, req.Header.Values("X-One")) @@ -71,14 +70,14 @@ func TestSetCustomHeaders(t *testing.T) { }) t.Run("multiple cookie values preserved", func(t *testing.T) { - req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) require.NoError(t, err) h.SetCustomHeaders(req, map[string][]string{"Cookie": {"a=1", "b=2"}}) require.Equal(t, []string{"a=1", "b=2"}, req.Header.Values("Cookie")) }) t.Run("empty value applied as-is", func(t *testing.T) { - req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) require.NoError(t, err) h.SetCustomHeaders(req, map[string][]string{"X-Empty": {""}}) require.Equal(t, []string{""}, req.Header.Values("X-Empty")) @@ -86,7 +85,7 @@ func TestSetCustomHeaders(t *testing.T) { t.Run("unsafe raw header line stored verbatim as key", func(t *testing.T) { hu := &HTTPX{Options: &Options{Unsafe: true}} - req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) require.NoError(t, err) // in unsafe mode the runner stores the whole raw header line as the key // with an empty value; it must survive canonicalization untouched @@ -102,21 +101,12 @@ func TestParseCustomCookies(t *testing.T) { require.Len(t, options.customCookies, 2) } -func TestHTTP11DisablesRetryableHTTP2FallbackClient(t *testing.T) { +func TestNewClientsInitialized(t *testing.T) { options := DefaultOptions - options.Protocol = HTTP11 ht, err := New(&options) require.NoError(t, err) require.NotNil(t, ht.client) - require.Same(t, ht.client.HTTPClient, ht.client.HTTPClient2) -} - -func TestDefaultProtocolKeepsRetryableHTTP2FallbackClient(t *testing.T) { - options := DefaultOptions - - ht, err := New(&options) - require.NoError(t, err) - require.NotNil(t, ht.client) - require.NotSame(t, ht.client.HTTPClient, ht.client.HTTPClient2) + require.NotNil(t, ht.client2) + require.NotNil(t, ht.clientRaw) } diff --git a/common/httpx/reqx.go b/common/httpx/reqx.go new file mode 100644 index 000000000..61501de3b --- /dev/null +++ b/common/httpx/reqx.go @@ -0,0 +1,200 @@ +package httpx + +import ( + "crypto/tls" + "io" + "math/rand" + "net/http" + "net/url" + "time" + + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/reqx" +) + +// proxyFromOptions resolves the effective proxy URL from the options. +func proxyFromOptions(options *Options) string { + if options.HTTPProxy != "" { + return options.HTTPProxy + } + if options.SocksProxy != "" { + return options.SocksProxy + } + return options.Proxy +} + +// impersonationProfiles is the set of reqx browser profiles used to satisfy +// the -tls-impersonate option (a randomized JA3/JA4 + HTTP/2 fingerprint). +var impersonationProfiles = []reqx.BrowserProfile{ + reqx.ProfileChrome131, + reqx.ProfileChrome132, + reqx.ProfileFirefox133, + reqx.ProfileFirefoxESR, + reqx.ProfileSafari18, + reqx.ProfileEdge131, + reqx.ProfileBrave, +} + +func randomBrowserProfile() reqx.BrowserProfile { + return impersonationProfiles[rand.Intn(len(impersonationProfiles))] +} + +// newReqxTransport builds the reqx.Transport that backs the safe request path. +// fastdialer is preserved as the dialer so DNS caching, dial history and +// network policy keep working; TLS is performed by reqx so response.TLS (and +// therefore TLSGrab) is populated. HTTP/1.1 is forced to match the historical +// httpx client, except under impersonation where a full browser profile +// (TLS + HTTP/2 + header fingerprint) is applied instead. +func newReqxTransport(dialer *fastdialer.Dialer, options *Options) http.RoundTripper { + opts := []reqx.Option{ + reqx.WithDialer(reqx.DialerFunc(dialer.Dial)), + reqx.WithInsecureSkipVerify(), + reqx.WithTLSMinVersion(tls.VersionTLS10), + reqx.WithKeepAlive(false), + reqx.WithTimeout(options.Timeout), + } + + if options.TlsImpersonate { + opts = append(opts, reqx.WithBrowserProfile(randomBrowserProfile())) + } else { + opts = append(opts, reqx.WithForceHTTP1()) + } + + if options.SniName != "" { + opts = append(opts, reqx.WithSNI(options.SniName)) + } + if proxy := proxyFromOptions(options); proxy != "" { + if u, err := url.Parse(proxy); err == nil { + opts = append(opts, reqx.WithProxyURL(u)) + } + } + + return reqx.NewTransport(opts...) +} + +// newReqxRawTransport builds a reqx.Transport that uses the wire-level raw +// engine for unsafe requests (replacing rawhttp). Disabling URL encoding routes +// requests through reqx's raw engine (sending the request line verbatim) while +// keeping rawhttp-equivalent automatic Host and Content-Length behavior. +func newReqxRawTransport(dialer *fastdialer.Dialer, options *Options) http.RoundTripper { + opts := []reqx.Option{ + reqx.WithDialer(reqx.DialerFunc(dialer.Dial)), + reqx.WithInsecureSkipVerify(), + reqx.WithTLSMinVersion(tls.VersionTLS10), + reqx.WithKeepAlive(false), + reqx.WithForceHTTP1(), + reqx.WithTimeout(options.Timeout), + // route through reqx's raw engine without disabling auto Host / + // Content-Length (mirrors rawhttp's AutomaticHostHeader / AutomaticContentLength) + reqx.WithURLEncoding(false), + // rawhttp does not inject Accept-Encoding; keep parity + reqx.WithAutoAcceptEncoding(false), + } + if options.SniName != "" { + opts = append(opts, reqx.WithSNI(options.SniName)) + } + if proxy := proxyFromOptions(options); proxy != "" { + if u, err := url.Parse(proxy); err == nil { + opts = append(opts, reqx.WithProxyURL(u)) + } + } + return reqx.NewTransport(opts...) +} + +// newReqxHTTP2Transport builds a reqx.Transport that negotiates HTTP/2, used by +// the dedicated client that probes for HTTP/2 support (replacing +// golang.org/x/net/http2.Transport) while keeping fastdialer as the dialer. +func newReqxHTTP2Transport(dialer *fastdialer.Dialer, options *Options) http.RoundTripper { + opts := []reqx.Option{ + reqx.WithDialer(reqx.DialerFunc(dialer.Dial)), + reqx.WithInsecureSkipVerify(), + reqx.WithTLSMinVersion(tls.VersionTLS10), + reqx.WithForceHTTP2(), + reqx.WithTimeout(options.Timeout), + } + if options.SniName != "" { + opts = append(opts, reqx.WithSNI(options.SniName)) + } + return reqx.NewTransport(opts...) +} + +// retryRoundTripper wraps an http.RoundTripper and retries transient failures, +// replacing the retry loop previously provided by retryablehttp. It mirrors the +// "spraying" defaults: exponential backoff between waitMin and waitMax, retrying +// on connection errors, 429 and 5xx (except 501). +type retryRoundTripper struct { + next http.RoundTripper + maxRetries int + waitMin time.Duration + waitMax time.Duration +} + +func newRetryRoundTripper(next http.RoundTripper, maxRetries int) *retryRoundTripper { + return &retryRoundTripper{ + next: next, + maxRetries: maxRetries, + waitMin: 1 * time.Second, + waitMax: 30 * time.Second, + } +} + +func (rt *retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + attempts := rt.maxRetries + 1 + if attempts < 1 { + attempts = 1 + } + + var resp *http.Response + var err error + for attempt := 0; attempt < attempts; attempt++ { + if attempt > 0 { + // rewind the body for the retry, if possible + if req.GetBody != nil { + if body, berr := req.GetBody(); berr == nil { + req.Body = body + } + } + select { + case <-req.Context().Done(): + return nil, req.Context().Err() + case <-time.After(rt.backoff(attempt)): + } + } + + resp, err = rt.next.RoundTrip(req) + if !shouldRetry(resp, err) { + return resp, err + } + // drain the failed response before retrying so the connection is reusable + if resp != nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + } + return resp, err +} + +func (rt *retryRoundTripper) backoff(attempt int) time.Duration { + d := rt.waitMin << (attempt - 1) + if d <= 0 || d > rt.waitMax { + d = rt.waitMax + } + return d +} + +// shouldRetry mirrors retryablehttp's default retry policy. +func shouldRetry(resp *http.Response, err error) bool { + if err != nil { + return true + } + if resp == nil { + return true + } + if resp.StatusCode == http.StatusTooManyRequests { + return true + } + if resp.StatusCode >= 500 && resp.StatusCode != http.StatusNotImplemented { + return true + } + return false +} diff --git a/common/httpx/trace.go b/common/httpx/trace.go new file mode 100644 index 000000000..3f320d438 --- /dev/null +++ b/common/httpx/trace.go @@ -0,0 +1,112 @@ +package httpx + +import ( + "context" + "crypto/tls" + "net/http" + "net/http/httptrace" + "time" +) + +// TraceEventInfo captures a single httptrace event with its timestamp. +type TraceEventInfo struct { + Time time.Time + Info interface{} +} + +// TraceInfo aggregates the httptrace events collected for a request. It mirrors +// the shape previously provided by retryablehttp so the JSON `trace` output is +// unchanged. +type TraceInfo struct { + GotConn TraceEventInfo + DNSDone TraceEventInfo + GetConn TraceEventInfo + PutIdleConn TraceEventInfo + GotFirstResponseByte TraceEventInfo + Got100Continue TraceEventInfo + DNSStart TraceEventInfo + ConnectStart TraceEventInfo + ConnectDone TraceEventInfo + TLSHandshakeStart TraceEventInfo + TLSHandshakeDone TraceEventInfo + WroteHeaders TraceEventInfo + WroteRequest TraceEventInfo +} + +type traceCtxKey struct{} + +// withTrace attaches a TraceInfo collector and the corresponding +// httptrace.ClientTrace to the request context. The reqx transport delegates to +// net/http.Transport in standard mode, so these hooks fire as usual. +func withTrace(req *http.Request) *http.Request { + traceInfo := &TraceInfo{} + trace := &httptrace.ClientTrace{ + GotConn: func(connInfo httptrace.GotConnInfo) { + traceInfo.GotConn = TraceEventInfo{Time: time.Now(), Info: connInfo} + }, + DNSDone: func(dnsInfo httptrace.DNSDoneInfo) { + traceInfo.DNSDone = TraceEventInfo{Time: time.Now(), Info: dnsInfo} + }, + GetConn: func(hostPort string) { + traceInfo.GetConn = TraceEventInfo{Time: time.Now(), Info: hostPort} + }, + PutIdleConn: func(err error) { + traceInfo.PutIdleConn = TraceEventInfo{Time: time.Now(), Info: err} + }, + GotFirstResponseByte: func() { + traceInfo.GotFirstResponseByte = TraceEventInfo{Time: time.Now()} + }, + Got100Continue: func() { + traceInfo.Got100Continue = TraceEventInfo{Time: time.Now()} + }, + DNSStart: func(di httptrace.DNSStartInfo) { + traceInfo.DNSStart = TraceEventInfo{Time: time.Now(), Info: di} + }, + ConnectStart: func(network, addr string) { + traceInfo.ConnectStart = TraceEventInfo{Time: time.Now(), Info: struct { + Network, Addr string + }{network, addr}} + }, + ConnectDone: func(network, addr string, err error) { + if err == nil { + traceInfo.ConnectDone = TraceEventInfo{Time: time.Now(), Info: struct { + Network, Addr string + Error error + }{network, addr, err}} + } + }, + TLSHandshakeStart: func() { + traceInfo.TLSHandshakeStart = TraceEventInfo{Time: time.Now()} + }, + TLSHandshakeDone: func(cs tls.ConnectionState, err error) { + if err == nil { + traceInfo.TLSHandshakeDone = TraceEventInfo{Time: time.Now(), Info: struct { + ConnectionState tls.ConnectionState + Error error + }{cs, err}} + } + }, + WroteHeaders: func() { + traceInfo.WroteHeaders = TraceEventInfo{Time: time.Now()} + }, + WroteRequest: func(wri httptrace.WroteRequestInfo) { + traceInfo.WroteRequest = TraceEventInfo{Time: time.Now(), Info: wri} + }, + } + + ctx := context.WithValue(req.Context(), traceCtxKey{}, traceInfo) + ctx = httptrace.WithClientTrace(ctx, trace) + return req.WithContext(ctx) +} + +// GetTraceInfo returns the TraceInfo collected for the request, or nil if +// tracing was not enabled for it. +func GetTraceInfo(req *http.Request) *TraceInfo { + if req == nil { + return nil + } + if ti, ok := req.Context().Value(traceCtxKey{}).(*TraceInfo); ok { + return ti + } + return nil +} diff --git a/common/httpx/virtualhost.go b/common/httpx/virtualhost.go index 77e3d4794..f92e27ac4 100644 --- a/common/httpx/virtualhost.go +++ b/common/httpx/virtualhost.go @@ -2,16 +2,16 @@ package httpx import ( "fmt" + "net/http" "github.com/hbakhtiyor/strsim" - retryablehttp "github.com/projectdiscovery/retryablehttp-go" "github.com/rs/xid" ) const simMultiplier = 100 // IsVirtualHost checks if the target endpoint is a virtual host -func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (bool, error) { +func (h *HTTPX) IsVirtualHost(req *http.Request, unsafeOptions UnsafeOptions) (bool, error) { httpresp1, err := h.Do(req, unsafeOptions) if err != nil { return false, err diff --git a/common/httpx/ztls.go b/common/httpx/ztls.go new file mode 100644 index 000000000..78f2810ad --- /dev/null +++ b/common/httpx/ztls.go @@ -0,0 +1,59 @@ +package httpx + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "net/url" + + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" +) + +// newZTLSTransport builds the net/http transport used only for the ZTLS +// (-ztls) request path. reqx does not yet implement the lenient zcrypto/ztls +// handshake, so ZTLS requests keep using fastdialer's DialTLS (which performs +// the ztls handshake when the dialer is created WithZTLS). Every other request +// path uses reqx. +// +// This mirrors the historical httpx transport: it forces HTTP/1.1 (no automatic +// HTTP/2 upgrade), disables keep-alives and accepts any certificate. When +// -tls-impersonate is combined with -ztls, impersonation takes precedence for +// the handshake, matching the previous behavior. +func newZTLSTransport(dialer *fastdialer.Dialer, options *Options) (http.RoundTripper, error) { + transport := &http.Transport{ + DialContext: dialer.Dial, + DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + if options.TlsImpersonate { + return dialer.DialTLSWithConfigImpersonate(ctx, network, addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}, impersonate.Random, nil) + } + return dialer.DialTLS(ctx, network, addr) + }, + MaxIdleConnsPerHost: -1, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS10, + }, + DisableKeepAlives: true, + // disable net/http's automatic HTTP/2 negotiation so the ztls path + // stays on HTTP/1.1 (matching the reqx default path) + TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{}, + } + + if options.SniName != "" { + transport.TLSClientConfig.ServerName = options.SniName + } + + if options.Proxy != "" { + proxyURL, parseErr := url.Parse(options.Proxy) + if parseErr != nil { + return nil, parseErr + } + transport.Proxy = http.ProxyURL(proxyURL) + } else { + transport.Proxy = http.ProxyFromEnvironment + } + + return transport, nil +} diff --git a/go.mod b/go.mod index cc4275f26..aba42e2c8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/httpx -go 1.26 +go 1.26.1 require ( github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 @@ -31,8 +31,7 @@ require ( github.com/projectdiscovery/mapcidr v1.1.97 github.com/projectdiscovery/networkpolicy v0.1.40 github.com/projectdiscovery/ratelimit v0.0.88 - github.com/projectdiscovery/rawhttp v0.1.90 - github.com/projectdiscovery/retryablehttp-go v1.3.15 + github.com/projectdiscovery/retryablehttp-go v1.3.15 // indirect github.com/projectdiscovery/tlsx v1.2.2 github.com/projectdiscovery/useragent v0.0.108 github.com/projectdiscovery/utils v0.11.1 @@ -57,12 +56,20 @@ require ( github.com/happyhackingspace/dit v0.0.27 github.com/lib/pq v1.12.3 github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 + github.com/projectdiscovery/reqx v0.0.0-20260508172251-6205af3493df github.com/seh-msft/burpxml v1.0.1 github.com/weppos/publicsuffix-go v0.50.3 go.mongodb.org/mongo-driver v1.17.9 gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.69.0 // indirect +) + require ( aead.dev/minisign v0.2.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect @@ -181,7 +188,7 @@ require ( golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.21.0 // indirect golang.org/x/term v0.44.0 // indirect - golang.org/x/time v0.14.0 // indirect + golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.45.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index be19c38ba..f66561ef4 100644 --- a/go.sum +++ b/go.sum @@ -357,14 +357,12 @@ github.com/projectdiscovery/networkpolicy v0.1.40 h1:kYin4u1/dgb0nuz5fE1bz4Q0Zh6 github.com/projectdiscovery/networkpolicy v0.1.40/go.mod h1:9ULLaMbdv9UnT0C5rmuK4nIwYs0o776xMnkPUb8TtaE= github.com/projectdiscovery/ratelimit v0.0.88 h1:AcurW9aLRzlEyPe9kSjnOpr3XzLMWTpiWAlW/w73ALU= github.com/projectdiscovery/ratelimit v0.0.88/go.mod h1:CU1s+68UUG2mctSl2wi32/DHLJA6TMg+4rxgP59LfVk= -github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw= -github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y= +github.com/projectdiscovery/reqx v0.0.0-20260508172251-6205af3493df h1:hQB3kGci6kFK8UtTYhgK70UOR4wrkmkVwl/RSPcmXyw= +github.com/projectdiscovery/reqx v0.0.0-20260508172251-6205af3493df/go.mod h1:hK9i+LySFhIu8tgt1j8BON9pJr/46avEJ5En7MgTRN0= github.com/projectdiscovery/retryabledns v1.0.115 h1:RKV63FNIznFHUoawg/1hs53pVH3wqPtFhwstCuxVSoA= github.com/projectdiscovery/retryabledns v1.0.115/go.mod h1:+fEMWoPigw+M0lGNKY7AZ+g8FIgj+4sONjsinMmeL3k= github.com/projectdiscovery/retryablehttp-go v1.3.15 h1:qhJzaWWRras9Il66HbWU0DJ35clFJoz/ktQvks1ogGU= github.com/projectdiscovery/retryablehttp-go v1.3.15/go.mod h1:s0azLAqAbcVCjHI9t0ezPhamevYGM1eoOvFkn4QmpZ8= -github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiBLvK+SE= github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4= github.com/projectdiscovery/useragent v0.0.108 h1:fb+uLuFJvC+MHZjCtxQJxtvp1X6A8n98CUGPyFcg3NE= @@ -374,6 +372,10 @@ github.com/projectdiscovery/utils v0.11.1/go.mod h1:yktGrHGk2CTjNiccXovnvGrLHX9s github.com/projectdiscovery/wappalyzergo v0.2.85 h1:RIJwaR2ViAoHSYBE/1ytNjvzXV+mfFDeTjr9eXAgaCU= github.com/projectdiscovery/wappalyzergo v0.2.85/go.mod h1:gMH0o5lBp65sKMwHx/tuUdOtW2RjodC6Ti+9QDsYMkY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -452,6 +454,10 @@ github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZ github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI= +github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw= github.com/vulncheck-oss/go-exploit v1.51.0 h1:HTmJ4Q94tbEDPb35mQZn6qMg4rT+Sw9n+L7g3Pjr+3o= github.com/vulncheck-oss/go-exploit v1.51.0/go.mod h1:J28w0dLnA6DnCrnBm9Sbt6smX8lvztnnN2wCXy7No6c= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= @@ -512,6 +518,8 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= @@ -695,8 +703,8 @@ golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go index fbad91abd..15815bb4e 100644 --- a/internal/pdcp/writer.go +++ b/internal/pdcp/writer.go @@ -14,7 +14,6 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/httpx/runner" - "github.com/projectdiscovery/retryablehttp-go" pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" "github.com/projectdiscovery/utils/conversion" "github.com/projectdiscovery/utils/env" @@ -46,7 +45,7 @@ var ( type UploadWriter struct { creds *pdcpauth.PDCPCredentials uploadURL *url.URL - client *retryablehttp.Client + client *http.Client done chan struct{} data chan runner.Result assetGroupID string @@ -78,10 +77,7 @@ func NewUploadWriterCallback(ctx context.Context, creds *pdcpauth.PDCPCredential u.uploadURL = tmp.URL // create http client - opts := retryablehttp.DefaultOptionsSingle - opts.NoAdjustTimeout = true - opts.Timeout = time.Duration(3) * time.Minute - u.client = retryablehttp.NewClient(opts) + u.client = &http.Client{Timeout: time.Duration(3) * time.Minute} // start auto commit // upload every 1 minute or when buffer is full go u.autoCommit(ctx) @@ -226,29 +222,32 @@ func (u *UploadWriter) upload(data []byte) error { // getRequest returns a new request for upload // if scanID is not provided create new scan by uploading the data // if scanID is provided append the data to existing scan -func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) { - var method, url string - - if u.assetGroupID == "" { +func (u *UploadWriter) getRequest(bin []byte) (*http.Request, error) { + var method string + isUpload := u.assetGroupID == "" + if isUpload { u.uploadURL.Path = uploadEndpoint method = http.MethodPost - url = u.uploadURL.String() } else { u.uploadURL.Path = fmt.Sprintf(appendEndpoint, u.assetGroupID) method = http.MethodPatch - url = u.uploadURL.String() } - req, err := retryablehttp.NewRequest(method, url, bytes.NewReader(bin)) + + // add pdtm meta params (already url-encoded) + values, err := url.ParseQuery(updateutils.GetpdtmParams(runner.Version)) if err != nil { - return nil, errkit.Wrap(err, "could not create cloud upload request") + return nil, errkit.Wrap(err, "could not parse pdtm params") } - // add pdtm meta params - req.Params.Merge(updateutils.GetpdtmParams(runner.Version)) // if it is upload endpoint also include name if it exists - if u.assetGroupName != "" && req.Path == uploadEndpoint { - req.Params.Add("name", u.assetGroupName) + if u.assetGroupName != "" && isUpload { + values.Add("name", u.assetGroupName) + } + u.uploadURL.RawQuery = values.Encode() + + req, err := http.NewRequest(method, u.uploadURL.String(), bytes.NewReader(bin)) + if err != nil { + return nil, errkit.Wrap(err, "could not create cloud upload request") } - req.Update() req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey) if u.TeamID != "" { diff --git a/runner/runner.go b/runner/runner.go index eed5d886b..b0d151a54 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -51,7 +51,6 @@ import ( "github.com/projectdiscovery/clistats" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/httpx/common/hashes" - "github.com/projectdiscovery/retryablehttp-go" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" @@ -68,7 +67,6 @@ import ( "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/httpx/common/stringz" "github.com/projectdiscovery/mapcidr" - "github.com/projectdiscovery/rawhttp" converstionutil "github.com/projectdiscovery/utils/conversion" errkit "github.com/projectdiscovery/utils/errkit" fileutil "github.com/projectdiscovery/utils/file" @@ -285,16 +283,6 @@ func New(options *Options) (*Runner, error) { options.RequestBody = rrBody } - // disable automatic host header for rawhttp if manually specified - // as it can be malformed the best approach is to remove spaces and check for lowercase "host" word - if options.Unsafe { - for name := range runner.hp.CustomHeaders { - nameLower := strings.TrimSpace(strings.ToLower(name)) - if strings.HasPrefix(nameLower, "host") { - rawhttp.AutomaticHostHeader(false) - } - } - } if strings.EqualFold(options.Methods, "all") { scanopts.Methods = pdhttputil.AllHTTPMethods() } else if options.Methods != "" { @@ -1865,7 +1853,7 @@ retry: gologger.Debug().Msgf("failed to merge paths of url %v and %v", URL.String(), scanopts.RequestURI) } var ( - req *retryablehttp.Request + req *http.Request ctx context.Context ) if target.CustomIP != "" { @@ -1903,7 +1891,7 @@ retry: if r.authProvider != nil { if strategies := r.authProvider.LookupURLX(URL); len(strategies) > 0 { for _, strategy := range strategies { - strategy.ApplyOnRR(req) + strategy.Apply(req) } } } @@ -1912,6 +1900,10 @@ retry: if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody)) + // GetBody enables the body to be rewound for retries and 307/308 redirects + req.GetBody = func() (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader(scanopts.RequestBody)), nil + } } else { req.ContentLength = 0 req.Body = nil @@ -1930,7 +1922,7 @@ retry: var requestDump []byte if scanopts.Unsafe { var errDump error - requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions) + requestDump, errDump = httpx.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body) if errDump != nil { return Result{URL: URL.String(), Input: origInput, Err: errDump} } @@ -1941,7 +1933,7 @@ retry: req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody)) } var errDump error - requestDump, errDump = httputil.DumpRequestOut(req.Request, true) + requestDump, errDump = httputil.DumpRequestOut(req, true) if errDump != nil { return Result{URL: URL.String(), Input: origInput, Err: errDump} } @@ -1953,7 +1945,7 @@ retry: } } // fix the final output url - fullURL := req.String() + fullURL := req.URL.String() if parsedURL, errParse := r.parseURL(fullURL); errParse != nil { return Result{URL: URL.String(), Input: origInput, Err: errParse} } else { @@ -2707,7 +2699,7 @@ retry: result.Domains = resp.BodyDomains.Domains } if r.options.Trace { - result.Trace = req.TraceInfo + result.Trace = httpx.GetTraceInfo(req) } return result } @@ -2742,11 +2734,11 @@ func calculatePerceptionHash(screenshotBytes []byte) (uint64, error) { return pHash.GetHash(), nil } -func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, finalURL string, defaultProbe bool) (string, string, string, []byte, string, error) { +func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *http.Request, currentResp []byte, finalURL string, defaultProbe bool) (string, string, string, []byte, string, error) { // Check if current URI is ending with .ico => use current body without additional requests - if path.Ext(req.Path) == ".ico" { + if path.Ext(req.URL.Path) == ".ico" { mmh3, md5h, err := r.calculateFaviconHashWithRaw(currentResp) - return mmh3, md5h, req.Path, currentResp, req.String(), err + return mmh3, md5h, req.URL.Path, currentResp, req.URL.String(), err } // Parse HTML: collect hrefs + optional @@ -2761,7 +2753,7 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, } // Determine base URL: prefer finalURL (redirect target) then apply - baseNet, _ := url.Parse(req.String()) + baseNet, _ := url.Parse(req.URL.String()) if finalURL != "" { if u, err := url.Parse(finalURL); err == nil { baseNet = u @@ -2819,7 +2811,8 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, continue } - clone.SetURL(resolvedURL) + resolvedURL.Update() + clone.URL = resolvedURL.URL // Update Host header to match resolved URL host (important after redirects) if resolvedURL.Host != "" && resolvedURL.Host != clone.Host { clone.Host = resolvedURL.Host @@ -2834,7 +2827,8 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, if err2 != nil { continue } - clone.SetURL(rootResolvedURL) + rootResolvedURL.Update() + clone.URL = rootResolvedURL.URL if respFav2, err3 := hp.Do(clone, httpx.UnsafeOptions{}); err3 == nil && len(respFav2.Data) > 0 { respFav = respFav2 } else { @@ -2853,7 +2847,7 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, faviconMMH3 = mmh3 faviconMD5 = md5h faviconPath = raw - faviconURL = clone.String() + faviconURL = clone.URL.String() faviconData = respFav.Data gologger.Debug().Msgf("favicon resolved url=%s raw_href=%s size=%d bytes", faviconURL, faviconPath, len(faviconData)) break diff --git a/runner/types.go b/runner/types.go index 1eaa3608e..56446a64d 100644 --- a/runner/types.go +++ b/runner/types.go @@ -12,7 +12,6 @@ import ( mapstructure "github.com/go-viper/mapstructure/v2" "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" mapsutil "github.com/projectdiscovery/utils/maps" wappalyzer "github.com/projectdiscovery/wappalyzergo" @@ -101,7 +100,7 @@ type Result struct { RequestRaw []byte `json:"-" csv:"-" md:"-" mapstructure:"-"` Response *httpx.Response `json:"-" csv:"-" md:"-" mapstructure:"-"` FaviconData []byte `json:"-" csv:"-" md:"-" mapstructure:"-"` - Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" md:"-" mapstructure:"trace"` + Trace *httpx.TraceInfo `json:"trace,omitempty" csv:"-" md:"-" mapstructure:"trace"` FileNameHash string `json:"-" csv:"-" md:"-" mapstructure:"-"` CPE []CPEInfo `json:"cpe,omitempty" csv:"cpe" md:"cpe" mapstructure:"cpe"` WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress" md:"wordpress" mapstructure:"wordpress"`