diff --git a/common/authprovider/authx/basic_auth.go b/common/authprovider/authx/basic_auth.go
index b75790662..8c5a67ebb 100644
--- a/common/authprovider/authx/basic_auth.go
+++ b/common/authprovider/authx/basic_auth.go
@@ -2,8 +2,6 @@ package authx
import (
"net/http"
-
- "github.com/projectdiscovery/retryablehttp-go"
)
var (
@@ -24,8 +22,3 @@ func NewBasicAuthStrategy(data *Secret) *BasicAuthStrategy {
func (s *BasicAuthStrategy) Apply(req *http.Request) {
req.SetBasicAuth(s.Data.Username, s.Data.Password)
}
-
-// ApplyOnRR applies the basic auth strategy to the retryable request
-func (s *BasicAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
- req.SetBasicAuth(s.Data.Username, s.Data.Password)
-}
diff --git a/common/authprovider/authx/bearer_auth.go b/common/authprovider/authx/bearer_auth.go
index edf6f439b..e23c081ab 100644
--- a/common/authprovider/authx/bearer_auth.go
+++ b/common/authprovider/authx/bearer_auth.go
@@ -2,8 +2,6 @@ package authx
import (
"net/http"
-
- "github.com/projectdiscovery/retryablehttp-go"
)
var (
@@ -24,8 +22,3 @@ func NewBearerTokenAuthStrategy(data *Secret) *BearerTokenAuthStrategy {
func (s *BearerTokenAuthStrategy) Apply(req *http.Request) {
req.Header.Set("Authorization", "Bearer "+s.Data.Token)
}
-
-// ApplyOnRR applies the bearer token auth strategy to the retryable request
-func (s *BearerTokenAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
- req.Header.Set("Authorization", "Bearer "+s.Data.Token)
-}
diff --git a/common/authprovider/authx/cookies_auth.go b/common/authprovider/authx/cookies_auth.go
index 0d7872b28..2ef04497c 100644
--- a/common/authprovider/authx/cookies_auth.go
+++ b/common/authprovider/authx/cookies_auth.go
@@ -2,8 +2,6 @@ package authx
import (
"net/http"
-
- "github.com/projectdiscovery/retryablehttp-go"
)
var (
@@ -20,18 +18,9 @@ func NewCookiesAuthStrategy(data *Secret) *CookiesAuthStrategy {
return &CookiesAuthStrategy{Data: data}
}
-// Apply applies the cookies auth strategy to the request
+// Apply applies the cookies auth strategy to the request, replacing any
+// existing cookies that share a name with the configured cookies.
func (s *CookiesAuthStrategy) Apply(req *http.Request) {
- for _, cookie := range s.Data.Cookies {
- req.AddCookie(&http.Cookie{
- Name: cookie.Key,
- Value: cookie.Value,
- })
- }
-}
-
-// ApplyOnRR applies the cookies auth strategy to the retryable request
-func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
// Build a set of cookie names to replace
newCookieNames := make(map[string]struct{}, len(s.Data.Cookies))
for _, cookie := range s.Data.Cookies {
diff --git a/common/authprovider/authx/headers_auth.go b/common/authprovider/authx/headers_auth.go
index d474f75bd..3e2a129fe 100644
--- a/common/authprovider/authx/headers_auth.go
+++ b/common/authprovider/authx/headers_auth.go
@@ -2,8 +2,6 @@ package authx
import (
"net/http"
-
- "github.com/projectdiscovery/retryablehttp-go"
)
var (
@@ -28,12 +26,3 @@ func (s *HeadersAuthStrategy) Apply(req *http.Request) {
req.Header[header.Key] = []string{header.Value}
}
}
-
-// ApplyOnRR applies the headers auth strategy to the retryable request
-// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken)
-// This is useful for APIs that require case-sensitive header names
-func (s *HeadersAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
- for _, header := range s.Data.Headers {
- req.Header[header.Key] = []string{header.Value}
- }
-}
diff --git a/common/authprovider/authx/query_auth.go b/common/authprovider/authx/query_auth.go
index 796d8b1fe..656eb5e09 100644
--- a/common/authprovider/authx/query_auth.go
+++ b/common/authprovider/authx/query_auth.go
@@ -3,7 +3,6 @@ package authx
import (
"net/http"
- "github.com/projectdiscovery/retryablehttp-go"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -30,13 +29,3 @@ func (s *QueryAuthStrategy) Apply(req *http.Request) {
}
req.URL.RawQuery = q.Encode()
}
-
-// ApplyOnRR applies the query auth strategy to the retryable request
-func (s *QueryAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
- q := urlutil.NewOrderedParams()
- q.Decode(req.Request.URL.RawQuery)
- for _, p := range s.Data.Params {
- q.Add(p.Key, p.Value)
- }
- req.Request.URL.RawQuery = q.Encode()
-}
diff --git a/common/authprovider/authx/strategy.go b/common/authprovider/authx/strategy.go
index 35e42e357..3ba377f32 100644
--- a/common/authprovider/authx/strategy.go
+++ b/common/authprovider/authx/strategy.go
@@ -2,8 +2,6 @@ package authx
import (
"net/http"
-
- "github.com/projectdiscovery/retryablehttp-go"
)
// AuthStrategy is an interface for auth strategies
@@ -11,6 +9,4 @@ import (
type AuthStrategy interface {
// Apply applies the strategy to the request
Apply(*http.Request)
- // ApplyOnRR applies the strategy to the retryable request
- ApplyOnRR(*retryablehttp.Request)
}
diff --git a/common/authprovider/authx/strategy_test.go b/common/authprovider/authx/strategy_test.go
index 01a8a9f51..04ce45018 100644
--- a/common/authprovider/authx/strategy_test.go
+++ b/common/authprovider/authx/strategy_test.go
@@ -3,8 +3,6 @@ package authx
import (
"net/http"
"testing"
-
- "github.com/projectdiscovery/retryablehttp-go"
)
func TestBasicAuthStrategy(t *testing.T) {
@@ -31,8 +29,8 @@ func TestBasicAuthStrategy(t *testing.T) {
})
t.Run("ApplyOnRR", func(t *testing.T) {
- req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil)
- strategy.ApplyOnRR(req)
+ req, _ := http.NewRequest("GET", "http://example.com", nil)
+ strategy.Apply(req)
user, pass, ok := req.BasicAuth()
if !ok {
@@ -65,8 +63,8 @@ func TestBearerTokenAuthStrategy(t *testing.T) {
})
t.Run("ApplyOnRR", func(t *testing.T) {
- req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil)
- strategy.ApplyOnRR(req)
+ req, _ := http.NewRequest("GET", "http://example.com", nil)
+ strategy.Apply(req)
auth := req.Header.Get("Authorization")
expected := "Bearer mytoken123"
@@ -101,8 +99,8 @@ func TestHeadersAuthStrategy(t *testing.T) {
})
t.Run("ApplyOnRR", func(t *testing.T) {
- req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil)
- strategy.ApplyOnRR(req)
+ req, _ := http.NewRequest("GET", "http://example.com", nil)
+ strategy.Apply(req)
// Use direct map access since headers preserve exact casing
//nolint
@@ -146,13 +144,13 @@ func TestCookiesAuthStrategy(t *testing.T) {
})
t.Run("ApplyOnRR replaces existing cookies", func(t *testing.T) {
- req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil)
+ req, _ := http.NewRequest("GET", "http://example.com", nil)
// Add existing cookie that should be replaced
req.AddCookie(&http.Cookie{Name: "session", Value: "old_value"})
// Add existing cookie that should be kept
req.AddCookie(&http.Cookie{Name: "other", Value: "keep_me"})
- strategy.ApplyOnRR(req)
+ strategy.Apply(req)
cookies := req.Cookies()
found := make(map[string]string)
@@ -200,10 +198,10 @@ func TestQueryAuthStrategy(t *testing.T) {
})
t.Run("ApplyOnRR", func(t *testing.T) {
- req, _ := retryablehttp.NewRequest("GET", "http://example.com/path?existing=value", nil)
- strategy.ApplyOnRR(req)
+ req, _ := http.NewRequest("GET", "http://example.com/path?existing=value", nil)
+ strategy.Apply(req)
- query := req.Request.URL.Query()
+ query := req.URL.Query()
if got := query.Get("api_key"); got != "secret123" {
t.Errorf("api_key = %v, want secret123", got)
}
diff --git a/common/httputilz/httputilz.go b/common/httputilz/httputilz.go
index 492fba691..efa782c35 100644
--- a/common/httputilz/httputilz.go
+++ b/common/httputilz/httputilz.go
@@ -4,10 +4,10 @@ import (
"bufio"
"fmt"
"io"
+ "net/http"
"net/http/httputil"
"strings"
- "github.com/projectdiscovery/retryablehttp-go"
urlutil "github.com/projectdiscovery/utils/url"
)
@@ -17,8 +17,8 @@ const (
)
// DumpRequest to string
-func DumpRequest(req *retryablehttp.Request) (string, error) {
- dump, err := httputil.DumpRequestOut(req.Request, true)
+func DumpRequest(req *http.Request) (string, error) {
+ dump, err := httputil.DumpRequestOut(req, true)
return string(dump), err
}
diff --git a/common/httpx/dump.go b/common/httpx/dump.go
new file mode 100644
index 000000000..4195d8c53
--- /dev/null
+++ b/common/httpx/dump.go
@@ -0,0 +1,69 @@
+package httpx
+
+import (
+ "io"
+ "net/http"
+ "strings"
+
+ urlutil "github.com/projectdiscovery/utils/url"
+)
+
+// rawNewLine is the wire-level line terminator used for raw request dumps.
+const rawNewLine = "\r\n"
+
+// DumpRequestRaw renders the wire-level representation of an unsafe request,
+// mirroring the previous rawhttp.DumpRequestRaw output: the request line, the
+// provided headers verbatim (adding Host from the URL only when absent) and the
+// body, separated by CRLFs. Content-Length is intentionally not synthesized,
+// matching rawhttp's dump behavior.
+func DumpRequestRaw(method, rawURL, uriPath string, headers http.Header, body io.Reader) ([]byte, error) {
+ u, err := urlutil.ParseURL(rawURL, true)
+ if err != nil {
+ return nil, err
+ }
+
+ h := headers.Clone()
+ if h == nil {
+ h = http.Header{}
+ }
+ if _, hasHost := h["Host"]; !hasHost {
+ h["Host"] = []string{u.Host}
+ }
+
+ path := u.Path
+ if path == "" {
+ path = "/"
+ }
+ if !u.Params.IsEmpty() {
+ path += "?" + u.Params.Encode()
+ }
+ // override with the custom URI path if specified
+ if uriPath != "" {
+ path = uriPath
+ }
+
+ var b strings.Builder
+ b.WriteString(method + " " + path + " HTTP/1.1" + rawNewLine)
+
+ for key, values := range h {
+ for _, value := range values {
+ if value != "" {
+ b.WriteString(key + ": " + value + rawNewLine)
+ } else {
+ b.WriteString(key + rawNewLine)
+ }
+ }
+ }
+
+ b.WriteString(rawNewLine)
+
+ if body != nil {
+ bodyBytes, err := io.ReadAll(body)
+ if err != nil {
+ return nil, err
+ }
+ b.Write(bodyBytes)
+ }
+
+ return []byte(b.String()), nil
+}
diff --git a/common/httpx/http2.go b/common/httpx/http2.go
index 9d1a70528..f85c46efa 100644
--- a/common/httpx/http2.go
+++ b/common/httpx/http2.go
@@ -7,7 +7,6 @@ import (
"net/http"
"github.com/projectdiscovery/gologger"
- "github.com/projectdiscovery/retryablehttp-go"
)
const (
@@ -25,7 +24,7 @@ const (
func (h *HTTPX) SupportHTTP2(protocol, method, targetURL string) bool {
// http => supports HTTP1.1 => HTTP/2 (H2C)
if protocol == HTTP {
- req, err := retryablehttp.NewRequest(method, targetURL, nil)
+ req, err := http.NewRequestWithContext(context.Background(), method, targetURL, nil)
if err != nil {
return false
}
diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go
index 27f4d63bc..7b0c7040c 100644
--- a/common/httpx/httpx.go
+++ b/common/httpx/httpx.go
@@ -2,14 +2,11 @@ package httpx
import (
"context"
- "crypto/tls"
"fmt"
"io"
- "net"
"net/http"
"net/textproto"
"net/url"
- "os"
"strconv"
"strings"
"time"
@@ -17,23 +14,20 @@ import (
"github.com/microcosm-cc/bluemonday"
"github.com/projectdiscovery/cdncheck"
"github.com/projectdiscovery/fastdialer/fastdialer"
- "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
"github.com/projectdiscovery/httpx/common/httputilz"
"github.com/projectdiscovery/networkpolicy"
- "github.com/projectdiscovery/rawhttp"
- retryablehttp "github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/useragent"
"github.com/projectdiscovery/utils/generic"
pdhttputil "github.com/projectdiscovery/utils/http"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
- "golang.org/x/net/http2"
)
// HTTPX represent an instance of the library client
type HTTPX struct {
- client *retryablehttp.Client
+ client *http.Client
client2 *http.Client
+ clientRaw http.RoundTripper
Filters []Filter
Options *Options
htmlPolicy *bluemonday.Policy
@@ -74,10 +68,6 @@ func New(options *Options) (*HTTPX, error) {
httpx.Options.parseCustomCookies()
- var retryablehttpOptions = retryablehttp.DefaultOptionsSpraying
- retryablehttpOptions.Timeout = httpx.Options.Timeout
- retryablehttpOptions.RetryMax = httpx.Options.RetryMax
- retryablehttpOptions.Trace = options.Trace
handleHSTS := func(req *http.Request) {
if req.Response.Header.Get("Strict-Transport-Security") == "" {
return
@@ -138,73 +128,44 @@ func New(options *Options) (*HTTPX, error) {
return nil
}
}
- transport := &http.Transport{
- DialContext: httpx.Dialer.Dial,
- DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
- if options.TlsImpersonate {
- return httpx.Dialer.DialTLSWithConfigImpersonate(ctx, network, addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}, impersonate.Random, nil)
- }
- return httpx.Dialer.DialTLS(ctx, network, addr)
- },
- MaxIdleConnsPerHost: -1,
- TLSClientConfig: &tls.Config{
- InsecureSkipVerify: true,
- MinVersion: tls.VersionTLS10,
- },
- DisableKeepAlives: true,
- }
-
- if httpx.Options.Protocol == HTTP11 {
- // disable http2
- _ = os.Setenv("GODEBUG", "http2client=0")
- transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
- }
-
- if httpx.Options.SniName != "" {
- transport.TLSClientConfig.ServerName = httpx.Options.SniName
- }
-
if httpx.Options.HTTPProxy != "" {
httpx.Options.Proxy = httpx.Options.HTTPProxy
} else if httpx.Options.SocksProxy != "" {
httpx.Options.Proxy = httpx.Options.SocksProxy
}
-
if httpx.Options.Proxy != "" {
- proxyURL, parseErr := url.Parse(httpx.Options.Proxy)
- if parseErr != nil {
+ if _, parseErr := url.Parse(httpx.Options.Proxy); parseErr != nil {
return nil, parseErr
}
- transport.Proxy = http.ProxyURL(proxyURL)
- } else {
- transport.Proxy = http.ProxyFromEnvironment
}
- httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{
- Transport: transport,
+ // reqx backs every request path except ZTLS: reqx has no zcrypto/ztls
+ // handshake yet, so -ztls keeps using fastdialer's lenient DialTLS through
+ // the net/http transport. The dedicated HTTP/2 probe client and the unsafe
+ // (raw) client always use reqx. Retries are layered on the safe client via a
+ // retry round tripper (replacing retryablehttp).
+ var safeTransport http.RoundTripper
+ if httpx.Options.ZTLS {
+ safeTransport, err = newZTLSTransport(httpx.Dialer, httpx.Options)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ safeTransport = newReqxTransport(httpx.Dialer, httpx.Options)
+ }
+ httpx.client = &http.Client{
+ Transport: newRetryRoundTripper(safeTransport, httpx.Options.RetryMax),
Timeout: httpx.Options.Timeout,
CheckRedirect: redirectFunc,
- }, retryablehttpOptions)
-
- if httpx.Options.Protocol == HTTP11 {
- httpx.client.HTTPClient2 = httpx.client.HTTPClient
}
- transport2 := &http2.Transport{
- TLSClientConfig: &tls.Config{
- InsecureSkipVerify: true,
- MinVersion: tls.VersionTLS10,
- },
- AllowHTTP: true,
- }
- if httpx.Options.SniName != "" {
- transport2.TLSClientConfig.ServerName = httpx.Options.SniName
- }
httpx.client2 = &http.Client{
- Transport: transport2,
+ Transport: newReqxHTTP2Transport(httpx.Dialer, httpx.Options),
Timeout: httpx.Options.Timeout,
}
+ httpx.clientRaw = newReqxRawTransport(httpx.Dialer, httpx.Options)
+
httpx.htmlPolicy = bluemonday.NewPolicy()
httpx.CustomHeaders = httpx.Options.CustomHeaders
@@ -220,7 +181,7 @@ func New(options *Options) (*HTTPX, error) {
}
// Do http request
-func (h *HTTPX) Do(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*Response, error) {
+func (h *HTTPX) Do(req *http.Request, unsafeOptions UnsafeOptions) (*Response, error) {
timeStart := time.Now()
var gzipRetry bool
@@ -230,6 +191,13 @@ get_response:
return nil, err
}
+ // Some transports (e.g. reqx TLS impersonation) may not populate
+ // Response.Request; ensure it is set so downstream redirect-chain
+ // extraction and request dumps don't dereference a nil request.
+ if httpresp != nil && httpresp.Request == nil {
+ httpresp.Request = req
+ }
+
var shouldIgnoreErrors, shouldIgnoreBodyErrors bool
if h.Options.Unsafe && req.Method == http.MethodHead && err != nil &&
!stringsutil.ContainsAny(err.Error(), "i/o timeout") {
@@ -364,26 +332,80 @@ type UnsafeOptions struct {
}
// getResponse returns response from safe / unsafe request
-func (h *HTTPX) getResponse(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (resp *http.Response, err error) {
+func (h *HTTPX) getResponse(req *http.Request, unsafeOptions UnsafeOptions) (resp *http.Response, err error) {
if h.Options.Unsafe {
return h.doUnsafeWithOptions(req, unsafeOptions)
}
return h.client.Do(req)
}
-// doUnsafe does an unsafe http request
-func (h *HTTPX) doUnsafeWithOptions(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) {
- method := req.Method
- headers := req.Header
- targetURL := req.String()
+// rawUnsafeMaxRedirects mirrors rawhttp.DefaultOptions.MaxRedirects.
+const rawUnsafeMaxRedirects = 10
+
+// isRawRedirect mirrors rawhttp's client Status.IsRedirect: any 3xx except 304.
+func isRawRedirect(code int) bool {
+ if code == http.StatusNotModified {
+ return false
+ }
+ return code >= http.StatusMultipleChoices && code < http.StatusBadRequest
+}
+
+// doUnsafeWithOptions performs an unsafe (wire-level) request via the reqx raw
+// engine. reqx's raw mode sends the request line verbatim (no URL
+// normalization) while retaining rawhttp-equivalent automatic Host and
+// Content-Length behavior.
+//
+// It follows redirects the same way as the legacy rawhttp path: preserving the
+// method, headers and custom URI path across hops, resolving root-relative
+// Location values against the current scheme/host, up to rawUnsafeMaxRedirects.
+func (h *HTTPX) doUnsafeWithOptions(req *http.Request, unsafeOptions UnsafeOptions) (*http.Response, error) {
+ header := req.Header.Clone()
+ host := req.Host
+ target := req.URL.String()
body := req.Body
- options := rawhttp.DefaultOptions
- options.Timeout = h.Options.Timeout
- return rawhttp.DoRawWithOptions(method, targetURL, unsafeOptions.URIPath, headers, body, options)
+
+ current := 0
+ for {
+ hreq, err := http.NewRequestWithContext(req.Context(), req.Method, target, body)
+ if err != nil {
+ return nil, err
+ }
+ hreq.Header = header.Clone()
+ if host != "" {
+ hreq.Host = host
+ }
+ // send the URI path exactly as provided (no normalization) via Opaque
+ if unsafeOptions.URIPath != "" {
+ hreq.URL.Opaque = unsafeOptions.URIPath
+ }
+
+ resp, err := h.clientRaw.RoundTrip(hreq)
+ if err != nil {
+ return nil, err
+ }
+
+ if !isRawRedirect(resp.StatusCode) || current > rawUnsafeMaxRedirects {
+ return resp, nil
+ }
+ loc := resp.Header.Get("Location")
+ if loc == "" {
+ return resp, nil
+ }
+ // drain and close the intermediate response body
+ _, _ = io.Copy(io.Discard, resp.Body)
+ _ = resp.Body.Close()
+ if strings.HasPrefix(loc, "/") {
+ loc = fmt.Sprintf("%s://%s%s", hreq.URL.Scheme, hreq.URL.Host, loc)
+ }
+ target = loc
+ // the body reader is consumed after the first write (matches rawhttp)
+ body = nil
+ current++
+ }
}
// Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true
-func (h *HTTPX) Verify(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (bool, error) {
+func (h *HTTPX) Verify(req *http.Request, unsafeOptions UnsafeOptions) (bool, error) {
resp, err := h.Do(req, unsafeOptions)
if err != nil {
return false, err
@@ -409,21 +431,32 @@ func (h *HTTPX) AddFilter(f Filter) {
}
// NewRequest from url
-func (h *HTTPX) NewRequest(method, targetURL string) (req *retryablehttp.Request, err error) {
+func (h *HTTPX) NewRequest(method, targetURL string) (req *http.Request, err error) {
return h.NewRequestWithContext(context.Background(), method, targetURL)
}
// NewRequest from url
-func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL string) (req *retryablehttp.Request, err error) {
+func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL string) (req *http.Request, err error) {
urlx, err := urlutil.ParseURL(targetURL, h.Options.Unsafe)
if err != nil {
return nil, err
}
- req, err = retryablehttp.NewRequestFromURLWithContext(ctx, method, urlx, nil)
+ // we provide a url without path to http.NewRequest and then replace the URL
+ // instance directly: http.NewRequest internally parses with url.Parse which
+ // would otherwise drop the patches urlutil.URL applies in unsafe mode
+ // (e.g. https://scanme.sh/%invalid). Only u.Host is read by net/http; the
+ // rest of the request URL is carried by the url.URL instance we assign.
+ req, err = http.NewRequestWithContext(ctx, method, "https://"+urlx.Host, nil)
if err != nil {
return nil, err
}
+ urlx.Update()
+ req.URL = urlx.URL
+ if req.URL.Host != "" && req.URL.Scheme == "" {
+ req.URL.Scheme = "https"
+ }
+
// Skip if unsafe is used
if !h.Options.Unsafe {
// set default user agent
@@ -431,11 +464,16 @@ func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL str
// set default encoding to accept utf8
req.Header.Add("Accept-Charset", "utf-8")
}
+
+ // attach httptrace collection when tracing is enabled
+ if h.Options.Trace {
+ req = withTrace(req)
+ }
return
}
// SetCustomHeaders on the provided request
-func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string][]string) {
+func (h *HTTPX) SetCustomHeaders(r *http.Request, headers map[string][]string) {
// Coalesce values by canonical header key first. net/http canonicalizes keys
// on Del/Add, so case-variant duplicates (e.g. "X-Test" and "x-test") would
// otherwise have the second key's Del wipe the values added for the first.
@@ -467,7 +505,7 @@ func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string][]
r.Header.Set("User-Agent", userAgent.Raw) //nolint
}
if h.Options.AutoReferer && r.Header.Get("Referer") == "" {
- r.Header.Set("Referer", r.String())
+ r.Header.Set("Referer", r.URL.String())
}
}
diff --git a/common/httpx/httpx_test.go b/common/httpx/httpx_test.go
index 0dd9cbbca..6810a9360 100644
--- a/common/httpx/httpx_test.go
+++ b/common/httpx/httpx_test.go
@@ -4,7 +4,6 @@ import (
"net/http"
"testing"
- "github.com/projectdiscovery/retryablehttp-go"
"github.com/stretchr/testify/require"
)
@@ -13,7 +12,7 @@ func TestDo(t *testing.T) {
require.Nil(t, err)
t.Run("content-length in header", func(t *testing.T) {
- req, err := retryablehttp.NewRequest(http.MethodGet, "https://scanme.sh", nil)
+ req, err := http.NewRequest(http.MethodGet, "https://scanme.sh", nil)
require.Nil(t, err)
resp, err := ht.Do(req, UnsafeOptions{})
require.Nil(t, err)
@@ -21,7 +20,7 @@ func TestDo(t *testing.T) {
})
t.Run("content-length with binary body", func(t *testing.T) {
- req, err := retryablehttp.NewRequest(http.MethodGet, "https://www.w3schools.com/images/favicon.ico", nil)
+ req, err := http.NewRequest(http.MethodGet, "https://www.w3schools.com/images/favicon.ico", nil)
require.Nil(t, err)
resp, err := ht.Do(req, UnsafeOptions{})
require.Nil(t, err)
@@ -33,21 +32,21 @@ func TestSetCustomHeaders(t *testing.T) {
h := &HTTPX{Options: &Options{}}
t.Run("duplicate values preserved in order", func(t *testing.T) {
- req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
+ req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
h.SetCustomHeaders(req, map[string][]string{"X-Test": {"one", "two"}})
require.Equal(t, []string{"one", "two"}, req.Header.Values("X-Test"))
})
t.Run("case-variant duplicates are coalesced", func(t *testing.T) {
- req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
+ req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
h.SetCustomHeaders(req, map[string][]string{"X-Test": {"one"}, "x-test": {"two"}})
require.ElementsMatch(t, []string{"one", "two"}, req.Header.Values("X-Test"))
})
t.Run("custom header replaces existing value", func(t *testing.T) {
- req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
+ req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
req.Header.Set("User-Agent", "default-agent")
h.SetCustomHeaders(req, map[string][]string{"User-Agent": {"custom-agent"}})
@@ -55,7 +54,7 @@ func TestSetCustomHeaders(t *testing.T) {
})
t.Run("host header sets request host", func(t *testing.T) {
- req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
+ req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
h.SetCustomHeaders(req, map[string][]string{"Host": {"custom.host"}})
require.Equal(t, "custom.host", req.Host)
@@ -63,7 +62,7 @@ func TestSetCustomHeaders(t *testing.T) {
})
t.Run("multiple distinct headers preserved", func(t *testing.T) {
- req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
+ req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
h.SetCustomHeaders(req, map[string][]string{"X-One": {"1"}, "X-Two": {"2"}})
require.Equal(t, []string{"1"}, req.Header.Values("X-One"))
@@ -71,14 +70,14 @@ func TestSetCustomHeaders(t *testing.T) {
})
t.Run("multiple cookie values preserved", func(t *testing.T) {
- req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
+ req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
h.SetCustomHeaders(req, map[string][]string{"Cookie": {"a=1", "b=2"}})
require.Equal(t, []string{"a=1", "b=2"}, req.Header.Values("Cookie"))
})
t.Run("empty value applied as-is", func(t *testing.T) {
- req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
+ req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
h.SetCustomHeaders(req, map[string][]string{"X-Empty": {""}})
require.Equal(t, []string{""}, req.Header.Values("X-Empty"))
@@ -86,7 +85,7 @@ func TestSetCustomHeaders(t *testing.T) {
t.Run("unsafe raw header line stored verbatim as key", func(t *testing.T) {
hu := &HTTPX{Options: &Options{Unsafe: true}}
- req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com", nil)
+ req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
require.NoError(t, err)
// in unsafe mode the runner stores the whole raw header line as the key
// with an empty value; it must survive canonicalization untouched
@@ -102,21 +101,12 @@ func TestParseCustomCookies(t *testing.T) {
require.Len(t, options.customCookies, 2)
}
-func TestHTTP11DisablesRetryableHTTP2FallbackClient(t *testing.T) {
+func TestNewClientsInitialized(t *testing.T) {
options := DefaultOptions
- options.Protocol = HTTP11
ht, err := New(&options)
require.NoError(t, err)
require.NotNil(t, ht.client)
- require.Same(t, ht.client.HTTPClient, ht.client.HTTPClient2)
-}
-
-func TestDefaultProtocolKeepsRetryableHTTP2FallbackClient(t *testing.T) {
- options := DefaultOptions
-
- ht, err := New(&options)
- require.NoError(t, err)
- require.NotNil(t, ht.client)
- require.NotSame(t, ht.client.HTTPClient, ht.client.HTTPClient2)
+ require.NotNil(t, ht.client2)
+ require.NotNil(t, ht.clientRaw)
}
diff --git a/common/httpx/reqx.go b/common/httpx/reqx.go
new file mode 100644
index 000000000..61501de3b
--- /dev/null
+++ b/common/httpx/reqx.go
@@ -0,0 +1,200 @@
+package httpx
+
+import (
+ "crypto/tls"
+ "io"
+ "math/rand"
+ "net/http"
+ "net/url"
+ "time"
+
+ "github.com/projectdiscovery/fastdialer/fastdialer"
+ "github.com/projectdiscovery/reqx"
+)
+
+// proxyFromOptions resolves the effective proxy URL from the options.
+func proxyFromOptions(options *Options) string {
+ if options.HTTPProxy != "" {
+ return options.HTTPProxy
+ }
+ if options.SocksProxy != "" {
+ return options.SocksProxy
+ }
+ return options.Proxy
+}
+
+// impersonationProfiles is the set of reqx browser profiles used to satisfy
+// the -tls-impersonate option (a randomized JA3/JA4 + HTTP/2 fingerprint).
+var impersonationProfiles = []reqx.BrowserProfile{
+ reqx.ProfileChrome131,
+ reqx.ProfileChrome132,
+ reqx.ProfileFirefox133,
+ reqx.ProfileFirefoxESR,
+ reqx.ProfileSafari18,
+ reqx.ProfileEdge131,
+ reqx.ProfileBrave,
+}
+
+func randomBrowserProfile() reqx.BrowserProfile {
+ return impersonationProfiles[rand.Intn(len(impersonationProfiles))]
+}
+
+// newReqxTransport builds the reqx.Transport that backs the safe request path.
+// fastdialer is preserved as the dialer so DNS caching, dial history and
+// network policy keep working; TLS is performed by reqx so response.TLS (and
+// therefore TLSGrab) is populated. HTTP/1.1 is forced to match the historical
+// httpx client, except under impersonation where a full browser profile
+// (TLS + HTTP/2 + header fingerprint) is applied instead.
+func newReqxTransport(dialer *fastdialer.Dialer, options *Options) http.RoundTripper {
+ opts := []reqx.Option{
+ reqx.WithDialer(reqx.DialerFunc(dialer.Dial)),
+ reqx.WithInsecureSkipVerify(),
+ reqx.WithTLSMinVersion(tls.VersionTLS10),
+ reqx.WithKeepAlive(false),
+ reqx.WithTimeout(options.Timeout),
+ }
+
+ if options.TlsImpersonate {
+ opts = append(opts, reqx.WithBrowserProfile(randomBrowserProfile()))
+ } else {
+ opts = append(opts, reqx.WithForceHTTP1())
+ }
+
+ if options.SniName != "" {
+ opts = append(opts, reqx.WithSNI(options.SniName))
+ }
+ if proxy := proxyFromOptions(options); proxy != "" {
+ if u, err := url.Parse(proxy); err == nil {
+ opts = append(opts, reqx.WithProxyURL(u))
+ }
+ }
+
+ return reqx.NewTransport(opts...)
+}
+
+// newReqxRawTransport builds a reqx.Transport that uses the wire-level raw
+// engine for unsafe requests (replacing rawhttp). Disabling URL encoding routes
+// requests through reqx's raw engine (sending the request line verbatim) while
+// keeping rawhttp-equivalent automatic Host and Content-Length behavior.
+func newReqxRawTransport(dialer *fastdialer.Dialer, options *Options) http.RoundTripper {
+ opts := []reqx.Option{
+ reqx.WithDialer(reqx.DialerFunc(dialer.Dial)),
+ reqx.WithInsecureSkipVerify(),
+ reqx.WithTLSMinVersion(tls.VersionTLS10),
+ reqx.WithKeepAlive(false),
+ reqx.WithForceHTTP1(),
+ reqx.WithTimeout(options.Timeout),
+ // route through reqx's raw engine without disabling auto Host /
+ // Content-Length (mirrors rawhttp's AutomaticHostHeader / AutomaticContentLength)
+ reqx.WithURLEncoding(false),
+ // rawhttp does not inject Accept-Encoding; keep parity
+ reqx.WithAutoAcceptEncoding(false),
+ }
+ if options.SniName != "" {
+ opts = append(opts, reqx.WithSNI(options.SniName))
+ }
+ if proxy := proxyFromOptions(options); proxy != "" {
+ if u, err := url.Parse(proxy); err == nil {
+ opts = append(opts, reqx.WithProxyURL(u))
+ }
+ }
+ return reqx.NewTransport(opts...)
+}
+
+// newReqxHTTP2Transport builds a reqx.Transport that negotiates HTTP/2, used by
+// the dedicated client that probes for HTTP/2 support (replacing
+// golang.org/x/net/http2.Transport) while keeping fastdialer as the dialer.
+func newReqxHTTP2Transport(dialer *fastdialer.Dialer, options *Options) http.RoundTripper {
+ opts := []reqx.Option{
+ reqx.WithDialer(reqx.DialerFunc(dialer.Dial)),
+ reqx.WithInsecureSkipVerify(),
+ reqx.WithTLSMinVersion(tls.VersionTLS10),
+ reqx.WithForceHTTP2(),
+ reqx.WithTimeout(options.Timeout),
+ }
+ if options.SniName != "" {
+ opts = append(opts, reqx.WithSNI(options.SniName))
+ }
+ return reqx.NewTransport(opts...)
+}
+
+// retryRoundTripper wraps an http.RoundTripper and retries transient failures,
+// replacing the retry loop previously provided by retryablehttp. It mirrors the
+// "spraying" defaults: exponential backoff between waitMin and waitMax, retrying
+// on connection errors, 429 and 5xx (except 501).
+type retryRoundTripper struct {
+ next http.RoundTripper
+ maxRetries int
+ waitMin time.Duration
+ waitMax time.Duration
+}
+
+func newRetryRoundTripper(next http.RoundTripper, maxRetries int) *retryRoundTripper {
+ return &retryRoundTripper{
+ next: next,
+ maxRetries: maxRetries,
+ waitMin: 1 * time.Second,
+ waitMax: 30 * time.Second,
+ }
+}
+
+func (rt *retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+ attempts := rt.maxRetries + 1
+ if attempts < 1 {
+ attempts = 1
+ }
+
+ var resp *http.Response
+ var err error
+ for attempt := 0; attempt < attempts; attempt++ {
+ if attempt > 0 {
+ // rewind the body for the retry, if possible
+ if req.GetBody != nil {
+ if body, berr := req.GetBody(); berr == nil {
+ req.Body = body
+ }
+ }
+ select {
+ case <-req.Context().Done():
+ return nil, req.Context().Err()
+ case <-time.After(rt.backoff(attempt)):
+ }
+ }
+
+ resp, err = rt.next.RoundTrip(req)
+ if !shouldRetry(resp, err) {
+ return resp, err
+ }
+ // drain the failed response before retrying so the connection is reusable
+ if resp != nil {
+ _, _ = io.Copy(io.Discard, resp.Body)
+ _ = resp.Body.Close()
+ }
+ }
+ return resp, err
+}
+
+func (rt *retryRoundTripper) backoff(attempt int) time.Duration {
+ d := rt.waitMin << (attempt - 1)
+ if d <= 0 || d > rt.waitMax {
+ d = rt.waitMax
+ }
+ return d
+}
+
+// shouldRetry mirrors retryablehttp's default retry policy.
+func shouldRetry(resp *http.Response, err error) bool {
+ if err != nil {
+ return true
+ }
+ if resp == nil {
+ return true
+ }
+ if resp.StatusCode == http.StatusTooManyRequests {
+ return true
+ }
+ if resp.StatusCode >= 500 && resp.StatusCode != http.StatusNotImplemented {
+ return true
+ }
+ return false
+}
diff --git a/common/httpx/trace.go b/common/httpx/trace.go
new file mode 100644
index 000000000..3f320d438
--- /dev/null
+++ b/common/httpx/trace.go
@@ -0,0 +1,112 @@
+package httpx
+
+import (
+ "context"
+ "crypto/tls"
+ "net/http"
+ "net/http/httptrace"
+ "time"
+)
+
+// TraceEventInfo captures a single httptrace event with its timestamp.
+type TraceEventInfo struct {
+ Time time.Time
+ Info interface{}
+}
+
+// TraceInfo aggregates the httptrace events collected for a request. It mirrors
+// the shape previously provided by retryablehttp so the JSON `trace` output is
+// unchanged.
+type TraceInfo struct {
+ GotConn TraceEventInfo
+ DNSDone TraceEventInfo
+ GetConn TraceEventInfo
+ PutIdleConn TraceEventInfo
+ GotFirstResponseByte TraceEventInfo
+ Got100Continue TraceEventInfo
+ DNSStart TraceEventInfo
+ ConnectStart TraceEventInfo
+ ConnectDone TraceEventInfo
+ TLSHandshakeStart TraceEventInfo
+ TLSHandshakeDone TraceEventInfo
+ WroteHeaders TraceEventInfo
+ WroteRequest TraceEventInfo
+}
+
+type traceCtxKey struct{}
+
+// withTrace attaches a TraceInfo collector and the corresponding
+// httptrace.ClientTrace to the request context. The reqx transport delegates to
+// net/http.Transport in standard mode, so these hooks fire as usual.
+func withTrace(req *http.Request) *http.Request {
+ traceInfo := &TraceInfo{}
+ trace := &httptrace.ClientTrace{
+ GotConn: func(connInfo httptrace.GotConnInfo) {
+ traceInfo.GotConn = TraceEventInfo{Time: time.Now(), Info: connInfo}
+ },
+ DNSDone: func(dnsInfo httptrace.DNSDoneInfo) {
+ traceInfo.DNSDone = TraceEventInfo{Time: time.Now(), Info: dnsInfo}
+ },
+ GetConn: func(hostPort string) {
+ traceInfo.GetConn = TraceEventInfo{Time: time.Now(), Info: hostPort}
+ },
+ PutIdleConn: func(err error) {
+ traceInfo.PutIdleConn = TraceEventInfo{Time: time.Now(), Info: err}
+ },
+ GotFirstResponseByte: func() {
+ traceInfo.GotFirstResponseByte = TraceEventInfo{Time: time.Now()}
+ },
+ Got100Continue: func() {
+ traceInfo.Got100Continue = TraceEventInfo{Time: time.Now()}
+ },
+ DNSStart: func(di httptrace.DNSStartInfo) {
+ traceInfo.DNSStart = TraceEventInfo{Time: time.Now(), Info: di}
+ },
+ ConnectStart: func(network, addr string) {
+ traceInfo.ConnectStart = TraceEventInfo{Time: time.Now(), Info: struct {
+ Network, Addr string
+ }{network, addr}}
+ },
+ ConnectDone: func(network, addr string, err error) {
+ if err == nil {
+ traceInfo.ConnectDone = TraceEventInfo{Time: time.Now(), Info: struct {
+ Network, Addr string
+ Error error
+ }{network, addr, err}}
+ }
+ },
+ TLSHandshakeStart: func() {
+ traceInfo.TLSHandshakeStart = TraceEventInfo{Time: time.Now()}
+ },
+ TLSHandshakeDone: func(cs tls.ConnectionState, err error) {
+ if err == nil {
+ traceInfo.TLSHandshakeDone = TraceEventInfo{Time: time.Now(), Info: struct {
+ ConnectionState tls.ConnectionState
+ Error error
+ }{cs, err}}
+ }
+ },
+ WroteHeaders: func() {
+ traceInfo.WroteHeaders = TraceEventInfo{Time: time.Now()}
+ },
+ WroteRequest: func(wri httptrace.WroteRequestInfo) {
+ traceInfo.WroteRequest = TraceEventInfo{Time: time.Now(), Info: wri}
+ },
+ }
+
+ ctx := context.WithValue(req.Context(), traceCtxKey{}, traceInfo)
+ ctx = httptrace.WithClientTrace(ctx, trace)
+ return req.WithContext(ctx)
+}
+
+// GetTraceInfo returns the TraceInfo collected for the request, or nil if
+// tracing was not enabled for it.
+func GetTraceInfo(req *http.Request) *TraceInfo {
+ if req == nil {
+ return nil
+ }
+ if ti, ok := req.Context().Value(traceCtxKey{}).(*TraceInfo); ok {
+ return ti
+ }
+ return nil
+}
diff --git a/common/httpx/virtualhost.go b/common/httpx/virtualhost.go
index 77e3d4794..f92e27ac4 100644
--- a/common/httpx/virtualhost.go
+++ b/common/httpx/virtualhost.go
@@ -2,16 +2,16 @@ package httpx
import (
"fmt"
+ "net/http"
"github.com/hbakhtiyor/strsim"
- retryablehttp "github.com/projectdiscovery/retryablehttp-go"
"github.com/rs/xid"
)
const simMultiplier = 100
// IsVirtualHost checks if the target endpoint is a virtual host
-func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (bool, error) {
+func (h *HTTPX) IsVirtualHost(req *http.Request, unsafeOptions UnsafeOptions) (bool, error) {
httpresp1, err := h.Do(req, unsafeOptions)
if err != nil {
return false, err
diff --git a/common/httpx/ztls.go b/common/httpx/ztls.go
new file mode 100644
index 000000000..78f2810ad
--- /dev/null
+++ b/common/httpx/ztls.go
@@ -0,0 +1,59 @@
+package httpx
+
+import (
+ "context"
+ "crypto/tls"
+ "net"
+ "net/http"
+ "net/url"
+
+ "github.com/projectdiscovery/fastdialer/fastdialer"
+ "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
+)
+
+// newZTLSTransport builds the net/http transport used only for the ZTLS
+// (-ztls) request path. reqx does not yet implement the lenient zcrypto/ztls
+// handshake, so ZTLS requests keep using fastdialer's DialTLS (which performs
+// the ztls handshake when the dialer is created WithZTLS). Every other request
+// path uses reqx.
+//
+// This mirrors the historical httpx transport: it forces HTTP/1.1 (no automatic
+// HTTP/2 upgrade), disables keep-alives and accepts any certificate. When
+// -tls-impersonate is combined with -ztls, impersonation takes precedence for
+// the handshake, matching the previous behavior.
+func newZTLSTransport(dialer *fastdialer.Dialer, options *Options) (http.RoundTripper, error) {
+ transport := &http.Transport{
+ DialContext: dialer.Dial,
+ DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+ if options.TlsImpersonate {
+ return dialer.DialTLSWithConfigImpersonate(ctx, network, addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}, impersonate.Random, nil)
+ }
+ return dialer.DialTLS(ctx, network, addr)
+ },
+ MaxIdleConnsPerHost: -1,
+ TLSClientConfig: &tls.Config{
+ InsecureSkipVerify: true,
+ MinVersion: tls.VersionTLS10,
+ },
+ DisableKeepAlives: true,
+ // disable net/http's automatic HTTP/2 negotiation so the ztls path
+ // stays on HTTP/1.1 (matching the reqx default path)
+ TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{},
+ }
+
+ if options.SniName != "" {
+ transport.TLSClientConfig.ServerName = options.SniName
+ }
+
+ if options.Proxy != "" {
+ proxyURL, parseErr := url.Parse(options.Proxy)
+ if parseErr != nil {
+ return nil, parseErr
+ }
+ transport.Proxy = http.ProxyURL(proxyURL)
+ } else {
+ transport.Proxy = http.ProxyFromEnvironment
+ }
+
+ return transport, nil
+}
diff --git a/go.mod b/go.mod
index cc4275f26..aba42e2c8 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module github.com/projectdiscovery/httpx
-go 1.26
+go 1.26.1
require (
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
@@ -31,8 +31,7 @@ require (
github.com/projectdiscovery/mapcidr v1.1.97
github.com/projectdiscovery/networkpolicy v0.1.40
github.com/projectdiscovery/ratelimit v0.0.88
- github.com/projectdiscovery/rawhttp v0.1.90
- github.com/projectdiscovery/retryablehttp-go v1.3.15
+ github.com/projectdiscovery/retryablehttp-go v1.3.15 // indirect
github.com/projectdiscovery/tlsx v1.2.2
github.com/projectdiscovery/useragent v0.0.108
github.com/projectdiscovery/utils v0.11.1
@@ -57,12 +56,20 @@ require (
github.com/happyhackingspace/dit v0.0.27
github.com/lib/pq v1.12.3
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193
+ github.com/projectdiscovery/reqx v0.0.0-20260508172251-6205af3493df
github.com/seh-msft/burpxml v1.0.1
github.com/weppos/publicsuffix-go v0.50.3
go.mongodb.org/mongo-driver v1.17.9
gopkg.in/yaml.v3 v3.0.1
)
+require (
+ github.com/quic-go/qpack v0.6.0 // indirect
+ github.com/quic-go/quic-go v0.59.0 // indirect
+ github.com/valyala/bytebufferpool v1.0.0 // indirect
+ github.com/valyala/fasthttp v1.69.0 // indirect
+)
+
require (
aead.dev/minisign v0.2.0 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
@@ -181,7 +188,7 @@ require (
golang.org/x/oauth2 v0.34.0 // indirect
golang.org/x/sync v0.21.0 // indirect
golang.org/x/term v0.44.0 // indirect
- golang.org/x/time v0.14.0 // indirect
+ golang.org/x/time v0.15.0 // indirect
golang.org/x/tools v0.45.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)
diff --git a/go.sum b/go.sum
index be19c38ba..f66561ef4 100644
--- a/go.sum
+++ b/go.sum
@@ -357,14 +357,12 @@ github.com/projectdiscovery/networkpolicy v0.1.40 h1:kYin4u1/dgb0nuz5fE1bz4Q0Zh6
github.com/projectdiscovery/networkpolicy v0.1.40/go.mod h1:9ULLaMbdv9UnT0C5rmuK4nIwYs0o776xMnkPUb8TtaE=
github.com/projectdiscovery/ratelimit v0.0.88 h1:AcurW9aLRzlEyPe9kSjnOpr3XzLMWTpiWAlW/w73ALU=
github.com/projectdiscovery/ratelimit v0.0.88/go.mod h1:CU1s+68UUG2mctSl2wi32/DHLJA6TMg+4rxgP59LfVk=
-github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw=
-github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y=
+github.com/projectdiscovery/reqx v0.0.0-20260508172251-6205af3493df h1:hQB3kGci6kFK8UtTYhgK70UOR4wrkmkVwl/RSPcmXyw=
+github.com/projectdiscovery/reqx v0.0.0-20260508172251-6205af3493df/go.mod h1:hK9i+LySFhIu8tgt1j8BON9pJr/46avEJ5En7MgTRN0=
github.com/projectdiscovery/retryabledns v1.0.115 h1:RKV63FNIznFHUoawg/1hs53pVH3wqPtFhwstCuxVSoA=
github.com/projectdiscovery/retryabledns v1.0.115/go.mod h1:+fEMWoPigw+M0lGNKY7AZ+g8FIgj+4sONjsinMmeL3k=
github.com/projectdiscovery/retryablehttp-go v1.3.15 h1:qhJzaWWRras9Il66HbWU0DJ35clFJoz/ktQvks1ogGU=
github.com/projectdiscovery/retryablehttp-go v1.3.15/go.mod h1:s0azLAqAbcVCjHI9t0ezPhamevYGM1eoOvFkn4QmpZ8=
-github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
-github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiBLvK+SE=
github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4=
github.com/projectdiscovery/useragent v0.0.108 h1:fb+uLuFJvC+MHZjCtxQJxtvp1X6A8n98CUGPyFcg3NE=
@@ -374,6 +372,10 @@ github.com/projectdiscovery/utils v0.11.1/go.mod h1:yktGrHGk2CTjNiccXovnvGrLHX9s
github.com/projectdiscovery/wappalyzergo v0.2.85 h1:RIJwaR2ViAoHSYBE/1ytNjvzXV+mfFDeTjr9eXAgaCU=
github.com/projectdiscovery/wappalyzergo v0.2.85/go.mod h1:gMH0o5lBp65sKMwHx/tuUdOtW2RjodC6Ti+9QDsYMkY=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
+github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
+github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo=
github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -452,6 +454,10 @@ github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZ
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI=
+github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw=
github.com/vulncheck-oss/go-exploit v1.51.0 h1:HTmJ4Q94tbEDPb35mQZn6qMg4rT+Sw9n+L7g3Pjr+3o=
github.com/vulncheck-oss/go-exploit v1.51.0/go.mod h1:J28w0dLnA6DnCrnBm9Sbt6smX8lvztnnN2wCXy7No6c=
github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
@@ -512,6 +518,8 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
+go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc=
@@ -695,8 +703,8 @@ golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go
index fbad91abd..15815bb4e 100644
--- a/internal/pdcp/writer.go
+++ b/internal/pdcp/writer.go
@@ -14,7 +14,6 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/httpx/runner"
- "github.com/projectdiscovery/retryablehttp-go"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
"github.com/projectdiscovery/utils/conversion"
"github.com/projectdiscovery/utils/env"
@@ -46,7 +45,7 @@ var (
type UploadWriter struct {
creds *pdcpauth.PDCPCredentials
uploadURL *url.URL
- client *retryablehttp.Client
+ client *http.Client
done chan struct{}
data chan runner.Result
assetGroupID string
@@ -78,10 +77,7 @@ func NewUploadWriterCallback(ctx context.Context, creds *pdcpauth.PDCPCredential
u.uploadURL = tmp.URL
// create http client
- opts := retryablehttp.DefaultOptionsSingle
- opts.NoAdjustTimeout = true
- opts.Timeout = time.Duration(3) * time.Minute
- u.client = retryablehttp.NewClient(opts)
+ u.client = &http.Client{Timeout: time.Duration(3) * time.Minute}
// start auto commit
// upload every 1 minute or when buffer is full
go u.autoCommit(ctx)
@@ -226,29 +222,32 @@ func (u *UploadWriter) upload(data []byte) error {
// getRequest returns a new request for upload
// if scanID is not provided create new scan by uploading the data
// if scanID is provided append the data to existing scan
-func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) {
- var method, url string
-
- if u.assetGroupID == "" {
+func (u *UploadWriter) getRequest(bin []byte) (*http.Request, error) {
+ var method string
+ isUpload := u.assetGroupID == ""
+ if isUpload {
u.uploadURL.Path = uploadEndpoint
method = http.MethodPost
- url = u.uploadURL.String()
} else {
u.uploadURL.Path = fmt.Sprintf(appendEndpoint, u.assetGroupID)
method = http.MethodPatch
- url = u.uploadURL.String()
}
- req, err := retryablehttp.NewRequest(method, url, bytes.NewReader(bin))
+
+ // add pdtm meta params (already url-encoded)
+ values, err := url.ParseQuery(updateutils.GetpdtmParams(runner.Version))
if err != nil {
- return nil, errkit.Wrap(err, "could not create cloud upload request")
+ return nil, errkit.Wrap(err, "could not parse pdtm params")
}
- // add pdtm meta params
- req.Params.Merge(updateutils.GetpdtmParams(runner.Version))
// if it is upload endpoint also include name if it exists
- if u.assetGroupName != "" && req.Path == uploadEndpoint {
- req.Params.Add("name", u.assetGroupName)
+ if u.assetGroupName != "" && isUpload {
+ values.Add("name", u.assetGroupName)
+ }
+ u.uploadURL.RawQuery = values.Encode()
+
+ req, err := http.NewRequest(method, u.uploadURL.String(), bytes.NewReader(bin))
+ if err != nil {
+ return nil, errkit.Wrap(err, "could not create cloud upload request")
}
- req.Update()
req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey)
if u.TeamID != "" {
diff --git a/runner/runner.go b/runner/runner.go
index eed5d886b..b0d151a54 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -51,7 +51,6 @@ import (
"github.com/projectdiscovery/clistats"
"github.com/projectdiscovery/goconfig"
"github.com/projectdiscovery/httpx/common/hashes"
- "github.com/projectdiscovery/retryablehttp-go"
sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
@@ -68,7 +67,6 @@ import (
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/httpx/common/stringz"
"github.com/projectdiscovery/mapcidr"
- "github.com/projectdiscovery/rawhttp"
converstionutil "github.com/projectdiscovery/utils/conversion"
errkit "github.com/projectdiscovery/utils/errkit"
fileutil "github.com/projectdiscovery/utils/file"
@@ -285,16 +283,6 @@ func New(options *Options) (*Runner, error) {
options.RequestBody = rrBody
}
- // disable automatic host header for rawhttp if manually specified
- // as it can be malformed the best approach is to remove spaces and check for lowercase "host" word
- if options.Unsafe {
- for name := range runner.hp.CustomHeaders {
- nameLower := strings.TrimSpace(strings.ToLower(name))
- if strings.HasPrefix(nameLower, "host") {
- rawhttp.AutomaticHostHeader(false)
- }
- }
- }
if strings.EqualFold(options.Methods, "all") {
scanopts.Methods = pdhttputil.AllHTTPMethods()
} else if options.Methods != "" {
@@ -1865,7 +1853,7 @@ retry:
gologger.Debug().Msgf("failed to merge paths of url %v and %v", URL.String(), scanopts.RequestURI)
}
var (
- req *retryablehttp.Request
+ req *http.Request
ctx context.Context
)
if target.CustomIP != "" {
@@ -1903,7 +1891,7 @@ retry:
if r.authProvider != nil {
if strategies := r.authProvider.LookupURLX(URL); len(strategies) > 0 {
for _, strategy := range strategies {
- strategy.ApplyOnRR(req)
+ strategy.Apply(req)
}
}
}
@@ -1912,6 +1900,10 @@ retry:
if scanopts.RequestBody != "" {
req.ContentLength = int64(len(scanopts.RequestBody))
req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody))
+ // GetBody enables the body to be rewound for retries and 307/308 redirects
+ req.GetBody = func() (io.ReadCloser, error) {
+ return io.NopCloser(strings.NewReader(scanopts.RequestBody)), nil
+ }
} else {
req.ContentLength = 0
req.Body = nil
@@ -1930,7 +1922,7 @@ retry:
var requestDump []byte
if scanopts.Unsafe {
var errDump error
- requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
+ requestDump, errDump = httpx.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body)
if errDump != nil {
return Result{URL: URL.String(), Input: origInput, Err: errDump}
}
@@ -1941,7 +1933,7 @@ retry:
req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody))
}
var errDump error
- requestDump, errDump = httputil.DumpRequestOut(req.Request, true)
+ requestDump, errDump = httputil.DumpRequestOut(req, true)
if errDump != nil {
return Result{URL: URL.String(), Input: origInput, Err: errDump}
}
@@ -1953,7 +1945,7 @@ retry:
}
}
// fix the final output url
- fullURL := req.String()
+ fullURL := req.URL.String()
if parsedURL, errParse := r.parseURL(fullURL); errParse != nil {
return Result{URL: URL.String(), Input: origInput, Err: errParse}
} else {
@@ -2707,7 +2699,7 @@ retry:
result.Domains = resp.BodyDomains.Domains
}
if r.options.Trace {
- result.Trace = req.TraceInfo
+ result.Trace = httpx.GetTraceInfo(req)
}
return result
}
@@ -2742,11 +2734,11 @@ func calculatePerceptionHash(screenshotBytes []byte) (uint64, error) {
return pHash.GetHash(), nil
}
-func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, finalURL string, defaultProbe bool) (string, string, string, []byte, string, error) {
+func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *http.Request, currentResp []byte, finalURL string, defaultProbe bool) (string, string, string, []byte, string, error) {
// Check if current URI is ending with .ico => use current body without additional requests
- if path.Ext(req.Path) == ".ico" {
+ if path.Ext(req.URL.Path) == ".ico" {
mmh3, md5h, err := r.calculateFaviconHashWithRaw(currentResp)
- return mmh3, md5h, req.Path, currentResp, req.String(), err
+ return mmh3, md5h, req.URL.Path, currentResp, req.URL.String(), err
}
// Parse HTML: collect hrefs + optional
@@ -2761,7 +2753,7 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
}
// Determine base URL: prefer finalURL (redirect target) then apply
- baseNet, _ := url.Parse(req.String())
+ baseNet, _ := url.Parse(req.URL.String())
if finalURL != "" {
if u, err := url.Parse(finalURL); err == nil {
baseNet = u
@@ -2819,7 +2811,8 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
continue
}
- clone.SetURL(resolvedURL)
+ resolvedURL.Update()
+ clone.URL = resolvedURL.URL
// Update Host header to match resolved URL host (important after redirects)
if resolvedURL.Host != "" && resolvedURL.Host != clone.Host {
clone.Host = resolvedURL.Host
@@ -2834,7 +2827,8 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
if err2 != nil {
continue
}
- clone.SetURL(rootResolvedURL)
+ rootResolvedURL.Update()
+ clone.URL = rootResolvedURL.URL
if respFav2, err3 := hp.Do(clone, httpx.UnsafeOptions{}); err3 == nil && len(respFav2.Data) > 0 {
respFav = respFav2
} else {
@@ -2853,7 +2847,7 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
faviconMMH3 = mmh3
faviconMD5 = md5h
faviconPath = raw
- faviconURL = clone.String()
+ faviconURL = clone.URL.String()
faviconData = respFav.Data
gologger.Debug().Msgf("favicon resolved url=%s raw_href=%s size=%d bytes", faviconURL, faviconPath, len(faviconData))
break
diff --git a/runner/types.go b/runner/types.go
index 1eaa3608e..56446a64d 100644
--- a/runner/types.go
+++ b/runner/types.go
@@ -12,7 +12,6 @@ import (
mapstructure "github.com/go-viper/mapstructure/v2"
"github.com/projectdiscovery/dsl"
"github.com/projectdiscovery/gologger"
- "github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
mapsutil "github.com/projectdiscovery/utils/maps"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
@@ -101,7 +100,7 @@ type Result struct {
RequestRaw []byte `json:"-" csv:"-" md:"-" mapstructure:"-"`
Response *httpx.Response `json:"-" csv:"-" md:"-" mapstructure:"-"`
FaviconData []byte `json:"-" csv:"-" md:"-" mapstructure:"-"`
- Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" md:"-" mapstructure:"trace"`
+ Trace *httpx.TraceInfo `json:"trace,omitempty" csv:"-" md:"-" mapstructure:"trace"`
FileNameHash string `json:"-" csv:"-" md:"-" mapstructure:"-"`
CPE []CPEInfo `json:"cpe,omitempty" csv:"cpe" md:"cpe" mapstructure:"cpe"`
WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress" md:"wordpress" mapstructure:"wordpress"`