.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go: [ '1.20.x', '1.21.x' ]
+        go: [ '1.22.x', '1.23.x' ]
         os: [ ubuntu-latest ]
     runs-on: ${{ matrix.os }}
     steps:
@@ -30,4 +30,4 @@ jobs:
         with:
           go-version: ${{ matrix.go }}
       - name: Test
-        run: go test ./... -coverprofile=coverage.txt
+        run: go test ./... -coverprofile=coverage.txt

README.md

@@ -35,7 +35,7 @@ Full documentation is available on the official website: https://req.cool.
 
 **Install**
 
-You first need [Go](https://go.dev/) installed (version 1.20+ is required), then you can use the below Go command to install req:
+You first need [Go](https://go.dev/) installed (version 1.22+ is required), then you can use the below Go command to install req:
 
 ``` sh
 go get github.com/imroc/req/v3
@@ -501,7 +501,6 @@ If you have questions, feel free to reach out to us in the following ways:
 
 * [Github Discussion](https://github.com/imroc/req/discussions)
 * [Slack](https://imroc-req.slack.com/archives/C03UFPGSNC8) | [Join](https://slack.req.cool/)
-* QQ Group (Chinese): 621411351 - <a href="https://qm.qq.com/cgi-bin/qm/qr?k=P8vOMuNytG-hhtPlgijwW6orJV765OAO&jump_from=webapi"><img src="https://pub.idqqimg.com/wpa/images/group.png"></a>
 
 ## Sponsors
 

SECURITY.md

@@ -0,0 +1,9 @@
+# Security Policy
+
+## Supported Versions
+
+req version >= `v3.43.x`
+
+## Reporting a Vulnerability
+
+Email: roc@imroc.cc

client.go

@@ -59,6 +59,7 @@ type Client struct {
 	jsonUnmarshal           func(data []byte, v interface{}) error
 	xmlMarshal              func(v interface{}) ([]byte, error)
 	xmlUnmarshal            func(data []byte, v interface{}) error
+	multipartBoundaryFunc   func() string
 	outputDirectory         string
 	scheme                  string
 	log                     Logger
@@ -239,6 +240,17 @@ func (c *Client) SetCommonFormData(data map[string]string) *Client {
 	return c
 }
 
+// SetMultipartBoundaryFunc overrides the default function used to generate
+// boundary delimiters for "multipart/form-data" requests with a customized one,
+// which returns a boundary delimiter (without the two leading hyphens).
+//
+// Boundary delimiter may only contain certain ASCII characters, and must be
+// non-empty and at most 70 bytes long (see RFC 2046, Section 5.1.1).
+func (c *Client) SetMultipartBoundaryFunc(fn func() string) *Client {
+	c.multipartBoundaryFunc = fn
+	return c
+}
+
 // SetBaseURL set the default base URL, will be used if request URL is
 // a relative URL.
 func (c *Client) SetBaseURL(u string) *Client {
@@ -278,7 +290,6 @@ func (c *Client) appendRootCertData(data []byte) {
 		config.RootCAs = x509.NewCertPool()
 	}
 	config.RootCAs.AppendCertsFromPEM(data)
-	return
 }
 
 // SetRootCertFromString set root certificates from string.
@@ -310,20 +321,10 @@ func (c *Client) GetTLSClientConfig() *tls.Config {
 	return c.TLSClientConfig
 }
 
-func (c *Client) defaultCheckRedirect(req *http.Request, via []*http.Request) error {
-	if len(via) >= 10 {
-		return errors.New("stopped after 10 redirects")
-	}
-	if c.DebugLog {
-		c.log.Debugf("<redirect> %s %s", req.Method, req.URL.String())
-	}
-	return nil
-}
-
 // SetRedirectPolicy set the RedirectPolicy which controls the behavior of receiving redirect
 // responses (usually responses with 301 and 302 status code), see the predefined
-// AllowedDomainRedirectPolicy, AllowedHostRedirectPolicy, MaxRedirectPolicy, NoRedirectPolicy,
-// SameDomainRedirectPolicy and SameHostRedirectPolicy.
+// AllowedDomainRedirectPolicy, AllowedHostRedirectPolicy, DefaultRedirectPolicy, MaxRedirectPolicy,
+// NoRedirectPolicy, SameDomainRedirectPolicy and SameHostRedirectPolicy.
 func (c *Client) SetRedirectPolicy(policies ...RedirectPolicy) *Client {
 	if len(policies) == 0 {
 		return c
@@ -381,6 +382,18 @@ func (c *Client) EnableCompression() *Client {
 	return c
 }
 
+// EnableAutoDecompress enables the automatic decompression (disabled by default).
+func (c *Client) EnableAutoDecompress() *Client {
+	c.Transport.AutoDecompression = true
+	return c
+}
+
+// DisableAutoDecompress disables the automatic decompression (disabled by default).
+func (c *Client) DisableAutoDecompress() *Client {
+	c.Transport.AutoDecompression = false
+	return c
+}
+
 // SetTLSClientConfig set the TLS client config. Be careful! Usually
 // you don't need this, you can directly set the tls configuration with
 // methods like EnableInsecureSkipVerify, SetCerts etc. Or you can call
@@ -1187,11 +1200,21 @@ func (c *Client) SetTLSFingerprint(clientHelloID utls.ClientHelloID) *Client {
 			colonPos = len(addr)
 		}
 		hostname := addr[:colonPos]
+		tlsConfig := c.GetTLSClientConfig()
 		utlsConfig := &utls.Config{
-			ServerName:         hostname,
-			RootCAs:            c.GetTLSClientConfig().RootCAs,
-			NextProtos:         c.GetTLSClientConfig().NextProtos,
-			InsecureSkipVerify: c.GetTLSClientConfig().InsecureSkipVerify,
+			ServerName:                  hostname,
+			Rand:                        tlsConfig.Rand,
+			Time:                        tlsConfig.Time,
+			RootCAs:                     tlsConfig.RootCAs,
+			NextProtos:                  tlsConfig.NextProtos,
+			ClientCAs:                   tlsConfig.ClientCAs,
+			InsecureSkipVerify:          tlsConfig.InsecureSkipVerify,
+			CipherSuites:                tlsConfig.CipherSuites,
+			SessionTicketsDisabled:      tlsConfig.SessionTicketsDisabled,
+			MinVersion:                  tlsConfig.MinVersion,
+			MaxVersion:                  tlsConfig.MaxVersion,
+			DynamicRecordSizingDisabled: tlsConfig.DynamicRecordSizingDisabled,
+			KeyLogWriter:                tlsConfig.KeyLogWriter,
 		}
 		uconn := &uTLSConn{utls.UClient(plainConn, utlsConfig, clientHelloID)}
 		err = uconn.HandshakeContext(ctx)
@@ -1532,7 +1555,7 @@ func C() *Client {
 		xmlUnmarshal:          xml.Unmarshal,
 		cookiejarFactory:      memoryCookieJarFactory,
 	}
-	httpClient.CheckRedirect = c.defaultCheckRedirect
+	c.SetRedirectPolicy(DefaultRedirectPolicy())
 	c.initCookieJar()
 
 	c.initTransport()

client_impersonate.go

@@ -1,10 +1,56 @@
 package req
 
 import (
+	"crypto/rand"
+	"encoding/binary"
+	"math/big"
+	"strconv"
+	"strings"
+
 	"github.com/imroc/req/v3/http2"
 	utls "github.com/refraction-networking/utls"
 )
 
+// Identical for both Blink-based browsers (Chrome, Chromium, etc.) and WebKit-based browsers (Safari, etc.)
+// Blink implementation: https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/platform/network/form_data_encoder.cc;drc=1d694679493c7b2f7b9df00e967b4f8699321093;l=130
+// WebKit implementation: https://github.com/WebKit/WebKit/blob/47eea119fe9462721e5cc75527a4280c6d5f5214/Source/WebCore/platform/network/FormDataBuilder.cpp#L120
+func webkitMultipartBoundaryFunc() string {
+	const letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789AB"
+
+	sb := strings.Builder{}
+	sb.WriteString("----WebKitFormBoundary")
+
+	for i := 0; i < 16; i++ {
+		index, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters)-1)))
+		if err != nil {
+			panic(err)
+		}
+
+		sb.WriteByte(letters[index.Int64()])
+	}
+
+	return sb.String()
+}
+
+// Firefox implementation: https://searchfox.org/mozilla-central/source/dom/html/HTMLFormSubmission.cpp#355
+func firefoxMultipartBoundaryFunc() string {
+	sb := strings.Builder{}
+	sb.WriteString("-------------------------")
+
+	for i := 0; i < 3; i++ {
+		var b [8]byte
+		if _, err := rand.Read(b[:]); err != nil {
+			panic(err)
+		}
+		u32 := binary.LittleEndian.Uint32(b[:])
+		s := strconv.FormatUint(uint64(u32), 10)
+
+		sb.WriteString(s)
+	}
+
+	return sb.String()
+}
+
 var (
 	chromeHttp2Settings = []http2.Setting{
 		{
@@ -59,35 +105,37 @@ var (
 	chromeHeaders = map[string]string{
 		"pragma":                    "no-cache",
 		"cache-control":             "no-cache",
-		"sec-ch-ua":                 `"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"`,
+		"sec-ch-ua":                 `"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"`,
 		"sec-ch-ua-mobile":          "?0",
 		"sec-ch-ua-platform":        `"macOS"`,
 		"upgrade-insecure-requests": "1",
-		"user-agent":                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
-		"accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+		"user-agent":                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+		"accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
 		"sec-fetch-site":            "none",
 		"sec-fetch-mode":            "navigate",
 		"sec-fetch-user":            "?1",
 		"sec-fetch-dest":            "document",
-		"accept-language":           "zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7,it;q=0.6",
+		"accept-language":           "zh-CN,zh;q=0.9",
 	}
+
 	chromeHeaderPriority = http2.PriorityParam{
 		StreamDep: 0,
 		Exclusive: true,
 		Weight:    255,
 	}
 )
 
-// ImpersonateChrome impersonates Chrome browser (version 109).
+// ImpersonateChrome impersonates Chrome browser (version 120).
 func (c *Client) ImpersonateChrome() *Client {
 	c.
-		SetTLSFingerprint(utls.HelloChrome_106_Shuffle). // Chrome 106~109 shares the same tls fingerprint.
+		SetTLSFingerprint(utls.HelloChrome_120).
 		SetHTTP2SettingsFrame(chromeHttp2Settings...).
 		SetHTTP2ConnectionFlow(15663105).
 		SetCommonPseudoHeaderOder(chromePseudoHeaderOrder...).
 		SetCommonHeaderOrder(chromeHeaderOrder...).
 		SetCommonHeaders(chromeHeaders).
-		SetHTTP2HeaderPriority(chromeHeaderPriority)
+		SetHTTP2HeaderPriority(chromeHeaderPriority).
+		SetMultipartBoundaryFunc(webkitMultipartBoundaryFunc)
 	return c
 }
 
@@ -106,6 +154,7 @@ var (
 			Val: 16384,
 		},
 	}
+
 	firefoxPriorityFrames = []http2.PriorityFrame{
 		{
 			StreamID: 3,
@@ -156,12 +205,14 @@ var (
 			},
 		},
 	}
+
 	firefoxPseudoHeaderOrder = []string{
 		":method",
 		":path",
 		":authority",
 		":scheme",
 	}
+
 	firefoxHeaderOrder = []string{
 		"user-agent",
 		"accept",
@@ -176,8 +227,9 @@ var (
 		"sec-fetch-user",
 		"te",
 	}
+
 	firefoxHeaders = map[string]string{
-		"user-agent":                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:105.0) Gecko/20100101 Firefox/105.0",
+		"user-agent":                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0",
 		"accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
 		"accept-language":           "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
 		"upgrade-insecure-requests": "1",
@@ -187,24 +239,26 @@ var (
 		"sec-fetch-user":            "?1",
 		//"te":                        "trailers",
 	}
+
 	firefoxHeaderPriority = http2.PriorityParam{
 		StreamDep: 13,
 		Exclusive: false,
 		Weight:    41,
 	}
 )
 
-// ImpersonateFirefox impersonates Firefox browser (version 105).
+// ImpersonateFirefox impersonates Firefox browser (version 120).
 func (c *Client) ImpersonateFirefox() *Client {
 	c.
-		SetTLSFingerprint(utls.HelloFirefox_105).
+		SetTLSFingerprint(utls.HelloFirefox_120).
 		SetHTTP2SettingsFrame(firefoxHttp2Settings...).
 		SetHTTP2ConnectionFlow(12517377).
 		SetHTTP2PriorityFrames(firefoxPriorityFrames...).
 		SetCommonPseudoHeaderOder(firefoxPseudoHeaderOrder...).
 		SetCommonHeaderOrder(firefoxHeaderOrder...).
 		SetCommonHeaders(firefoxHeaders).
-		SetHTTP2HeaderPriority(firefoxHeaderPriority)
+		SetHTTP2HeaderPriority(firefoxHeaderPriority).
+		SetMultipartBoundaryFunc(firefoxMultipartBoundaryFunc)
 	return c
 }
 
@@ -255,7 +309,7 @@ var (
 	}
 )
 
-// ImpersonateSafari impersonates Safari browser (version 16).
+// ImpersonateSafari impersonates Safari browser (version 16.6).
 func (c *Client) ImpersonateSafari() *Client {
 	c.
 		SetTLSFingerprint(utls.HelloSafari_16_0).
@@ -264,6 +318,7 @@ func (c *Client) ImpersonateSafari() *Client {
 		SetCommonPseudoHeaderOder(safariPseudoHeaderOrder...).
 		SetCommonHeaderOrder(safariHeaderOrder...).
 		SetCommonHeaders(safariHeaders).
-		SetHTTP2HeaderPriority(safariHeaderPriority)
+		SetHTTP2HeaderPriority(safariHeaderPriority).
+		SetMultipartBoundaryFunc(webkitMultipartBoundaryFunc)
 	return c
 }