hoelee/curl-impersonate
1
0
Fork 0
mirror of https://github.com/lwthiker/curl-impersonate.git synced 2025-08-08 04:42:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Impersonate Chrome 110 (#148)

Browse Source 
Add support for impersonating Chrome 110. Chrome 110 comes with TLS
extension permutation enabled by default. We mimic this behavior in libcurl with
the new CURLOPT_SSL_PERMUTE_EXTENSIONS option, which enables the
corresponding flag in BoringSSL.

---------

Co-authored-by: Johann Saunier <saunier.johann@gmail.com>
This commit is contained in:
lwthiker
2023-02-25 11:29:14 +02:00
committed by GitHub
parent b2b1ea0f8e
commit 9d05a81030
7 changed files with 367 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@@ -37,6 +37,7 @@ The following browsers can be impersonated.
| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 101 | 101.0.4951.67 | Windows 10 | `chrome101` | [curl_chrome101](chrome/curl_chrome101) |
| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 104 | 104.0.5112.81 | Windows 10 | `chrome104` | [curl_chrome104](chrome/curl_chrome104) |
| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 107 | 107.0.5304.107 | Windows 10 | `chrome107` | [curl_chrome107](chrome/curl_chrome107) |
| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 110 | 110.0.5481.177 | Windows 10 | `chrome110` | [curl_chrome110](chrome/curl_chrome110) |
| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 99 | 99.0.4844.73 | Android 12 | `chrome99_android` | [curl_chrome99_android](chrome/curl_chrome99_android) |
| ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") | 99 | 99.0.1150.30 | Windows 10 | `edge99` | [curl_edge99](chrome/curl_edge99) |
| ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") | 101 | 101.0.1210.47 | Windows 10 | `edge101` | [curl_edge101](chrome/curl_edge101) |
@@ -123,7 +124,7 @@ Calling the above function sets the following libcurl options:
* `CURLOPT_HTTPBASEHEADER`, if `default_headers` is non-zero (this is a non-standard HTTP option created for this project).
* `CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER`, `CURLOPT_HTTP2_NO_SERVER_PUSH` (non-standard HTTP/2 options created for this project).
* `CURLOPT_SSL_ENABLE_ALPS`, `CURLOPT_SSL_SIG_HASH_ALGS`, `CURLOPT_SSL_CERT_COMPRESSION`, `CURLOPT_SSL_ENABLE_TICKET` (non-standard TLS options created for this project).
* `CURLOPT_SSL_PERMUTE_EXTENSIONS` (non-standard TLS options created for this project).
Note that if you call `curl_easy_setopt()` later with one of the above it will override the options set by `curl_easy_impersonate()`.
### Using CURL_IMPERSONATE env var

10
browsers.json
View File

@@ -50,6 +50,16 @@
"binary": "curl-impersonate-chrome",
"wrapper_script": "curl_chrome107"
},
{
"name": "chrome110",
"browser": {
"name": "chrome",
"version": "110.0.5481.177",
"os": "win10"
},
"binary": "curl-impersonate-chrome",
"wrapper_script": "curl_chrome110"
},
{
"name": "chrome99_android",
"browser": {

26
chrome/curl_chrome110 Executable file
View File

@@ -0,0 +1,26 @@
#!/usr/bin/env bash
# Find the directory of this script
dir=${0%/*}
# The list of ciphers can be obtained by looking at the Client Hello message in
# Wireshark, then converting it using this reference
# https://wiki.mozilla.org/Security/Cipher_Suites
"$dir/curl-impersonate-chrome" \
--ciphers TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA \
-H 'sec-ch-ua: "Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110"' \
-H 'sec-ch-ua-mobile: ?0' \
-H 'sec-ch-ua-platform: "Windows"' \
-H 'Upgrade-Insecure-Requests: 1' \
-H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36' \
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \
-H 'Sec-Fetch-Site: none' \
-H 'Sec-Fetch-Mode: navigate' \
-H 'Sec-Fetch-User: ?1' \
-H 'Sec-Fetch-Dest: document' \
-H 'Accept-Encoding: gzip, deflate, br' \
-H 'Accept-Language: en-US,en;q=0.9' \
--http2 --http2-no-server-push --false-start --compressed \
--tlsv1.2 --no-npn --alps --tls-permute-extensions \
--cert-compression brotli \
"$@"

189
chrome/patches/curl-impersonate.patch
View File

@@ -82,10 +82,10 @@ index aaf2b8a43..ccfa52985 100644
echo "curl was built with static libraries disabled" >&2
exit 1
diff --git a/include/curl/curl.h b/include/curl/curl.h
index b00648e79..963d68382 100644
index b00648e79..a8edd2e2d 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -2143,6 +2143,44 @@ typedef enum {
@@ -2143,6 +2143,50 @@ typedef enum {
/* set the SSH host key callback custom pointer */
CURLOPT(CURLOPT_SSH_HOSTKEYDATA, CURLOPTTYPE_CBPOINT, 317),
@@ -126,6 +126,12 @@ index b00648e79..963d68382 100644
+ * Disable HTTP2 server push in the HTTP2 SETTINGS.
+ */
+ CURLOPT(CURLOPT_HTTP2_NO_SERVER_PUSH, CURLOPTTYPE_LONG, 324),
+
+ /*
+ * curl-impersonate: Whether to enable Boringssl permute extensions
+ * See https://commondatastorage.googleapis.com/chromium-boringssl-docs/ssl.h.html#SSL_set_permute_extensions.
+ */
+ CURLOPT(CURLOPT_SSL_PERMUTE_EXTENSIONS, CURLOPTTYPE_LONG, 325),
+
CURLOPT_LASTENTRY /* the last unused */
} CURLoption;
@@ -244,7 +250,7 @@ index 9bd8e324b..bfd5e90e2 100644
inet_pton.c \
krb5.c \
diff --git a/lib/easy.c b/lib/easy.c
index 704a59df6..1d0251b8b 100644
index 704a59df6..f486362c0 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -81,6 +81,8 @@
@@ -256,7 +262,7 @@ index 704a59df6..1d0251b8b 100644
/* The last 3 #include files should be in this order */
#include "curl_printf.h"
@@ -332,6 +334,128 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name,
@@ -332,6 +334,134 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name,
return rc;
}
@@ -333,6 +339,12 @@ index 704a59df6..1d0251b8b 100644
+ if(ret)
+ return ret;
+
+ if(opts->tls_permute_extensions) {
+ ret = curl_easy_setopt(data, CURLOPT_SSL_PERMUTE_EXTENSIONS, 1);
+ if(ret)
+ return ret;
+ }
+
+ if(opts->cert_compression) {
+ ret = curl_easy_setopt(data,
+ CURLOPT_SSL_CERT_COMPRESSION,
@@ -385,7 +397,7 @@ index 704a59df6..1d0251b8b 100644
/*
* curl_easy_init() is the external interface to alloc, setup and init an
* easy handle that is returned. If anything goes wrong, NULL is returned.
@@ -340,6 +464,8 @@ struct Curl_easy *curl_easy_init(void)
@@ -340,6 +470,8 @@ struct Curl_easy *curl_easy_init(void)
{
CURLcode result;
struct Curl_easy *data;
@@ -394,7 +406,7 @@ index 704a59df6..1d0251b8b 100644
/* Make sure we inited the global SSL stuff */
global_init_lock();
@@ -362,6 +488,29 @@ struct Curl_easy *curl_easy_init(void)
@@ -362,6 +494,29 @@ struct Curl_easy *curl_easy_init(void)
return NULL;
}
@@ -424,7 +436,7 @@ index 704a59df6..1d0251b8b 100644
return data;
}
@@ -936,6 +1085,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
@@ -936,6 +1091,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
outcurl->state.referer_alloc = TRUE;
}
@@ -438,7 +450,7 @@ index 704a59df6..1d0251b8b 100644
/* Reinitialize an SSL engine for the new handle
* note: the engine name has already been copied by dupset */
if(outcurl->set.str[STRING_SSL_ENGINE]) {
@@ -1025,6 +1181,9 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
@@ -1025,6 +1187,9 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
*/
void curl_easy_reset(struct Curl_easy *data)
{
@@ -448,7 +460,7 @@ index 704a59df6..1d0251b8b 100644
Curl_free_request_state(data);
/* zero out UserDefined data: */
@@ -1049,6 +1208,23 @@ void curl_easy_reset(struct Curl_easy *data)
@@ -1049,6 +1214,23 @@ void curl_easy_reset(struct Curl_easy *data)
#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH)
Curl_http_auth_cleanup_digest(data);
#endif
@@ -473,7 +485,7 @@ index 704a59df6..1d0251b8b 100644
/*
diff --git a/lib/easyoptions.c b/lib/easyoptions.c
index c99f135ff..4a68332b9 100644
index c99f135ff..6b63fe4ae 100644
--- a/lib/easyoptions.c
+++ b/lib/easyoptions.c
@@ -130,8 +130,12 @@ struct curl_easyoption Curl_easyopts[] = {
@@ -489,7 +501,7 @@ index c99f135ff..4a68332b9 100644
{"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0},
{"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0},
{"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0},
@@ -297,15 +301,19 @@ struct curl_easyoption Curl_easyopts[] = {
@@ -297,18 +301,23 @@ struct curl_easyoption Curl_easyopts[] = {
{"SSLKEYTYPE", CURLOPT_SSLKEYTYPE, CURLOT_STRING, 0},
{"SSLKEY_BLOB", CURLOPT_SSLKEY_BLOB, CURLOT_BLOB, 0},
{"SSLVERSION", CURLOPT_SSLVERSION, CURLOT_VALUES, 0},
@@ -509,12 +521,16 @@ index c99f135ff..4a68332b9 100644
{"SSL_VERIFYHOST", CURLOPT_SSL_VERIFYHOST, CURLOT_LONG, 0},
{"SSL_VERIFYPEER", CURLOPT_SSL_VERIFYPEER, CURLOT_LONG, 0},
{"SSL_VERIFYSTATUS", CURLOPT_SSL_VERIFYSTATUS, CURLOT_LONG, 0},
@@ -364,6 +372,6 @@ struct curl_easyoption Curl_easyopts[] = {
+ {"SSL_PERMUTE_EXTENSIONS", CURLOPT_SSL_PERMUTE_EXTENSIONS, CURLOT_LONG, 0},
{"STDERR", CURLOPT_STDERR, CURLOT_OBJECT, 0},
{"STREAM_DEPENDS", CURLOPT_STREAM_DEPENDS, CURLOT_OBJECT, 0},
{"STREAM_DEPENDS_E", CURLOPT_STREAM_DEPENDS_E, CURLOT_OBJECT, 0},
@@ -364,6 +373,6 @@ struct curl_easyoption Curl_easyopts[] = {
*/
int Curl_easyopts_check(void)
{
- return ((CURLOPT_LASTENTRY%10000) != (317 + 1));
+ return ((CURLOPT_LASTENTRY%10000) != (324 + 1));
+ return ((CURLOPT_LASTENTRY%10000) != (325 + 1));
}
#endif
diff --git a/lib/h2h3.c b/lib/h2h3.c
@@ -1001,10 +1017,10 @@ index f0390596c..cf9b7a9d5 100644
* Store nghttp2 version info in this buffer.
diff --git a/lib/impersonate.c b/lib/impersonate.c
new file mode 100644
index 000000000..2c8a4d3f9
index 000000000..025bd58ef
--- /dev/null
+++ b/lib/impersonate.c
@@ -0,0 +1,438 @@
@@ -0,0 +1,480 @@
+#include "curl_setup.h"
+
+#include "impersonate.h"
@@ -1212,6 +1228,48 @@ index 000000000..2c8a4d3f9
+ .http2_no_server_push = true
+ },
+ {
+ .target = "chrome110",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "TLS_AES_128_GCM_SHA256,"
+ "TLS_AES_256_GCM_SHA384,"
+ "TLS_CHACHA20_POLY1305_SHA256,"
+ "ECDHE-ECDSA-AES128-GCM-SHA256,"
+ "ECDHE-RSA-AES128-GCM-SHA256,"
+ "ECDHE-ECDSA-AES256-GCM-SHA384,"
+ "ECDHE-RSA-AES256-GCM-SHA384,"
+ "ECDHE-ECDSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-AES128-SHA,"
+ "ECDHE-RSA-AES256-SHA,"
+ "AES128-GCM-SHA256,"
+ "AES256-GCM-SHA384,"
+ "AES128-SHA,"
+ "AES256-SHA",
+ .npn = false,
+ .alpn = true,
+ .alps = true,
+ .tls_permute_extensions = true,
+ .tls_session_ticket = true,
+ .cert_compression = "brotli",
+ .http_headers = {
+ "sec-ch-ua: \"Chromium\";v=\"110\", \"Not A(Brand\";v=\"24\", \"Google Chrome\";v=\"110\"",
+ "sec-ch-ua-mobile: ?0",
+ "sec-ch-ua-platform: \"Windows\"",
+ "Upgrade-Insecure-Requests: 1",
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-User: ?1",
+ "Sec-Fetch-Dest: document",
+ "Accept-Encoding: gzip, deflate, br",
+ "Accept-Language: en-US,en;q=0.9"
+ },
+ .http2_no_server_push = true
+ },
+ {
+ .target = "chrome99_android",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
@@ -1445,10 +1503,10 @@ index 000000000..2c8a4d3f9
+};
diff --git a/lib/impersonate.h b/lib/impersonate.h
new file mode 100644
index 000000000..05245c171
index 000000000..c62991c5a
--- /dev/null
+++ b/lib/impersonate.h
@@ -0,0 +1,44 @@
@@ -0,0 +1,45 @@
+#ifndef HEADER_CURL_IMPERSONATE_H
+#define HEADER_CURL_IMPERSONATE_H
+
@@ -1482,6 +1540,7 @@ index 000000000..05245c171
+ const char *http_headers[IMPERSONATE_MAX_HEADERS];
+ const char *http2_pseudo_headers_order;
+ bool http2_no_server_push;
+ bool tls_permute_extensions;
+ /* Other TLS options will come here in the future once they are
+ * configurable through curl_easy_setopt() */
+};
@@ -1508,7 +1567,7 @@ index e0280447c..dc1fdab68 100644
#ifdef USE_WINSOCK
diff --git a/lib/setopt.c b/lib/setopt.c
index 6b16e1c7c..a6d34ebd7 100644
index 6b16e1c7c..189b54025 100644
--- a/lib/setopt.c
+++ b/lib/setopt.c
@@ -50,6 +50,7 @@
@@ -1571,7 +1630,7 @@ index 6b16e1c7c..a6d34ebd7 100644
#endif
case CURLOPT_IPRESOLVE:
arg = va_arg(param, long);
@@ -2861,6 +2900,19 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
@@ -2861,6 +2900,22 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
case CURLOPT_SSL_ENABLE_ALPN:
data->set.ssl_enable_alpn = (0 != va_arg(param, long)) ? TRUE : FALSE;
break;
@@ -1581,6 +1640,9 @@ index 6b16e1c7c..a6d34ebd7 100644
+ case CURLOPT_SSL_ENABLE_TICKET:
+ data->set.ssl_enable_ticket = (0 != va_arg(param, long)) ? TRUE : FALSE;
+ break;
+ case CURLOPT_SSL_PERMUTE_EXTENSIONS:
+ data->set.ssl_permute_extensions = (0 != va_arg(param, long)) ? TRUE : FALSE;
+ break;
+ case CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER:
+ result = Curl_setstropt(&data->set.str[STRING_HTTP2_PSEUDO_HEADERS_ORDER],
+ va_arg(param, char *));
@@ -1613,7 +1675,7 @@ index 1720b24b1..dcae3c143 100644
Curl_headersep(head->data[thislen]) )
return head->data;
diff --git a/lib/url.c b/lib/url.c
index 1114c6c12..b16628e96 100644
index 1114c6c12..75b724357 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -465,6 +465,11 @@ CURLcode Curl_close(struct Curl_easy **datap)
@@ -1646,7 +1708,7 @@ index 1114c6c12..b16628e96 100644
#ifndef CURL_DISABLE_PROXY
data->set.proxy_ssl.primary.CApath = data->set.str[STRING_SSL_CAPATH_PROXY];
@@ -3996,8 +4005,17 @@ static CURLcode create_conn(struct Curl_easy *data,
@@ -3996,8 +4005,21 @@ static CURLcode create_conn(struct Curl_easy *data,
conn->bits.tls_enable_alpn = TRUE;
if(data->set.ssl_enable_npn)
conn->bits.tls_enable_npn = TRUE;
@@ -1660,12 +1722,16 @@ index 1114c6c12..b16628e96 100644
+ /* curl-impersonate: Add the TLS session ticket extension. */
+ if(data->set.ssl_enable_ticket)
+ conn->bits.tls_enable_ticket = TRUE;
+
+ /* curl-impersonate: Add the TLS extension permutation. */
+ if(data->set.ssl_permute_extensions)
+ conn->bits.tls_permute_extensions = TRUE;
+
if(waitpipe)
/* There is a connection that *might* become usable for multiplexing
"soon", and we wait for that */
diff --git a/lib/urldata.h b/lib/urldata.h
index bcb4d460c..0eaa3e1c7 100644
index bcb4d460c..01e015706 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -254,6 +254,8 @@ struct ssl_primary_config {
@@ -1677,16 +1743,17 @@ index bcb4d460c..0eaa3e1c7 100644
unsigned char ssl_options; /* the CURLOPT_SSL_OPTIONS bitmask */
BIT(verifypeer); /* set TRUE if this is desired */
BIT(verifyhost); /* set TRUE if CN/SAN must match hostname */
@@ -509,6 +511,8 @@ struct ConnectBits {
@@ -509,6 +511,9 @@ struct ConnectBits {
BIT(tcp_fastopen); /* use TCP Fast Open */
BIT(tls_enable_npn); /* TLS NPN extension? */
BIT(tls_enable_alpn); /* TLS ALPN extension? */
+ BIT(tls_enable_alps); /* TLS ALPS extension? */
+ BIT(tls_enable_ticket); /* TLS session ticket extension? */
+ BIT(tls_permute_extensions); /* TLS extension permutations */
BIT(connect_only);
#ifndef CURL_DISABLE_DOH
BIT(doh);
@@ -1453,6 +1457,19 @@ struct UrlState {
@@ -1453,6 +1458,19 @@ struct UrlState {
CURLcode hresult; /* used to pass return codes back from hyper callbacks */
#endif
@@ -1706,7 +1773,7 @@ index bcb4d460c..0eaa3e1c7 100644
/* Dynamically allocated strings, MUST be freed before this struct is
killed. */
struct dynamically_allocated_data {
@@ -1608,6 +1625,9 @@ enum dupstring {
@@ -1608,6 +1626,9 @@ enum dupstring {
STRING_DNS_LOCAL_IP4,
STRING_DNS_LOCAL_IP6,
STRING_SSL_EC_CURVES,
@@ -1716,16 +1783,17 @@ index bcb4d460c..0eaa3e1c7 100644
/* -- end of null-terminated strings -- */
@@ -1893,6 +1913,8 @@ struct UserDefined {
@@ -1893,6 +1914,9 @@ struct UserDefined {
BIT(tcp_fastopen); /* use TCP Fast Open */
BIT(ssl_enable_npn); /* TLS NPN extension? */
BIT(ssl_enable_alpn);/* TLS ALPN extension? */
+ BIT(ssl_enable_alps);/* TLS ALPS extension? */
+ BIT(ssl_enable_ticket); /* TLS session ticket extension */
+ BIT(ssl_permute_extensions); /* TLS Permute extensions */
BIT(path_as_is); /* allow dotdots? */
BIT(pipewait); /* wait for multiplex status before starting a new
connection */
@@ -1911,6 +1933,9 @@ struct UserDefined {
@@ -1911,6 +1935,9 @@ struct UserDefined {
BIT(doh_verifystatus); /* DoH certificate status verification */
#endif
BIT(http09_allowed); /* allow HTTP/0.9 responses */
@@ -1736,7 +1804,7 @@ index bcb4d460c..0eaa3e1c7 100644
struct Names {
diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c
index 78aacd022..a29ca8055 100644
index 78aacd022..9f66af846 100644
--- a/lib/vtls/openssl.c
+++ b/lib/vtls/openssl.c
@@ -78,6 +78,13 @@
@@ -2071,7 +2139,7 @@ index 78aacd022..a29ca8055 100644
#ifdef USE_OPENSSL_SRP
if((ssl_authtype == CURL_TLSAUTH_SRP) &&
Curl_allow_auth_to_host(data)) {
@@ -2933,6 +3228,20 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
@@ -2933,6 +3228,28 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
}
#endif
@@ -2084,6 +2152,14 @@ index 78aacd022..a29ca8055 100644
+ /* Enable TLS GREASE. */
+ SSL_CTX_set_grease_enabled(backend->ctx, 1);
+
+ /*
+ * curl-impersonate: Enable TLS extension permutation, enabled by default
+ * since Chrome 110.
+ */
+ if(conn->bits.tls_permute_extensions) {
+ SSL_CTX_set_permute_extensions(backend->ctx, 1);
+ }
+
+ if(SSL_CONN_CONFIG(cert_compression) &&
+ add_cert_compression(data,
+ backend->ctx,
@@ -2092,7 +2168,7 @@ index 78aacd022..a29ca8055 100644
#if defined(USE_WIN32_CRYPTO)
/* Import certificates from the Windows root certificate store if requested.
@@ -3232,6 +3541,33 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
@@ -3232,6 +3549,33 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
SSL_set_connect_state(backend->handle);
@@ -2205,7 +2281,7 @@ index 706f0aac3..7124bf13e 100644
# if unit tests are enabled, build a static library to link them with
diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h
index 7e43fe754..071779251 100644
index 7e43fe754..6b698c3b1 100644
--- a/src/tool_cfgable.h
+++ b/src/tool_cfgable.h
@@ -165,8 +165,12 @@ struct OperationConfig {
@@ -2221,7 +2297,15 @@ index 7e43fe754..071779251 100644
long httpversion;
bool http09_allowed;
bool nobuffer;
@@ -275,6 +279,8 @@ struct OperationConfig {
@@ -196,6 +200,7 @@ struct OperationConfig {
struct curl_slist *prequote;
long ssl_version;
long ssl_version_max;
+ bool ssl_permute_extensions;
long proxy_ssl_version;
long ip_version;
long create_file_mode; /* CURLOPT_NEW_FILE_PERMS */
@@ -275,6 +280,8 @@ struct OperationConfig {
char *oauth_bearer; /* OAuth 2.0 bearer token */
bool nonpn; /* enable/disable TLS NPN extension */
bool noalpn; /* enable/disable TLS ALPN extension */
@@ -2231,10 +2315,10 @@ index 7e43fe754..071779251 100644
bool abstract_unix_socket; /* path to an abstract Unix domain socket */
bool falsestart;
diff --git a/src/tool_getparam.c b/src/tool_getparam.c
index 27e801a98..4f255d9bf 100644
index 27e801a98..d02540f76 100644
--- a/src/tool_getparam.c
+++ b/src/tool_getparam.c
@@ -282,6 +282,12 @@ static const struct LongShort aliases[]= {
@@ -282,6 +282,13 @@ static const struct LongShort aliases[]= {
{"EC", "etag-save", ARG_FILENAME},
{"ED", "etag-compare", ARG_FILENAME},
{"EE", "curves", ARG_STRING},
@@ -2244,10 +2328,11 @@ index 27e801a98..4f255d9bf 100644
+ {"EJ", "tls-session-ticket", ARG_BOOL},
+ {"EK", "http2-pseudo-headers-order", ARG_STRING},
+ {"EL", "http2-no-server-push", ARG_BOOL},
+ {"EM", "tls-permute-extensions", ARG_BOOL},
{"f", "fail", ARG_BOOL},
{"fa", "fail-early", ARG_BOOL},
{"fb", "styled-output", ARG_BOOL},
@@ -1859,6 +1865,36 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
@@ -1859,6 +1866,39 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
GetStr(&config->ssl_ec_curves, nextarg);
break;
@@ -2280,15 +2365,18 @@ index 27e801a98..4f255d9bf 100644
+ /* --http2-no-server-push */
+ config->http2_no_server_push = toggle;
+ break;
+
+ case 'M':
+ /* --tls-permute-extensions */
+ config->ssl_permute_extensions = toggle;
+ break;
default: /* unknown flag */
return PARAM_OPTION_UNKNOWN;
}
diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c
index 266f9b0bd..7eafa91b5 100644
index 266f9b0bd..a96c12de0 100644
--- a/src/tool_listhelp.c
+++ b/src/tool_listhelp.c
@@ -108,6 +108,21 @@ const struct helptxt helptext[] = {
@@ -108,6 +108,24 @@ const struct helptxt helptext[] = {
{" --curves <algorithm list>",
"(EC) TLS key exchange algorithm(s) to request",
CURLHELP_TLS},
@@ -2307,10 +2395,13 @@ index 266f9b0bd..7eafa91b5 100644
+ {" --http2-no-server-push",
+ "Send HTTP2 setting to disable server push",
+ CURLHELP_HTTP},
+ {" --tls-permute-extensions",
+ "Enable BoringSSL TLS extensions permutations on client hello",
+ CURLHELP_TLS},
{"-d, --data <data>",
"HTTP POST data",
CURLHELP_IMPORTANT | CURLHELP_HTTP | CURLHELP_POST | CURLHELP_UPLOAD},
@@ -384,6 +399,9 @@ const struct helptxt helptext[] = {
@@ -384,6 +402,9 @@ const struct helptxt helptext[] = {
{" --no-alpn",
"Disable the ALPN TLS extension",
CURLHELP_TLS | CURLHELP_HTTP},
@@ -2321,7 +2412,7 @@ index 266f9b0bd..7eafa91b5 100644
"Disable buffering of the output stream",
CURLHELP_CURL},
diff --git a/src/tool_operate.c b/src/tool_operate.c
index c317b3ba7..68e482357 100644
index c317b3ba7..481c6cd00 100644
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@@ -1433,6 +1433,15 @@ static CURLcode single_transfer(struct GlobalConfig *global,
@@ -2355,7 +2446,18 @@ index c317b3ba7..68e482357 100644
if(curlinfo->features & CURL_VERSION_SSL) {
/* Check if config->cert is a PKCS#11 URI and set the
* config->cert_type if necessary */
@@ -2057,6 +2074,14 @@ static CURLcode single_transfer(struct GlobalConfig *global,
@@ -1846,6 +1863,10 @@ static CURLcode single_transfer(struct GlobalConfig *global,
my_setopt_str(curl, CURLOPT_PROXY_TLS13_CIPHERS,
config->proxy_cipher13_list);
+ /* curl-impersonate */
+ if(config->ssl_permute_extensions)
+ my_setopt(curl, CURLOPT_SSL_PERMUTE_EXTENSIONS, 1L);
+
/* new in libcurl 7.9.2: */
if(config->disable_epsv)
/* disable it */
@@ -2057,6 +2078,14 @@ static CURLcode single_transfer(struct GlobalConfig *global,
my_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, 0L);
}
@@ -2371,14 +2473,15 @@ index c317b3ba7..68e482357 100644
if(config->unix_socket_path) {
if(config->abstract_unix_socket) {
diff --git a/src/tool_setopt.c b/src/tool_setopt.c
index 5ff86c7f5..e7b093d2d 100644
index 5ff86c7f5..be26f91ea 100644
--- a/src/tool_setopt.c
+++ b/src/tool_setopt.c
@@ -180,6 +180,7 @@ static const struct NameValue setopt_nv_CURLNONZERODEFAULTS[] = {
@@ -180,6 +180,8 @@ static const struct NameValue setopt_nv_CURLNONZERODEFAULTS[] = {
NV1(CURLOPT_SSL_VERIFYHOST, 1),
NV1(CURLOPT_SSL_ENABLE_NPN, 1),
NV1(CURLOPT_SSL_ENABLE_ALPN, 1),
+ NV1(CURLOPT_SSL_ENABLE_TICKET, 1),
+ NV1(CURLOPT_SSL_PERMUTE_EXTENSIONS, 1),
NV1(CURLOPT_TCP_NODELAY, 1),
NV1(CURLOPT_PROXY_SSL_VERIFYPEER, 1),
NV1(CURLOPT_PROXY_SSL_VERIFYHOST, 1),

78
tests/signature.py
View File

@@ -1,7 +1,7 @@
import enum
import struct
import collections
from typing import List, Any
from typing import List, Dict, Any
import yaml
@@ -45,6 +45,7 @@ class TLSExtensionType(enum.Enum):
record_size_limit = 28
delegated_credentials = 34
session_ticket = 35
pre_shared_key = 41
supported_versions = 43
psk_key_exchange_modes = 45
keyshare = 51
@@ -648,7 +649,20 @@ class TLSClientHelloSignature():
def extension_list(self):
return list(map(lambda ext: ext.ext_type, self.extensions))
def _compare_extensions(self, other: 'TLSClientHelloSignature'):
def _is_permuted_extension(self, ext: TLSExtensionSignature):
# Chrome permutes all TLS extensions except for GREASE and pre_shared_key
# (and the trailing padding)
return ext.ext_type not in [
TLSExtensionType.GREASE,
TLSExtensionType.pre_shared_key,
TLSExtensionType.padding
]
def _compare_extensions(
self,
other: 'TLSClientHelloSignature',
allow_tls_permutation: bool = False
):
"""Compare the TLS extensions of two Client Hello messages."""
# Check that the extension lists are identical in content.
if set(self.extension_list) != set(other.extension_list):
@@ -658,15 +672,23 @@ class TLSClientHelloSignature():
return False, (f"TLS extension lists differ: "
f"Symmatric difference {symdiff}")
if self.extension_list != other.extension_list:
if not allow_tls_permutation and self.extension_list != other.extension_list:
return False, "TLS extension lists identical but differ in order"
# Check the extensions' parameters.
for i, ext in enumerate(self.extensions):
if not ext.equals(other.extensions[i]):
if allow_tls_permutation and self._is_permuted_extension(ext):
# If TLS extension permutation is enabled, locate this extension
# in the other signature by type.
other_ext = next(
e for e in other.extensions if e.ext_type == ext.ext_type
)
else:
other_ext = other.extensions[i]
if not ext.equals(other_ext):
ours = ext.to_dict()
ours.pop("type")
theirs = other.extensions[i].to_dict()
theirs = other_ext.to_dict()
theirs.pop("type")
msg = (f"TLS extension {ext.ext_type.name} is different. "
f"{ours} != {theirs}")
@@ -674,7 +696,11 @@ class TLSClientHelloSignature():
return True, None
def _equals(self, other: 'TLSClientHelloSignature', reason: bool = False):
def _equals(
self,
other: 'TLSClientHelloSignature',
allow_tls_permutation: bool = False
):
"""Check if another TLSClientHelloSignature is identical."""
if self.record_version != other.record_version:
msg = (f"TLS record versions differ: "
@@ -700,20 +726,32 @@ class TLSClientHelloSignature():
msg = f"TLS compression methods differ in contents or order. "
return False, msg
return self._compare_extensions(other)
return self._compare_extensions(other, allow_tls_permutation)
def equals(self, other: 'TLSClientHelloSignature', reason: bool = False):
def equals(
self,
other: 'TLSClientHelloSignature',
allow_tls_permutation: bool = False,
reason: bool = False
):
"""Checks whether two Client Hello messages have the same signature.
Parameters
----------
other : TLSClientHelloSignature
The signature of the other Client Hello message.
allow_tls_permutation : bool
Allow TLS extension permutations. If set to True, and the TLS
extensions are identical between the signatures but differ in
order, the signatures will be considered equal.
reason : bool
If True, returns an additional string describing the reason of the
difference in case of a difference, and None otherwise.
"""
equal, msg = self._equals(other)
equal, msg = self._equals(
other,
allow_tls_permutation=allow_tls_permutation
)
if reason:
return equal, msg
else:
@@ -939,13 +977,17 @@ class BrowserSignature:
http2 : HTTP2Signature
The HTTP/2 signature of the browser.
Can be None, in which case it is ignored.
options: dict
Optional parameters specifying how to
"""
def __init__(self,
tls_client_hello: TLSClientHelloSignature = None,
http2: HTTP2Signature = None):
http2: HTTP2Signature = None,
options: Dict = None):
self.tls_client_hello = tls_client_hello
self.http2 = http2
self.options = options
def _equals(self, other: 'BrowserSignature'):
# If one is None, so must be the other
@@ -968,6 +1010,14 @@ class BrowserSignature:
if not equal:
return equal, msg
if (self.options is None) != (other.options is None):
return False, "Options present in one signature but not the other"
if self.options is not None:
if self.options != other.options:
msg = (f"Options differ: {self.options} != {other.options}")
return False, msg
return True, None
def equals(self, other: 'BrowserSignature', reason: bool = False):
@@ -990,6 +1040,8 @@ class BrowserSignature:
def to_dict(self):
"""Serialize to a dict object."""
d = {}
if self.options is not None:
d["options"] = self.options
if self.tls_client_hello is not None:
d["tls_client_hello"] = self.tls_client_hello.to_dict()
if self.http2 is not None:
@@ -1011,4 +1063,8 @@ class BrowserSignature:
else:
http2 = None
return BrowserSignature(tls_client_hello=tls_client_hello, http2=http2)
return BrowserSignature(
tls_client_hello=tls_client_hello,
http2=http2,
options=d.get("options")
)

98
tests/signatures/chrome.yaml
View File

@@ -576,6 +576,104 @@ signature:
- 'accept-encoding: gzip, deflate, br'
- 'accept-language: en-US,en;q=0.9'
---
name: chrome_110.0.5481.177_win10
browser:
name: chrome
version: 110.0.5481.177
os: win10
mode: regular
signature:
options:
tls_permute_extensions: true
tls_client_hello:
record_version: 'TLS_VERSION_1_0'
handshake_version: 'TLS_VERSION_1_2'
session_id_length: 32
ciphersuites: [
'GREASE',
0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030,
0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f,
0x0035
]
comp_methods: [0x00]
extensions:
- type: GREASE
length: 0
- type: server_name
- type: extended_master_secret
length: 0
- type: renegotiation_info
length: 1
- type: supported_groups
length: 10
supported_groups: [
'GREASE',
0x001d, 0x0017, 0x0018
]
- type: ec_point_formats
length: 2
ec_point_formats: [0]
- type: session_ticket
length: 0
- type: application_layer_protocol_negotiation
length: 14
alpn_list: ['h2', 'http/1.1']
- type: status_request
length: 5
status_request_type: 0x01
- type: signature_algorithms
length: 18
sig_hash_algs: [
0x0403, 0x0804, 0x0401, 0x0503,
0x0805, 0x0501, 0x0806, 0x0601
]
- type: signed_certificate_timestamp
length: 0
- type: keyshare
length: 43
key_shares:
- group: GREASE
length: 1
- group: 29
length: 32
- type: psk_key_exchange_modes
length: 2
psk_ke_mode: 1
- type: supported_versions
length: 7
supported_versions: [
'GREASE', 'TLS_VERSION_1_3', 'TLS_VERSION_1_2'
]
- type: compress_certificate
length: 3
algorithms: [0x02]
- type: application_settings
length: 5
alps_alpn_list: ['h2']
- type: GREASE
length: 1
data: !!binary AA==
- type: padding
http2:
pseudo_headers:
- ':method'
- ':authority'
- ':scheme'
- ':path'
headers:
- 'sec-ch-ua: "Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110"'
- 'sec-ch-ua-mobile: ?0'
- 'sec-ch-ua-platform: "Windows"'
- 'upgrade-insecure-requests: 1'
- 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36'
- 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7'
- 'sec-fetch-site: none'
- 'sec-fetch-mode: navigate'
- 'sec-fetch-user: ?1'
- 'sec-fetch-dest: document'
- 'accept-encoding: gzip, deflate, br'
- 'accept-language: en-US,en;q=0.9'
---
name: chrome_99.0.4844.73_android12-pixel6
browser:
name: chrome

19
tests/test_impersonate.py
View File

@@ -148,6 +148,7 @@ class TestImpersonation:
("curl_chrome101", None, None, "chrome_101.0.4951.67_win10"),
("curl_chrome104", None, None, "chrome_104.0.5112.81_win10"),
("curl_chrome107", None, None, "chrome_107.0.5304.107_win10"),
("curl_chrome110", None, None, "chrome_110.0.5481.177_win10"),
("curl_chrome99_android", None, None, "chrome_99.0.4844.73_android12-pixel6"),
("curl_edge99", None, None, "edge_99.0.1150.30_win10"),
("curl_edge101", None, None, "edge_101.0.1210.47_win10"),
@@ -203,6 +204,14 @@ class TestImpersonation:
"libcurl-impersonate-chrome",
"chrome_107.0.5304.107_win10"
),
(
"minicurl",
{
"CURL_IMPERSONATE": "chrome110"
},
"libcurl-impersonate-chrome",
"chrome_110.0.5481.177_win10"
),
(
"minicurl",
{
@@ -556,7 +565,15 @@ class TestImpersonation:
["tls_client_hello"]
)
equals, msg = sig.equals(expected_sig, reason=True)
allow_tls_permutation=browser_signatures[expected_signature] \
["signature"] \
.get("options", {}) \
.get("tls_permute_extensions", False)
equals, msg = sig.equals(
expected_sig,
allow_tls_permutation=allow_tls_permutation,
reason=True
)
assert equals, msg
@pytest.mark.asyncio