From 9d05a810302ccaa82cd08fa4f5ea6546401484ff Mon Sep 17 00:00:00 2001 From: lwthiker <99899249+lwthiker@users.noreply.github.com> Date: Sat, 25 Feb 2023 11:29:14 +0200 Subject: [PATCH] Impersonate Chrome 110 (#148) Add support for impersonating Chrome 110. Chrome 110 comes with TLS extension permutation enabled by default. We mimic this behavior in libcurl with the new CURLOPT_SSL_PERMUTE_EXTENSIONS option, which enables the corresponding flag in BoringSSL. --------- Co-authored-by: Johann Saunier --- README.md | 3 +- browsers.json | 10 ++ chrome/curl_chrome110 | 26 ++++ chrome/patches/curl-impersonate.patch | 189 ++++++++++++++++++++------ tests/signature.py | 78 +++++++++-- tests/signatures/chrome.yaml | 98 +++++++++++++ tests/test_impersonate.py | 19 ++- 7 files changed, 367 insertions(+), 56 deletions(-) create mode 100755 chrome/curl_chrome110 diff --git a/README.md b/README.md index caa199f..3cf506f 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ The following browsers can be impersonated. | ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 101 | 101.0.4951.67 | Windows 10 | `chrome101` | [curl_chrome101](chrome/curl_chrome101) | | ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 104 | 104.0.5112.81 | Windows 10 | `chrome104` | [curl_chrome104](chrome/curl_chrome104) | | ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 107 | 107.0.5304.107 | Windows 10 | `chrome107` | [curl_chrome107](chrome/curl_chrome107) | +| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 110 | 110.0.5481.177 | Windows 10 | `chrome110` | [curl_chrome110](chrome/curl_chrome110) | | ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 99 | 99.0.4844.73 | Android 12 | `chrome99_android` | [curl_chrome99_android](chrome/curl_chrome99_android) | | ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") | 99 | 99.0.1150.30 | Windows 10 | `edge99` | [curl_edge99](chrome/curl_edge99) | | ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") | 101 | 101.0.1210.47 | Windows 10 | `edge101` | [curl_edge101](chrome/curl_edge101) | @@ -123,7 +124,7 @@ Calling the above function sets the following libcurl options: * `CURLOPT_HTTPBASEHEADER`, if `default_headers` is non-zero (this is a non-standard HTTP option created for this project). * `CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER`, `CURLOPT_HTTP2_NO_SERVER_PUSH` (non-standard HTTP/2 options created for this project). * `CURLOPT_SSL_ENABLE_ALPS`, `CURLOPT_SSL_SIG_HASH_ALGS`, `CURLOPT_SSL_CERT_COMPRESSION`, `CURLOPT_SSL_ENABLE_TICKET` (non-standard TLS options created for this project). - +* `CURLOPT_SSL_PERMUTE_EXTENSIONS` (non-standard TLS options created for this project). Note that if you call `curl_easy_setopt()` later with one of the above it will override the options set by `curl_easy_impersonate()`. ### Using CURL_IMPERSONATE env var diff --git a/browsers.json b/browsers.json index 08095f9..7f57067 100644 --- a/browsers.json +++ b/browsers.json @@ -50,6 +50,16 @@ "binary": "curl-impersonate-chrome", "wrapper_script": "curl_chrome107" }, + { + "name": "chrome110", + "browser": { + "name": "chrome", + "version": "110.0.5481.177", + "os": "win10" + }, + "binary": "curl-impersonate-chrome", + "wrapper_script": "curl_chrome110" + }, { "name": "chrome99_android", "browser": { diff --git a/chrome/curl_chrome110 b/chrome/curl_chrome110 new file mode 100755 index 0000000..23617b5 --- /dev/null +++ b/chrome/curl_chrome110 @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Find the directory of this script +dir=${0%/*} + +# The list of ciphers can be obtained by looking at the Client Hello message in +# Wireshark, then converting it using this reference +# https://wiki.mozilla.org/Security/Cipher_Suites +"$dir/curl-impersonate-chrome" \ + --ciphers TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA \ + -H 'sec-ch-ua: "Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "Windows"' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \ + -H 'Sec-Fetch-Site: none' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-User: ?1' \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Accept-Encoding: gzip, deflate, br' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + --http2 --http2-no-server-push --false-start --compressed \ + --tlsv1.2 --no-npn --alps --tls-permute-extensions \ + --cert-compression brotli \ + "$@" diff --git a/chrome/patches/curl-impersonate.patch b/chrome/patches/curl-impersonate.patch index 82e4965..fd3bbf1 100644 --- a/chrome/patches/curl-impersonate.patch +++ b/chrome/patches/curl-impersonate.patch @@ -82,10 +82,10 @@ index aaf2b8a43..ccfa52985 100644 echo "curl was built with static libraries disabled" >&2 exit 1 diff --git a/include/curl/curl.h b/include/curl/curl.h -index b00648e79..963d68382 100644 +index b00648e79..a8edd2e2d 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h -@@ -2143,6 +2143,44 @@ typedef enum { +@@ -2143,6 +2143,50 @@ typedef enum { /* set the SSH host key callback custom pointer */ CURLOPT(CURLOPT_SSH_HOSTKEYDATA, CURLOPTTYPE_CBPOINT, 317), @@ -126,6 +126,12 @@ index b00648e79..963d68382 100644 + * Disable HTTP2 server push in the HTTP2 SETTINGS. + */ + CURLOPT(CURLOPT_HTTP2_NO_SERVER_PUSH, CURLOPTTYPE_LONG, 324), ++ ++ /* ++ * curl-impersonate: Whether to enable Boringssl permute extensions ++ * See https://commondatastorage.googleapis.com/chromium-boringssl-docs/ssl.h.html#SSL_set_permute_extensions. ++ */ ++ CURLOPT(CURLOPT_SSL_PERMUTE_EXTENSIONS, CURLOPTTYPE_LONG, 325), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; @@ -244,7 +250,7 @@ index 9bd8e324b..bfd5e90e2 100644 inet_pton.c \ krb5.c \ diff --git a/lib/easy.c b/lib/easy.c -index 704a59df6..1d0251b8b 100644 +index 704a59df6..f486362c0 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -81,6 +81,8 @@ @@ -256,7 +262,7 @@ index 704a59df6..1d0251b8b 100644 /* The last 3 #include files should be in this order */ #include "curl_printf.h" -@@ -332,6 +334,128 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, +@@ -332,6 +334,134 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, return rc; } @@ -333,6 +339,12 @@ index 704a59df6..1d0251b8b 100644 + if(ret) + return ret; + ++ if(opts->tls_permute_extensions) { ++ ret = curl_easy_setopt(data, CURLOPT_SSL_PERMUTE_EXTENSIONS, 1); ++ if(ret) ++ return ret; ++ } ++ + if(opts->cert_compression) { + ret = curl_easy_setopt(data, + CURLOPT_SSL_CERT_COMPRESSION, @@ -385,7 +397,7 @@ index 704a59df6..1d0251b8b 100644 /* * curl_easy_init() is the external interface to alloc, setup and init an * easy handle that is returned. If anything goes wrong, NULL is returned. -@@ -340,6 +464,8 @@ struct Curl_easy *curl_easy_init(void) +@@ -340,6 +470,8 @@ struct Curl_easy *curl_easy_init(void) { CURLcode result; struct Curl_easy *data; @@ -394,7 +406,7 @@ index 704a59df6..1d0251b8b 100644 /* Make sure we inited the global SSL stuff */ global_init_lock(); -@@ -362,6 +488,29 @@ struct Curl_easy *curl_easy_init(void) +@@ -362,6 +494,29 @@ struct Curl_easy *curl_easy_init(void) return NULL; } @@ -424,7 +436,7 @@ index 704a59df6..1d0251b8b 100644 return data; } -@@ -936,6 +1085,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) +@@ -936,6 +1091,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) outcurl->state.referer_alloc = TRUE; } @@ -438,7 +450,7 @@ index 704a59df6..1d0251b8b 100644 /* Reinitialize an SSL engine for the new handle * note: the engine name has already been copied by dupset */ if(outcurl->set.str[STRING_SSL_ENGINE]) { -@@ -1025,6 +1181,9 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) +@@ -1025,6 +1187,9 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) */ void curl_easy_reset(struct Curl_easy *data) { @@ -448,7 +460,7 @@ index 704a59df6..1d0251b8b 100644 Curl_free_request_state(data); /* zero out UserDefined data: */ -@@ -1049,6 +1208,23 @@ void curl_easy_reset(struct Curl_easy *data) +@@ -1049,6 +1214,23 @@ void curl_easy_reset(struct Curl_easy *data) #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH) Curl_http_auth_cleanup_digest(data); #endif @@ -473,7 +485,7 @@ index 704a59df6..1d0251b8b 100644 /* diff --git a/lib/easyoptions.c b/lib/easyoptions.c -index c99f135ff..4a68332b9 100644 +index c99f135ff..6b63fe4ae 100644 --- a/lib/easyoptions.c +++ b/lib/easyoptions.c @@ -130,8 +130,12 @@ struct curl_easyoption Curl_easyopts[] = { @@ -489,7 +501,7 @@ index c99f135ff..4a68332b9 100644 {"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0}, {"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0}, {"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0}, -@@ -297,15 +301,19 @@ struct curl_easyoption Curl_easyopts[] = { +@@ -297,18 +301,23 @@ struct curl_easyoption Curl_easyopts[] = { {"SSLKEYTYPE", CURLOPT_SSLKEYTYPE, CURLOT_STRING, 0}, {"SSLKEY_BLOB", CURLOPT_SSLKEY_BLOB, CURLOT_BLOB, 0}, {"SSLVERSION", CURLOPT_SSLVERSION, CURLOT_VALUES, 0}, @@ -509,12 +521,16 @@ index c99f135ff..4a68332b9 100644 {"SSL_VERIFYHOST", CURLOPT_SSL_VERIFYHOST, CURLOT_LONG, 0}, {"SSL_VERIFYPEER", CURLOPT_SSL_VERIFYPEER, CURLOT_LONG, 0}, {"SSL_VERIFYSTATUS", CURLOPT_SSL_VERIFYSTATUS, CURLOT_LONG, 0}, -@@ -364,6 +372,6 @@ struct curl_easyoption Curl_easyopts[] = { ++ {"SSL_PERMUTE_EXTENSIONS", CURLOPT_SSL_PERMUTE_EXTENSIONS, CURLOT_LONG, 0}, + {"STDERR", CURLOPT_STDERR, CURLOT_OBJECT, 0}, + {"STREAM_DEPENDS", CURLOPT_STREAM_DEPENDS, CURLOT_OBJECT, 0}, + {"STREAM_DEPENDS_E", CURLOPT_STREAM_DEPENDS_E, CURLOT_OBJECT, 0}, +@@ -364,6 +373,6 @@ struct curl_easyoption Curl_easyopts[] = { */ int Curl_easyopts_check(void) { - return ((CURLOPT_LASTENTRY%10000) != (317 + 1)); -+ return ((CURLOPT_LASTENTRY%10000) != (324 + 1)); ++ return ((CURLOPT_LASTENTRY%10000) != (325 + 1)); } #endif diff --git a/lib/h2h3.c b/lib/h2h3.c @@ -1001,10 +1017,10 @@ index f0390596c..cf9b7a9d5 100644 * Store nghttp2 version info in this buffer. diff --git a/lib/impersonate.c b/lib/impersonate.c new file mode 100644 -index 000000000..2c8a4d3f9 +index 000000000..025bd58ef --- /dev/null +++ b/lib/impersonate.c -@@ -0,0 +1,438 @@ +@@ -0,0 +1,480 @@ +#include "curl_setup.h" + +#include "impersonate.h" @@ -1212,6 +1228,48 @@ index 000000000..2c8a4d3f9 + .http2_no_server_push = true + }, + { ++ .target = "chrome110", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "ECDHE-ECDSA-AES128-GCM-SHA256," ++ "ECDHE-RSA-AES128-GCM-SHA256," ++ "ECDHE-ECDSA-AES256-GCM-SHA384," ++ "ECDHE-RSA-AES256-GCM-SHA384," ++ "ECDHE-ECDSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-AES128-SHA," ++ "ECDHE-RSA-AES256-SHA," ++ "AES128-GCM-SHA256," ++ "AES256-GCM-SHA384," ++ "AES128-SHA," ++ "AES256-SHA", ++ .npn = false, ++ .alpn = true, ++ .alps = true, ++ .tls_permute_extensions = true, ++ .tls_session_ticket = true, ++ .cert_compression = "brotli", ++ .http_headers = { ++ "sec-ch-ua: \"Chromium\";v=\"110\", \"Not A(Brand\";v=\"24\", \"Google Chrome\";v=\"110\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"Windows\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br", ++ "Accept-Language: en-US,en;q=0.9" ++ }, ++ .http2_no_server_push = true ++ }, ++ { + .target = "chrome99_android", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, @@ -1445,10 +1503,10 @@ index 000000000..2c8a4d3f9 +}; diff --git a/lib/impersonate.h b/lib/impersonate.h new file mode 100644 -index 000000000..05245c171 +index 000000000..c62991c5a --- /dev/null +++ b/lib/impersonate.h -@@ -0,0 +1,44 @@ +@@ -0,0 +1,45 @@ +#ifndef HEADER_CURL_IMPERSONATE_H +#define HEADER_CURL_IMPERSONATE_H + @@ -1482,6 +1540,7 @@ index 000000000..05245c171 + const char *http_headers[IMPERSONATE_MAX_HEADERS]; + const char *http2_pseudo_headers_order; + bool http2_no_server_push; ++ bool tls_permute_extensions; + /* Other TLS options will come here in the future once they are + * configurable through curl_easy_setopt() */ +}; @@ -1508,7 +1567,7 @@ index e0280447c..dc1fdab68 100644 #ifdef USE_WINSOCK diff --git a/lib/setopt.c b/lib/setopt.c -index 6b16e1c7c..a6d34ebd7 100644 +index 6b16e1c7c..189b54025 100644 --- a/lib/setopt.c +++ b/lib/setopt.c @@ -50,6 +50,7 @@ @@ -1571,7 +1630,7 @@ index 6b16e1c7c..a6d34ebd7 100644 #endif case CURLOPT_IPRESOLVE: arg = va_arg(param, long); -@@ -2861,6 +2900,19 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) +@@ -2861,6 +2900,22 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) case CURLOPT_SSL_ENABLE_ALPN: data->set.ssl_enable_alpn = (0 != va_arg(param, long)) ? TRUE : FALSE; break; @@ -1581,6 +1640,9 @@ index 6b16e1c7c..a6d34ebd7 100644 + case CURLOPT_SSL_ENABLE_TICKET: + data->set.ssl_enable_ticket = (0 != va_arg(param, long)) ? TRUE : FALSE; + break; ++ case CURLOPT_SSL_PERMUTE_EXTENSIONS: ++ data->set.ssl_permute_extensions = (0 != va_arg(param, long)) ? TRUE : FALSE; ++ break; + case CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER: + result = Curl_setstropt(&data->set.str[STRING_HTTP2_PSEUDO_HEADERS_ORDER], + va_arg(param, char *)); @@ -1613,7 +1675,7 @@ index 1720b24b1..dcae3c143 100644 Curl_headersep(head->data[thislen]) ) return head->data; diff --git a/lib/url.c b/lib/url.c -index 1114c6c12..b16628e96 100644 +index 1114c6c12..75b724357 100644 --- a/lib/url.c +++ b/lib/url.c @@ -465,6 +465,11 @@ CURLcode Curl_close(struct Curl_easy **datap) @@ -1646,7 +1708,7 @@ index 1114c6c12..b16628e96 100644 #ifndef CURL_DISABLE_PROXY data->set.proxy_ssl.primary.CApath = data->set.str[STRING_SSL_CAPATH_PROXY]; -@@ -3996,8 +4005,17 @@ static CURLcode create_conn(struct Curl_easy *data, +@@ -3996,8 +4005,21 @@ static CURLcode create_conn(struct Curl_easy *data, conn->bits.tls_enable_alpn = TRUE; if(data->set.ssl_enable_npn) conn->bits.tls_enable_npn = TRUE; @@ -1660,12 +1722,16 @@ index 1114c6c12..b16628e96 100644 + /* curl-impersonate: Add the TLS session ticket extension. */ + if(data->set.ssl_enable_ticket) + conn->bits.tls_enable_ticket = TRUE; ++ ++ /* curl-impersonate: Add the TLS extension permutation. */ ++ if(data->set.ssl_permute_extensions) ++ conn->bits.tls_permute_extensions = TRUE; + if(waitpipe) /* There is a connection that *might* become usable for multiplexing "soon", and we wait for that */ diff --git a/lib/urldata.h b/lib/urldata.h -index bcb4d460c..0eaa3e1c7 100644 +index bcb4d460c..01e015706 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -254,6 +254,8 @@ struct ssl_primary_config { @@ -1677,16 +1743,17 @@ index bcb4d460c..0eaa3e1c7 100644 unsigned char ssl_options; /* the CURLOPT_SSL_OPTIONS bitmask */ BIT(verifypeer); /* set TRUE if this is desired */ BIT(verifyhost); /* set TRUE if CN/SAN must match hostname */ -@@ -509,6 +511,8 @@ struct ConnectBits { +@@ -509,6 +511,9 @@ struct ConnectBits { BIT(tcp_fastopen); /* use TCP Fast Open */ BIT(tls_enable_npn); /* TLS NPN extension? */ BIT(tls_enable_alpn); /* TLS ALPN extension? */ + BIT(tls_enable_alps); /* TLS ALPS extension? */ + BIT(tls_enable_ticket); /* TLS session ticket extension? */ ++ BIT(tls_permute_extensions); /* TLS extension permutations */ BIT(connect_only); #ifndef CURL_DISABLE_DOH BIT(doh); -@@ -1453,6 +1457,19 @@ struct UrlState { +@@ -1453,6 +1458,19 @@ struct UrlState { CURLcode hresult; /* used to pass return codes back from hyper callbacks */ #endif @@ -1706,7 +1773,7 @@ index bcb4d460c..0eaa3e1c7 100644 /* Dynamically allocated strings, MUST be freed before this struct is killed. */ struct dynamically_allocated_data { -@@ -1608,6 +1625,9 @@ enum dupstring { +@@ -1608,6 +1626,9 @@ enum dupstring { STRING_DNS_LOCAL_IP4, STRING_DNS_LOCAL_IP6, STRING_SSL_EC_CURVES, @@ -1716,16 +1783,17 @@ index bcb4d460c..0eaa3e1c7 100644 /* -- end of null-terminated strings -- */ -@@ -1893,6 +1913,8 @@ struct UserDefined { +@@ -1893,6 +1914,9 @@ struct UserDefined { BIT(tcp_fastopen); /* use TCP Fast Open */ BIT(ssl_enable_npn); /* TLS NPN extension? */ BIT(ssl_enable_alpn);/* TLS ALPN extension? */ + BIT(ssl_enable_alps);/* TLS ALPS extension? */ + BIT(ssl_enable_ticket); /* TLS session ticket extension */ ++ BIT(ssl_permute_extensions); /* TLS Permute extensions */ BIT(path_as_is); /* allow dotdots? */ BIT(pipewait); /* wait for multiplex status before starting a new connection */ -@@ -1911,6 +1933,9 @@ struct UserDefined { +@@ -1911,6 +1935,9 @@ struct UserDefined { BIT(doh_verifystatus); /* DoH certificate status verification */ #endif BIT(http09_allowed); /* allow HTTP/0.9 responses */ @@ -1736,7 +1804,7 @@ index bcb4d460c..0eaa3e1c7 100644 struct Names { diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c -index 78aacd022..a29ca8055 100644 +index 78aacd022..9f66af846 100644 --- a/lib/vtls/openssl.c +++ b/lib/vtls/openssl.c @@ -78,6 +78,13 @@ @@ -2071,7 +2139,7 @@ index 78aacd022..a29ca8055 100644 #ifdef USE_OPENSSL_SRP if((ssl_authtype == CURL_TLSAUTH_SRP) && Curl_allow_auth_to_host(data)) { -@@ -2933,6 +3228,20 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data, +@@ -2933,6 +3228,28 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data, } #endif @@ -2084,6 +2152,14 @@ index 78aacd022..a29ca8055 100644 + /* Enable TLS GREASE. */ + SSL_CTX_set_grease_enabled(backend->ctx, 1); + ++ /* ++ * curl-impersonate: Enable TLS extension permutation, enabled by default ++ * since Chrome 110. ++ */ ++ if(conn->bits.tls_permute_extensions) { ++ SSL_CTX_set_permute_extensions(backend->ctx, 1); ++ } ++ + if(SSL_CONN_CONFIG(cert_compression) && + add_cert_compression(data, + backend->ctx, @@ -2092,7 +2168,7 @@ index 78aacd022..a29ca8055 100644 #if defined(USE_WIN32_CRYPTO) /* Import certificates from the Windows root certificate store if requested. -@@ -3232,6 +3541,33 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data, +@@ -3232,6 +3549,33 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data, SSL_set_connect_state(backend->handle); @@ -2205,7 +2281,7 @@ index 706f0aac3..7124bf13e 100644 # if unit tests are enabled, build a static library to link them with diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h -index 7e43fe754..071779251 100644 +index 7e43fe754..6b698c3b1 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -165,8 +165,12 @@ struct OperationConfig { @@ -2221,7 +2297,15 @@ index 7e43fe754..071779251 100644 long httpversion; bool http09_allowed; bool nobuffer; -@@ -275,6 +279,8 @@ struct OperationConfig { +@@ -196,6 +200,7 @@ struct OperationConfig { + struct curl_slist *prequote; + long ssl_version; + long ssl_version_max; ++ bool ssl_permute_extensions; + long proxy_ssl_version; + long ip_version; + long create_file_mode; /* CURLOPT_NEW_FILE_PERMS */ +@@ -275,6 +280,8 @@ struct OperationConfig { char *oauth_bearer; /* OAuth 2.0 bearer token */ bool nonpn; /* enable/disable TLS NPN extension */ bool noalpn; /* enable/disable TLS ALPN extension */ @@ -2231,10 +2315,10 @@ index 7e43fe754..071779251 100644 bool abstract_unix_socket; /* path to an abstract Unix domain socket */ bool falsestart; diff --git a/src/tool_getparam.c b/src/tool_getparam.c -index 27e801a98..4f255d9bf 100644 +index 27e801a98..d02540f76 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c -@@ -282,6 +282,12 @@ static const struct LongShort aliases[]= { +@@ -282,6 +282,13 @@ static const struct LongShort aliases[]= { {"EC", "etag-save", ARG_FILENAME}, {"ED", "etag-compare", ARG_FILENAME}, {"EE", "curves", ARG_STRING}, @@ -2244,10 +2328,11 @@ index 27e801a98..4f255d9bf 100644 + {"EJ", "tls-session-ticket", ARG_BOOL}, + {"EK", "http2-pseudo-headers-order", ARG_STRING}, + {"EL", "http2-no-server-push", ARG_BOOL}, ++ {"EM", "tls-permute-extensions", ARG_BOOL}, {"f", "fail", ARG_BOOL}, {"fa", "fail-early", ARG_BOOL}, {"fb", "styled-output", ARG_BOOL}, -@@ -1859,6 +1865,36 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ +@@ -1859,6 +1866,39 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ GetStr(&config->ssl_ec_curves, nextarg); break; @@ -2280,15 +2365,18 @@ index 27e801a98..4f255d9bf 100644 + /* --http2-no-server-push */ + config->http2_no_server_push = toggle; + break; -+ ++ case 'M': ++ /* --tls-permute-extensions */ ++ config->ssl_permute_extensions = toggle; ++ break; default: /* unknown flag */ return PARAM_OPTION_UNKNOWN; } diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c -index 266f9b0bd..7eafa91b5 100644 +index 266f9b0bd..a96c12de0 100644 --- a/src/tool_listhelp.c +++ b/src/tool_listhelp.c -@@ -108,6 +108,21 @@ const struct helptxt helptext[] = { +@@ -108,6 +108,24 @@ const struct helptxt helptext[] = { {" --curves ", "(EC) TLS key exchange algorithm(s) to request", CURLHELP_TLS}, @@ -2307,10 +2395,13 @@ index 266f9b0bd..7eafa91b5 100644 + {" --http2-no-server-push", + "Send HTTP2 setting to disable server push", + CURLHELP_HTTP}, ++ {" --tls-permute-extensions", ++ "Enable BoringSSL TLS extensions permutations on client hello", ++ CURLHELP_TLS}, {"-d, --data ", "HTTP POST data", CURLHELP_IMPORTANT | CURLHELP_HTTP | CURLHELP_POST | CURLHELP_UPLOAD}, -@@ -384,6 +399,9 @@ const struct helptxt helptext[] = { +@@ -384,6 +402,9 @@ const struct helptxt helptext[] = { {" --no-alpn", "Disable the ALPN TLS extension", CURLHELP_TLS | CURLHELP_HTTP}, @@ -2321,7 +2412,7 @@ index 266f9b0bd..7eafa91b5 100644 "Disable buffering of the output stream", CURLHELP_CURL}, diff --git a/src/tool_operate.c b/src/tool_operate.c -index c317b3ba7..68e482357 100644 +index c317b3ba7..481c6cd00 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -1433,6 +1433,15 @@ static CURLcode single_transfer(struct GlobalConfig *global, @@ -2355,7 +2446,18 @@ index c317b3ba7..68e482357 100644 if(curlinfo->features & CURL_VERSION_SSL) { /* Check if config->cert is a PKCS#11 URI and set the * config->cert_type if necessary */ -@@ -2057,6 +2074,14 @@ static CURLcode single_transfer(struct GlobalConfig *global, +@@ -1846,6 +1863,10 @@ static CURLcode single_transfer(struct GlobalConfig *global, + my_setopt_str(curl, CURLOPT_PROXY_TLS13_CIPHERS, + config->proxy_cipher13_list); + ++ /* curl-impersonate */ ++ if(config->ssl_permute_extensions) ++ my_setopt(curl, CURLOPT_SSL_PERMUTE_EXTENSIONS, 1L); ++ + /* new in libcurl 7.9.2: */ + if(config->disable_epsv) + /* disable it */ +@@ -2057,6 +2078,14 @@ static CURLcode single_transfer(struct GlobalConfig *global, my_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, 0L); } @@ -2371,14 +2473,15 @@ index c317b3ba7..68e482357 100644 if(config->unix_socket_path) { if(config->abstract_unix_socket) { diff --git a/src/tool_setopt.c b/src/tool_setopt.c -index 5ff86c7f5..e7b093d2d 100644 +index 5ff86c7f5..be26f91ea 100644 --- a/src/tool_setopt.c +++ b/src/tool_setopt.c -@@ -180,6 +180,7 @@ static const struct NameValue setopt_nv_CURLNONZERODEFAULTS[] = { +@@ -180,6 +180,8 @@ static const struct NameValue setopt_nv_CURLNONZERODEFAULTS[] = { NV1(CURLOPT_SSL_VERIFYHOST, 1), NV1(CURLOPT_SSL_ENABLE_NPN, 1), NV1(CURLOPT_SSL_ENABLE_ALPN, 1), + NV1(CURLOPT_SSL_ENABLE_TICKET, 1), ++ NV1(CURLOPT_SSL_PERMUTE_EXTENSIONS, 1), NV1(CURLOPT_TCP_NODELAY, 1), NV1(CURLOPT_PROXY_SSL_VERIFYPEER, 1), NV1(CURLOPT_PROXY_SSL_VERIFYHOST, 1), diff --git a/tests/signature.py b/tests/signature.py index 868fd20..5244dd3 100644 --- a/tests/signature.py +++ b/tests/signature.py @@ -1,7 +1,7 @@ import enum import struct import collections -from typing import List, Any +from typing import List, Dict, Any import yaml @@ -45,6 +45,7 @@ class TLSExtensionType(enum.Enum): record_size_limit = 28 delegated_credentials = 34 session_ticket = 35 + pre_shared_key = 41 supported_versions = 43 psk_key_exchange_modes = 45 keyshare = 51 @@ -648,7 +649,20 @@ class TLSClientHelloSignature(): def extension_list(self): return list(map(lambda ext: ext.ext_type, self.extensions)) - def _compare_extensions(self, other: 'TLSClientHelloSignature'): + def _is_permuted_extension(self, ext: TLSExtensionSignature): + # Chrome permutes all TLS extensions except for GREASE and pre_shared_key + # (and the trailing padding) + return ext.ext_type not in [ + TLSExtensionType.GREASE, + TLSExtensionType.pre_shared_key, + TLSExtensionType.padding + ] + + def _compare_extensions( + self, + other: 'TLSClientHelloSignature', + allow_tls_permutation: bool = False + ): """Compare the TLS extensions of two Client Hello messages.""" # Check that the extension lists are identical in content. if set(self.extension_list) != set(other.extension_list): @@ -658,15 +672,23 @@ class TLSClientHelloSignature(): return False, (f"TLS extension lists differ: " f"Symmatric difference {symdiff}") - if self.extension_list != other.extension_list: + if not allow_tls_permutation and self.extension_list != other.extension_list: return False, "TLS extension lists identical but differ in order" # Check the extensions' parameters. for i, ext in enumerate(self.extensions): - if not ext.equals(other.extensions[i]): + if allow_tls_permutation and self._is_permuted_extension(ext): + # If TLS extension permutation is enabled, locate this extension + # in the other signature by type. + other_ext = next( + e for e in other.extensions if e.ext_type == ext.ext_type + ) + else: + other_ext = other.extensions[i] + if not ext.equals(other_ext): ours = ext.to_dict() ours.pop("type") - theirs = other.extensions[i].to_dict() + theirs = other_ext.to_dict() theirs.pop("type") msg = (f"TLS extension {ext.ext_type.name} is different. " f"{ours} != {theirs}") @@ -674,7 +696,11 @@ class TLSClientHelloSignature(): return True, None - def _equals(self, other: 'TLSClientHelloSignature', reason: bool = False): + def _equals( + self, + other: 'TLSClientHelloSignature', + allow_tls_permutation: bool = False + ): """Check if another TLSClientHelloSignature is identical.""" if self.record_version != other.record_version: msg = (f"TLS record versions differ: " @@ -700,20 +726,32 @@ class TLSClientHelloSignature(): msg = f"TLS compression methods differ in contents or order. " return False, msg - return self._compare_extensions(other) + return self._compare_extensions(other, allow_tls_permutation) - def equals(self, other: 'TLSClientHelloSignature', reason: bool = False): + def equals( + self, + other: 'TLSClientHelloSignature', + allow_tls_permutation: bool = False, + reason: bool = False + ): """Checks whether two Client Hello messages have the same signature. Parameters ---------- other : TLSClientHelloSignature The signature of the other Client Hello message. + allow_tls_permutation : bool + Allow TLS extension permutations. If set to True, and the TLS + extensions are identical between the signatures but differ in + order, the signatures will be considered equal. reason : bool If True, returns an additional string describing the reason of the difference in case of a difference, and None otherwise. """ - equal, msg = self._equals(other) + equal, msg = self._equals( + other, + allow_tls_permutation=allow_tls_permutation + ) if reason: return equal, msg else: @@ -939,13 +977,17 @@ class BrowserSignature: http2 : HTTP2Signature The HTTP/2 signature of the browser. Can be None, in which case it is ignored. + options: dict + Optional parameters specifying how to """ def __init__(self, tls_client_hello: TLSClientHelloSignature = None, - http2: HTTP2Signature = None): + http2: HTTP2Signature = None, + options: Dict = None): self.tls_client_hello = tls_client_hello self.http2 = http2 + self.options = options def _equals(self, other: 'BrowserSignature'): # If one is None, so must be the other @@ -968,6 +1010,14 @@ class BrowserSignature: if not equal: return equal, msg + if (self.options is None) != (other.options is None): + return False, "Options present in one signature but not the other" + + if self.options is not None: + if self.options != other.options: + msg = (f"Options differ: {self.options} != {other.options}") + return False, msg + return True, None def equals(self, other: 'BrowserSignature', reason: bool = False): @@ -990,6 +1040,8 @@ class BrowserSignature: def to_dict(self): """Serialize to a dict object.""" d = {} + if self.options is not None: + d["options"] = self.options if self.tls_client_hello is not None: d["tls_client_hello"] = self.tls_client_hello.to_dict() if self.http2 is not None: @@ -1011,4 +1063,8 @@ class BrowserSignature: else: http2 = None - return BrowserSignature(tls_client_hello=tls_client_hello, http2=http2) + return BrowserSignature( + tls_client_hello=tls_client_hello, + http2=http2, + options=d.get("options") + ) diff --git a/tests/signatures/chrome.yaml b/tests/signatures/chrome.yaml index 1353a18..d023227 100644 --- a/tests/signatures/chrome.yaml +++ b/tests/signatures/chrome.yaml @@ -576,6 +576,104 @@ signature: - 'accept-encoding: gzip, deflate, br' - 'accept-language: en-US,en;q=0.9' --- +name: chrome_110.0.5481.177_win10 +browser: + name: chrome + version: 110.0.5481.177 + os: win10 + mode: regular +signature: + options: + tls_permute_extensions: true + tls_client_hello: + record_version: 'TLS_VERSION_1_0' + handshake_version: 'TLS_VERSION_1_2' + session_id_length: 32 + ciphersuites: [ + 'GREASE', + 0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, + 0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, + 0x0035 + ] + comp_methods: [0x00] + extensions: + - type: GREASE + length: 0 + - type: server_name + - type: extended_master_secret + length: 0 + - type: renegotiation_info + length: 1 + - type: supported_groups + length: 10 + supported_groups: [ + 'GREASE', + 0x001d, 0x0017, 0x0018 + ] + - type: ec_point_formats + length: 2 + ec_point_formats: [0] + - type: session_ticket + length: 0 + - type: application_layer_protocol_negotiation + length: 14 + alpn_list: ['h2', 'http/1.1'] + - type: status_request + length: 5 + status_request_type: 0x01 + - type: signature_algorithms + length: 18 + sig_hash_algs: [ + 0x0403, 0x0804, 0x0401, 0x0503, + 0x0805, 0x0501, 0x0806, 0x0601 + ] + - type: signed_certificate_timestamp + length: 0 + - type: keyshare + length: 43 + key_shares: + - group: GREASE + length: 1 + - group: 29 + length: 32 + - type: psk_key_exchange_modes + length: 2 + psk_ke_mode: 1 + - type: supported_versions + length: 7 + supported_versions: [ + 'GREASE', 'TLS_VERSION_1_3', 'TLS_VERSION_1_2' + ] + - type: compress_certificate + length: 3 + algorithms: [0x02] + - type: application_settings + length: 5 + alps_alpn_list: ['h2'] + - type: GREASE + length: 1 + data: !!binary AA== + - type: padding + http2: + pseudo_headers: + - ':method' + - ':authority' + - ':scheme' + - ':path' + headers: + - 'sec-ch-ua: "Chromium";v="110", "Not A(Brand";v="24", "Google Chrome";v="110"' + - 'sec-ch-ua-mobile: ?0' + - 'sec-ch-ua-platform: "Windows"' + - 'upgrade-insecure-requests: 1' + - 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' + - 'sec-fetch-site: none' + - 'sec-fetch-mode: navigate' + - 'sec-fetch-user: ?1' + - 'sec-fetch-dest: document' + - 'accept-encoding: gzip, deflate, br' + - 'accept-language: en-US,en;q=0.9' +--- name: chrome_99.0.4844.73_android12-pixel6 browser: name: chrome diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index 83f5d5d..d1de267 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -148,6 +148,7 @@ class TestImpersonation: ("curl_chrome101", None, None, "chrome_101.0.4951.67_win10"), ("curl_chrome104", None, None, "chrome_104.0.5112.81_win10"), ("curl_chrome107", None, None, "chrome_107.0.5304.107_win10"), + ("curl_chrome110", None, None, "chrome_110.0.5481.177_win10"), ("curl_chrome99_android", None, None, "chrome_99.0.4844.73_android12-pixel6"), ("curl_edge99", None, None, "edge_99.0.1150.30_win10"), ("curl_edge101", None, None, "edge_101.0.1210.47_win10"), @@ -203,6 +204,14 @@ class TestImpersonation: "libcurl-impersonate-chrome", "chrome_107.0.5304.107_win10" ), + ( + "minicurl", + { + "CURL_IMPERSONATE": "chrome110" + }, + "libcurl-impersonate-chrome", + "chrome_110.0.5481.177_win10" + ), ( "minicurl", { @@ -556,7 +565,15 @@ class TestImpersonation: ["tls_client_hello"] ) - equals, msg = sig.equals(expected_sig, reason=True) + allow_tls_permutation=browser_signatures[expected_signature] \ + ["signature"] \ + .get("options", {}) \ + .get("tls_permute_extensions", False) + equals, msg = sig.equals( + expected_sig, + allow_tls_permutation=allow_tls_permutation, + reason=True + ) assert equals, msg @pytest.mark.asyncio