From 14de5bb0dbe5c1800c51f515736c38036168e2ae Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sun, 15 May 2022 19:55:36 +0300 Subject: [PATCH] Add impersonation support for Chrome 100 and 101 .. and for Edge 101 as well. The TLS fingerprint is identical to previous versions. The HTTP headers have the usual differences in the user agents. One important change though is in the way the HTTP2 SETTINGS frame is formed. Up until Chrome 98, there was an additional randomly-generated setting in the frame. This seems to have been removed since. Therefore it was removed from curl-impersonate as well, and support for Chrome/Edge 98 was deprecated, since supporting both signatures requires a lot of work. --- README.md | 5 +- chrome/{curl_chrome98 => curl_chrome100} | 4 +- chrome/curl_chrome101 | 26 ++ chrome/{curl_edge98 => curl_edge101} | 4 +- chrome/patches/curl-impersonate.patch | 241 +++++++++++-------- tests/signatures.yaml | 288 +++++++++++++++++++++++ tests/test_impersonate.py | 45 ++-- 7 files changed, 490 insertions(+), 123 deletions(-) rename chrome/{curl_chrome98 => curl_chrome100} (89%) create mode 100755 chrome/curl_chrome101 rename chrome/{curl_edge98 => curl_edge101} (87%) diff --git a/README.md b/README.md index 4eb3d12..8f33f9b 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,12 @@ Read the full description in the blog post: [part a](https://lwthiker.com/revers The following browsers can be impersonated. | Browser | Version | Build | OS | Target name | Wrapper script | | --- | --- | --- | --- | --- | --- | -| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 98 | 98.0.4758.102 | Windows 10 | `chrome98` | [curl_chrome98](chrome/curl_chrome98) | | ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 99 | 99.0.4844.51 | Windows 10 | `chrome99` | [curl_chrome99](chrome/curl_chrome99) | +| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 100 | 100.0.4896.75 | Windows 10 | `chrome100` | [curl_chrome100](chrome/curl_chrome100) | +| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 101 | 101.0.4951.67 | Windows 10 | `chrome101` | [curl_chrome101](chrome/curl_chrome101) | | ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 99 | 99.0.4844.73 | Android 12 | `chrome99_android` | [curl_chrome99_android](chrome/curl_chrome99_android) | -| ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") | 98 | 98.0.1108.62 | Windows 10 | `edge98` | [curl_edge98](chrome/curl_edge98) | | ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") | 99 | 99.0.1150.30 | Windows 10 | `edge99` | [curl_edge99](chrome/curl_edge99) | +| ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") | 101 | 101.0.1210.47 | Windows 10 | `edge101` | [curl_edge101](chrome/curl_edge101) | | ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") | 91 ESR | 91.6.0esr | Windows 10 | `ff91esr` | [curl_ff91esr](firefox/curl_ff91esr) | | ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") | 95 | 95.0.2 | Windows 10 | `ff95` | [curl_ff95](firefox/curl_ff95) | | ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") | 98 | 98.0 | Windows 10 | `ff98` | [curl_ff98](firefox/curl_ff98) | diff --git a/chrome/curl_chrome98 b/chrome/curl_chrome100 similarity index 89% rename from chrome/curl_chrome98 rename to chrome/curl_chrome100 index a4c0cb3..cd7cc0e 100755 --- a/chrome/curl_chrome98 +++ b/chrome/curl_chrome100 @@ -8,11 +8,11 @@ dir=`echo "$0" | sed 's%/[^/]*$%%'` # https://wiki.mozilla.org/Security/Cipher_Suites "$dir/curl-impersonate-chrome" \ --ciphers TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA \ - -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"' \ + -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \ -H 'sec-ch-ua-mobile: ?0' \ -H 'sec-ch-ua-platform: "Windows"' \ -H 'Upgrade-Insecure-Requests: 1' \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' \ + -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36' \ -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ -H 'Sec-Fetch-Site: none' \ -H 'Sec-Fetch-Mode: navigate' \ diff --git a/chrome/curl_chrome101 b/chrome/curl_chrome101 new file mode 100755 index 0000000..64ddcc4 --- /dev/null +++ b/chrome/curl_chrome101 @@ -0,0 +1,26 @@ +#!/bin/bash + +# Find the directory of this script +dir=`echo "$0" | sed 's%/[^/]*$%%'` + +# The list of ciphers can be obtained by looking at the Client Hello message in +# Wireshark, then converting it using this reference +# https://wiki.mozilla.org/Security/Cipher_Suites +"$dir/curl-impersonate-chrome" \ + --ciphers TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA \ + -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="101", "Google Chrome";v="101"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "Windows"' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ + -H 'Sec-Fetch-Site: none' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-User: ?1' \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Accept-Encoding: gzip, deflate, br' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + --http2 --false-start --compressed \ + --tlsv1.2 --no-npn --alps \ + --cert-compression brotli \ + "$@" diff --git a/chrome/curl_edge98 b/chrome/curl_edge101 similarity index 87% rename from chrome/curl_edge98 rename to chrome/curl_edge101 index e41ad38..359a848 100755 --- a/chrome/curl_edge98 +++ b/chrome/curl_edge101 @@ -8,11 +8,11 @@ dir=`echo "$0" | sed 's%/[^/]*$%%'` # https://wiki.mozilla.org/Security/Cipher_Suites "$dir/curl-impersonate-chrome" \ --ciphers TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA \ - -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="98", "Microsoft Edge";v="98"' \ + -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="101", "Microsoft Edge";v="101"' \ -H 'sec-ch-ua-mobile: ?0' \ -H 'sec-ch-ua-platform: "Windows"' \ -H 'Upgrade-Insecure-Requests: 1' \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.62' \ + -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.47' \ -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ -H 'Sec-Fetch-Site: none' \ -H 'Sec-Fetch-Mode: navigate' \ diff --git a/chrome/patches/curl-impersonate.patch b/chrome/patches/curl-impersonate.patch index b1d115d..65d8e05 100644 --- a/chrome/patches/curl-impersonate.patch +++ b/chrome/patches/curl-impersonate.patch @@ -244,7 +244,7 @@ index 769363941..cd59ad4b2 100644 CHECKSRC = $(CS_$(V)) diff --git a/lib/easy.c b/lib/easy.c -index 20293a710..23141fe16 100644 +index 20293a710..37cda0992 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -80,6 +80,7 @@ @@ -255,7 +255,7 @@ index 20293a710..23141fe16 100644 /* The last 3 #include files should be in this order */ #include "curl_printf.h" -@@ -282,6 +283,414 @@ void curl_global_cleanup(void) +@@ -282,6 +283,454 @@ void curl_global_cleanup(void) init_flags = 0; } @@ -293,46 +293,6 @@ index 20293a710..23141fe16 100644 + * configurable through curl_easy_setopt() */ +} impersonations[] = { + { -+ .target = "chrome98", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = -+ "TLS_AES_128_GCM_SHA256," -+ "TLS_AES_256_GCM_SHA384," -+ "TLS_CHACHA20_POLY1305_SHA256," -+ "ECDHE-ECDSA-AES128-GCM-SHA256," -+ "ECDHE-RSA-AES128-GCM-SHA256," -+ "ECDHE-ECDSA-AES256-GCM-SHA384," -+ "ECDHE-RSA-AES256-GCM-SHA384," -+ "ECDHE-ECDSA-CHACHA20-POLY1305," -+ "ECDHE-RSA-CHACHA20-POLY1305," -+ "ECDHE-RSA-AES128-SHA," -+ "ECDHE-RSA-AES256-SHA," -+ "AES128-GCM-SHA256," -+ "AES256-GCM-SHA384," -+ "AES128-SHA," -+ "AES256-SHA", -+ .npn = false, -+ .alpn = true, -+ .alps = true, -+ .tls_session_ticket = true, -+ .cert_compression = "brotli", -+ .http_headers = { -+ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"98\", \"Google Chrome\";v=\"98\"", -+ "sec-ch-ua-mobile: ?0", -+ "sec-ch-ua-platform: \"Windows\"", -+ "Upgrade-Insecure-Requests: 1", -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-User: ?1", -+ "Sec-Fetch-Dest: document", -+ "Accept-Encoding: gzip, deflate, br", -+ "Accept-Language: en-US,en;q=0.9" -+ } -+ }, -+ { + .target = "chrome99", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, @@ -373,6 +333,86 @@ index 20293a710..23141fe16 100644 + } + }, + { ++ .target = "chrome100", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "ECDHE-ECDSA-AES128-GCM-SHA256," ++ "ECDHE-RSA-AES128-GCM-SHA256," ++ "ECDHE-ECDSA-AES256-GCM-SHA384," ++ "ECDHE-RSA-AES256-GCM-SHA384," ++ "ECDHE-ECDSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-AES128-SHA," ++ "ECDHE-RSA-AES256-SHA," ++ "AES128-GCM-SHA256," ++ "AES256-GCM-SHA384," ++ "AES128-SHA," ++ "AES256-SHA", ++ .npn = false, ++ .alpn = true, ++ .alps = true, ++ .tls_session_ticket = true, ++ .cert_compression = "brotli", ++ .http_headers = { ++ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"Windows\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br", ++ "Accept-Language: en-US,en;q=0.9" ++ } ++ }, ++ { ++ .target = "chrome101", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "ECDHE-ECDSA-AES128-GCM-SHA256," ++ "ECDHE-RSA-AES128-GCM-SHA256," ++ "ECDHE-ECDSA-AES256-GCM-SHA384," ++ "ECDHE-RSA-AES256-GCM-SHA384," ++ "ECDHE-ECDSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-AES128-SHA," ++ "ECDHE-RSA-AES256-SHA," ++ "AES128-GCM-SHA256," ++ "AES256-GCM-SHA384," ++ "AES128-SHA," ++ "AES256-SHA", ++ .npn = false, ++ .alpn = true, ++ .alps = true, ++ .tls_session_ticket = true, ++ .cert_compression = "brotli", ++ .http_headers = { ++ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"101\", \"Google Chrome\";v=\"101\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"Windows\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br", ++ "Accept-Language: en-US,en;q=0.9" ++ } ++ }, ++ { + .target = "chrome99_android", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, @@ -413,46 +453,6 @@ index 20293a710..23141fe16 100644 + } + }, + { -+ .target = "edge98", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = -+ "TLS_AES_128_GCM_SHA256," -+ "TLS_AES_256_GCM_SHA384," -+ "TLS_CHACHA20_POLY1305_SHA256," -+ "ECDHE-ECDSA-AES128-GCM-SHA256," -+ "ECDHE-RSA-AES128-GCM-SHA256," -+ "ECDHE-ECDSA-AES256-GCM-SHA384," -+ "ECDHE-RSA-AES256-GCM-SHA384," -+ "ECDHE-ECDSA-CHACHA20-POLY1305," -+ "ECDHE-RSA-CHACHA20-POLY1305," -+ "ECDHE-RSA-AES128-SHA," -+ "ECDHE-RSA-AES256-SHA," -+ "AES128-GCM-SHA256," -+ "AES256-GCM-SHA384," -+ "AES128-SHA," -+ "AES256-SHA", -+ .npn = false, -+ .alpn = true, -+ .alps = true, -+ .tls_session_ticket = true, -+ .cert_compression = "brotli", -+ .http_headers = { -+ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"98\", \"Microsoft Edge\";v=\"98\"", -+ "sec-ch-ua-mobile: ?0", -+ "sec-ch-ua-platform: \"Windows\"", -+ "Upgrade-Insecure-Requests: 1", -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.62", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-User: ?1", -+ "Sec-Fetch-Dest: document", -+ "Accept-Encoding: gzip, deflate, br", -+ "Accept-Language: en-US,en;q=0.9" -+ } -+ }, -+ { + .target = "edge99", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, @@ -493,6 +493,46 @@ index 20293a710..23141fe16 100644 + } + }, + { ++ .target = "edge101", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "ECDHE-ECDSA-AES128-GCM-SHA256," ++ "ECDHE-RSA-AES128-GCM-SHA256," ++ "ECDHE-ECDSA-AES256-GCM-SHA384," ++ "ECDHE-RSA-AES256-GCM-SHA384," ++ "ECDHE-ECDSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-AES128-SHA," ++ "ECDHE-RSA-AES256-SHA," ++ "AES128-GCM-SHA256," ++ "AES256-GCM-SHA384," ++ "AES128-SHA," ++ "AES256-SHA", ++ .npn = false, ++ .alpn = true, ++ .alps = true, ++ .tls_session_ticket = true, ++ .cert_compression = "brotli", ++ .http_headers = { ++ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"101\", \"Microsoft Edge\";v=\"101\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"Windows\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.47", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br", ++ "Accept-Language: en-US,en;q=0.9" ++ } ++ }, ++ { + .target = "safari15_3", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_0 | CURL_SSLVERSION_MAX_DEFAULT, @@ -670,7 +710,7 @@ index 20293a710..23141fe16 100644 /* * curl_easy_init() is the external interface to alloc, setup and init an * easy handle that is returned. If anything goes wrong, NULL is returned. -@@ -290,6 +699,7 @@ struct Curl_easy *curl_easy_init(void) +@@ -290,6 +739,7 @@ struct Curl_easy *curl_easy_init(void) { CURLcode result; struct Curl_easy *data; @@ -678,7 +718,7 @@ index 20293a710..23141fe16 100644 /* Make sure we inited the global SSL stuff */ if(!initialized) { -@@ -308,6 +718,22 @@ struct Curl_easy *curl_easy_init(void) +@@ -308,6 +758,22 @@ struct Curl_easy *curl_easy_init(void) return NULL; } @@ -701,7 +741,7 @@ index 20293a710..23141fe16 100644 return data; } -@@ -878,6 +1304,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) +@@ -878,6 +1344,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) outcurl->state.referer_alloc = TRUE; } @@ -921,7 +961,7 @@ index f08a343e3..2bbce4b23 100644 if(result) return result; diff --git a/lib/http.h b/lib/http.h -index b4aaba2a2..1cf65c4b1 100644 +index b4aaba2a2..0d716640c 100644 --- a/lib/http.h +++ b/lib/http.h @@ -278,7 +278,8 @@ struct http_conn { @@ -930,12 +970,12 @@ index b4aaba2a2..1cf65c4b1 100644 /* list of settings that will be sent */ - nghttp2_settings_entry local_settings[3]; + /* curl-impersonate: Align HTTP/2 settings to Chrome's */ -+ nghttp2_settings_entry local_settings[5]; ++ nghttp2_settings_entry local_settings[4]; size_t local_settings_num; #else int unused; /* prevent a compiler warning */ diff --git a/lib/http2.c b/lib/http2.c -index e74400a4c..09ae839de 100644 +index e74400a4c..b0e6674f2 100644 --- a/lib/http2.c +++ b/lib/http2.c @@ -41,6 +41,7 @@ @@ -955,7 +995,7 @@ index e74400a4c..09ae839de 100644 #ifdef DEBUG_HTTP2 #define H2BUGF(x) x -@@ -1193,16 +1194,27 @@ static void populate_settings(struct Curl_easy *data, +@@ -1193,16 +1194,30 @@ static void populate_settings(struct Curl_easy *data, { nghttp2_settings_entry *iv = httpc->local_settings; @@ -982,15 +1022,18 @@ index e74400a4c..09ae839de 100644 + // iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; + // iv[2].value = data->multi->push_cb != NULL; + -+ // Looks like random setting set by Chrome, maybe similar to TLS GREASE. */ -+ Curl_rand(data, (unsigned char *)&iv[4].settings_id, sizeof(iv[4].settings_id)); -+ Curl_rand(data, (unsigned char *)&iv[4].value, sizeof(iv[4].value)); ++ // curl-impersonate: ++ // Up until Chrome 98, there was a randomly chosen setting number in the ++ // HTTP2 SETTINGS frame. This might be something similar to TLS GREASE. ++ // However, it seems to have been removed since. ++ // Curl_rand(data, (unsigned char *)&iv[4].settings_id, sizeof(iv[4].settings_id)); ++ // Curl_rand(data, (unsigned char *)&iv[4].value, sizeof(iv[4].value)); + -+ httpc->local_settings_num = 5; ++ httpc->local_settings_num = 4; } void Curl_http2_done(struct Curl_easy *data, bool premature) -@@ -1816,10 +1828,6 @@ static ssize_t http2_recv(struct Curl_easy *data, int sockindex, +@@ -1816,10 +1831,6 @@ static ssize_t http2_recv(struct Curl_easy *data, int sockindex, return -1; } @@ -1001,7 +1044,7 @@ index e74400a4c..09ae839de 100644 /* USHRT_MAX is 65535 == 0xffff */ #define HEADER_OVERFLOW(x) \ (x.namelen > 0xffff || x.valuelen > 0xffff - x.namelen) -@@ -1890,6 +1898,53 @@ static header_instruction inspect_header(const char *name, size_t namelen, +@@ -1890,6 +1901,53 @@ static header_instruction inspect_header(const char *name, size_t namelen, } } @@ -1055,7 +1098,7 @@ index e74400a4c..09ae839de 100644 static ssize_t http2_send(struct Curl_easy *data, int sockindex, const void *mem, size_t len, CURLcode *err) { -@@ -1905,6 +1960,7 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex, +@@ -1905,6 +1963,7 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex, nghttp2_nv *nva = NULL; size_t nheader; size_t i; @@ -1063,7 +1106,7 @@ index e74400a4c..09ae839de 100644 size_t authority_idx; char *hdbuf = (char *)mem; char *end, *line_end; -@@ -2010,12 +2066,21 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex, +@@ -2010,12 +2069,21 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex, end = memchr(hdbuf, ' ', line_end - hdbuf); if(!end || end == hdbuf) goto fail; @@ -1091,7 +1134,7 @@ index e74400a4c..09ae839de 100644 failf(data, "Failed sending HTTP request: Header overflow"); goto fail; } -@@ -2032,25 +2097,35 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex, +@@ -2032,25 +2100,35 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex, } if(!end || end == hdbuf) goto fail; @@ -1140,7 +1183,7 @@ index e74400a4c..09ae839de 100644 failf(data, "Failed sending HTTP request: Header overflow"); goto fail; } -@@ -2117,10 +2192,13 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex, +@@ -2117,10 +2195,13 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex, ++i; } diff --git a/tests/signatures.yaml b/tests/signatures.yaml index 2d581c8..a2aa4d4 100644 --- a/tests/signatures.yaml +++ b/tests/signatures.yaml @@ -199,6 +199,198 @@ signature: - 'accept-encoding: gzip, deflate, br' - 'accept-language: en-US,en;q=0.9' --- +name: chrome_100.0.4896.127_win10 +browser: + name: chrome + version: 100.0.4896.127 + os: win10 + mode: regular +signature: + tls_client_hello: + record_version: 'TLS_VERSION_1_0' + handshake_version: 'TLS_VERSION_1_2' + session_id_length: 32 + ciphersuites: [ + 'GREASE', + 0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, + 0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, + 0x0035 + ] + comp_methods: [0x00] + extensions: + - type: GREASE + length: 0 + - type: server_name + - type: extended_master_secret + length: 0 + - type: renegotiation_info + length: 1 + - type: supported_groups + length: 10 + supported_groups: [ + 'GREASE', + 0x001d, 0x0017, 0x0018 + ] + - type: ec_point_formats + length: 2 + ec_point_formats: [0] + - type: session_ticket + length: 0 + - type: application_layer_protocol_negotiation + length: 14 + alpn_list: ['h2', 'http/1.1'] + - type: status_request + length: 5 + status_request_type: 0x01 + - type: signature_algorithms + length: 18 + sig_hash_algs: [ + 0x0403, 0x0804, 0x0401, 0x0503, + 0x0805, 0x0501, 0x0806, 0x0601 + ] + - type: signed_certificate_timestamp + length: 0 + - type: keyshare + length: 43 + key_shares: + - group: GREASE + length: 1 + - group: 29 + length: 32 + - type: psk_key_exchange_modes + length: 2 + psk_ke_mode: 1 + - type: supported_versions + length: 7 + supported_versions: [ + 'GREASE', 'TLS_VERSION_1_3', 'TLS_VERSION_1_2' + ] + - type: compress_certificate + length: 3 + algorithms: [0x02] + - type: application_settings + length: 5 + alps_alpn_list: ['h2'] + - type: GREASE + length: 1 + data: !!binary AA== + - type: padding + http2: + pseudo_headers: + - ':method' + - ':authority' + - ':scheme' + - ':path' + headers: + - 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' + - 'sec-ch-ua-mobile: ?0' + - 'sec-ch-ua-platform: "Windows"' + - 'upgrade-insecure-requests: 1' + - 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' + - 'sec-fetch-site: none' + - 'sec-fetch-mode: navigate' + - 'sec-fetch-user: ?1' + - 'sec-fetch-dest: document' + - 'accept-encoding: gzip, deflate, br' + - 'accept-language: en-US,en;q=0.9' +--- +name: chrome_101.0.4951.67_win10 +browser: + name: chrome + version: 101.0.4951.67 + os: win10 + mode: regular +signature: + tls_client_hello: + record_version: 'TLS_VERSION_1_0' + handshake_version: 'TLS_VERSION_1_2' + session_id_length: 32 + ciphersuites: [ + 'GREASE', + 0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, + 0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, + 0x0035 + ] + comp_methods: [0x00] + extensions: + - type: GREASE + length: 0 + - type: server_name + - type: extended_master_secret + length: 0 + - type: renegotiation_info + length: 1 + - type: supported_groups + length: 10 + supported_groups: [ + 'GREASE', + 0x001d, 0x0017, 0x0018 + ] + - type: ec_point_formats + length: 2 + ec_point_formats: [0] + - type: session_ticket + length: 0 + - type: application_layer_protocol_negotiation + length: 14 + alpn_list: ['h2', 'http/1.1'] + - type: status_request + length: 5 + status_request_type: 0x01 + - type: signature_algorithms + length: 18 + sig_hash_algs: [ + 0x0403, 0x0804, 0x0401, 0x0503, + 0x0805, 0x0501, 0x0806, 0x0601 + ] + - type: signed_certificate_timestamp + length: 0 + - type: keyshare + length: 43 + key_shares: + - group: GREASE + length: 1 + - group: 29 + length: 32 + - type: psk_key_exchange_modes + length: 2 + psk_ke_mode: 1 + - type: supported_versions + length: 7 + supported_versions: [ + 'GREASE', 'TLS_VERSION_1_3', 'TLS_VERSION_1_2' + ] + - type: compress_certificate + length: 3 + algorithms: [0x02] + - type: application_settings + length: 5 + alps_alpn_list: ['h2'] + - type: GREASE + length: 1 + data: !!binary AA== + - type: padding + http2: + pseudo_headers: + - ':method' + - ':authority' + - ':scheme' + - ':path' + headers: + - 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="101", "Google Chrome";v="101"' + - 'sec-ch-ua-mobile: ?0' + - 'sec-ch-ua-platform: "Windows"' + - 'upgrade-insecure-requests: 1' + - 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' + - 'sec-fetch-site: none' + - 'sec-fetch-mode: navigate' + - 'sec-fetch-user: ?1' + - 'sec-fetch-dest: document' + - 'accept-encoding: gzip, deflate, br' + - 'accept-language: en-US,en;q=0.9' +--- name: chrome_99.0.4844.73_android12-pixel6 browser: name: chrome @@ -487,6 +679,102 @@ signature: - 'accept-encoding: gzip, deflate, br' - 'accept-language: en-US,en;q=0.9' --- +name: edge_101.0.1210.47_win10 +browser: + name: edge + version: 101.0.1210.47 + os: win10 + mode: regular +signature: + tls_client_hello: + record_version: 'TLS_VERSION_1_0' + handshake_version: 'TLS_VERSION_1_2' + session_id_length: 32 + ciphersuites: [ + 'GREASE', + 0x1301, 0x1302, 0x1303, 0xc02b, 0xc02f, 0xc02c, 0xc030, + 0xcca9, 0xcca8, 0xc013, 0xc014, 0x009c, 0x009d, 0x002f, + 0x0035 + ] + comp_methods: [0x00] + extensions: + - type: GREASE + length: 0 + - type: server_name + - type: extended_master_secret + length: 0 + - type: renegotiation_info + length: 1 + - type: supported_groups + length: 10 + supported_groups: [ + 'GREASE', + 0x001d, 0x0017, 0x0018 + ] + - type: ec_point_formats + length: 2 + ec_point_formats: [0] + - type: session_ticket + length: 0 + - type: application_layer_protocol_negotiation + length: 14 + alpn_list: ['h2', 'http/1.1'] + - type: status_request + length: 5 + status_request_type: 0x01 + - type: signature_algorithms + length: 18 + sig_hash_algs: [ + 0x0403, 0x0804, 0x0401, 0x0503, + 0x0805, 0x0501, 0x0806, 0x0601 + ] + - type: signed_certificate_timestamp + length: 0 + - type: keyshare + length: 43 + key_shares: + - group: GREASE + length: 1 + - group: 29 + length: 32 + - type: psk_key_exchange_modes + length: 2 + psk_ke_mode: 1 + - type: supported_versions + length: 7 + supported_versions: [ + 'GREASE', 'TLS_VERSION_1_3', 'TLS_VERSION_1_2' + ] + - type: compress_certificate + length: 3 + algorithms: [0x02] + - type: application_settings + length: 5 + alps_alpn_list: ['h2'] + - type: GREASE + length: 1 + data: !!binary AA== + - type: padding + http2: + pseudo_headers: + - ':method' + - ':authority' + - ':scheme' + - ':path' + headers: + - 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="101", "Microsoft Edge";v="101"' + - 'sec-ch-ua-mobile: ?0' + - 'sec-ch-ua-platform: "Windows"' + - 'upgrade-insecure-requests: 1' + - 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.47' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' + - 'sec-fetch-site: none' + - 'sec-fetch-mode: navigate' + - 'sec-fetch-user: ?1' + - 'sec-fetch-dest: document' + - 'accept-encoding: gzip, deflate, br' + - 'accept-language: en-US,en;q=0.9' +--- name: firefox_91.6.0esr_win10 browser: name: firefox diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index 07ecda3..45232cf 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -129,11 +129,12 @@ class TestImpersonation: # List of binaries and their expected signatures CURL_BINARIES_AND_SIGNATURES = [ # Test wrapper scripts - ("curl_chrome98", None, None, "chrome_98.0.4758.102_win10"), ("curl_chrome99", None, None, "chrome_99.0.4844.51_win10"), + ("curl_chrome100", None, None, "chrome_100.0.4896.127_win10"), + ("curl_chrome101", None, None, "chrome_101.0.4951.67_win10"), ("curl_chrome99_android", None, None, "chrome_99.0.4844.73_android12-pixel6"), - ("curl_edge98", None, None, "edge_98.0.1108.62_win10"), ("curl_edge99", None, None, "edge_99.0.1150.30_win10"), + ("curl_edge101", None, None, "edge_101.0.1210.47_win10"), ("curl_safari15_3", None, None, "safari_15.3_macos11.6.4"), ("curl_ff91esr", None, None, "firefox_91.6.0esr_win10"), ("curl_ff95", None, None, "firefox_95.0.2_win10"), @@ -143,14 +144,6 @@ class TestImpersonation: # Test libcurl-impersonate by loading it with LD_PRELOAD to an app # linked against the regular libcurl and setting the # CURL_IMPERSONATE env var. - ( - "minicurl", - { - "CURL_IMPERSONATE": "chrome98" - }, - "libcurl-impersonate-chrome", - "chrome_98.0.4758.102_win10" - ), ( "minicurl", { @@ -159,6 +152,22 @@ class TestImpersonation: "libcurl-impersonate-chrome", "chrome_99.0.4844.51_win10" ), + ( + "minicurl", + { + "CURL_IMPERSONATE": "chrome100" + }, + "libcurl-impersonate-chrome", + "chrome_100.0.4896.127_win10" + ), + ( + "minicurl", + { + "CURL_IMPERSONATE": "chrome101" + }, + "libcurl-impersonate-chrome", + "chrome_101.0.4951.67_win10" + ), ( "minicurl", { @@ -167,14 +176,6 @@ class TestImpersonation: "libcurl-impersonate-chrome", "chrome_99.0.4844.73_android12-pixel6" ), - ( - "minicurl", - { - "CURL_IMPERSONATE": "edge98" - }, - "libcurl-impersonate-chrome", - "edge_98.0.1108.62_win10" - ), ( "minicurl", { @@ -183,6 +184,14 @@ class TestImpersonation: "libcurl-impersonate-chrome", "edge_99.0.1150.30_win10" ), + ( + "minicurl", + { + "CURL_IMPERSONATE": "edge101", + }, + "libcurl-impersonate-chrome", + "edge_101.0.1210.47_win10" + ), ( "minicurl", {