From d22e09eb4a2870dc65fdbc9d0eba2fd67edc3073 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sat, 26 Feb 2022 12:21:24 +0200 Subject: [PATCH 1/7] Compile libcurl.so in addition to curl-impersonate In addition to the statically linked curl-impersonate binary, compile libcurl.so for dynamic linking as well. The output file is saved at /build/out/libcurl-impersonate.so. Also strip the output binaries to reduce their size. For now, support is for the Chrome build only. --- chrome/Dockerfile | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/chrome/Dockerfile b/chrome/Dockerfile index c1cf986..00f2925 100644 --- a/chrome/Dockerfile +++ b/chrome/Dockerfile @@ -47,7 +47,7 @@ RUN cd ${NGHTTP2_VERSION} && \ # Compile nghttp2 RUN cd ${NGHTTP2_VERSION} && \ - ./configure && \ + ./configure --with-pic && \ make && make install # Download curl. @@ -67,13 +67,29 @@ RUN cd ${CURL_VERSION} && \ ./configure --with-openssl=/build/boringssl/build --enable-static --disable-shared --with-nghttp2=/usr/local LIBS="-pthread" CFLAGS="-I/build/boringssl/build" USE_CURL_SSLKEYLOGFILE=true && \ make -# 'xxd' is needed for the wrapper script -RUN apt-get install -y xxd - RUN mkdir out && \ - cp ${CURL_VERSION}/src/curl out/curl-impersonate + cp ${CURL_VERSION}/src/curl out/curl-impersonate && \ + strip out/curl-impersonate -# Wrapper script +# Re-compile libcurl dynamically +RUN cd ${CURL_VERSION} && \ + ./configure --enable-versioned-symbols \ + --with-openssl=/build/boringssl/build \ + --with-nghttp2=/usr/local \ + LIBS="-pthread" \ + CFLAGS="-I/build/boringssl/build" \ + USE_CURL_SSLKEYLOGFILE=true && \ + make clean && make + +# Rename to 'libcurl-impersonate' to avoid confusion, and recreate the +# symbolic links. +RUN ver=$(readlink -f curl-7.81.0/lib/.libs/libcurl.so | sed 's/.*so\.//') && \ + major=$(echo -n $ver | cut -d'.' -f1) && \ + cp "${CURL_VERSION}/lib/.libs/libcurl.so.$ver" "out/libcurl-impersonate.so.$ver" && \ + ln -s "libcurl-impersonate.so.$ver" "out/libcurl-impersonate.so.$major" && \ + ln -s "libcurl-impersonate.so.$ver" "out/libcurl-impersonate.so" && \ + strip "out/libcurl-impersonate.so.$ver" + +# Wrapper scripts COPY curl_chrome* out/ - -RUN chmod +x out/* +RUN chmod +x out/curl_* From 48415a4b00354c5a9c95d35d78172b0cb28d15ff Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sat, 26 Feb 2022 12:23:09 +0200 Subject: [PATCH 2/7] Add impersonation support to libcurl Patch generated from https://github.com/lwthiker/curl/commit/e8cd43c8ebd2624b3c635ec96b165c28f675cdf2 Add curl_easy_impersonate() API function that sets the needed options on the curl 'easy' handle. It sets the various TLS options needed for impersonation and the HTTP headers that the browser sends by default. In addition, libcurl will check for the environment variable CURL_IMPERSONATE when curl_easy_init() is called, and if it exists it will call curl_easy_impersonate() internally. This theoretically allows replacing an existing libcurl by setting the LD_LIBRARY_PATH and CURL_IMPERSONATE env vars, without having to recompile the app. --- chrome/patches/curl-impersonate.patch | 489 ++++++++++++++++++++++++++ 1 file changed, 489 insertions(+) diff --git a/chrome/patches/curl-impersonate.patch b/chrome/patches/curl-impersonate.patch index b8f8512..d5f6df6 100644 --- a/chrome/patches/curl-impersonate.patch +++ b/chrome/patches/curl-impersonate.patch @@ -21,6 +21,398 @@ index 63e320236..deb054300 100644 AC_MSG_NOTICE([-L is $LD_H2]) LDFLAGS="$LDFLAGS $LD_H2" +diff --git a/include/curl/curl.h b/include/curl/curl.h +index 7b69ce2d6..fe4bb36b9 100644 +--- a/include/curl/curl.h ++++ b/include/curl/curl.h +@@ -2135,6 +2135,10 @@ typedef enum { + /* Set MIME option flags. */ + CURLOPT(CURLOPT_MIME_OPTIONS, CURLOPTTYPE_LONG, 315), + ++ /* curl-impersonate: A list of headers used by the impersonated browser. ++ * If given, merged with CURLOPT_HTTPHEADER. */ ++ CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 316), ++ + CURLOPT_LASTENTRY /* the last unused */ + } CURLoption; + +diff --git a/include/curl/easy.h b/include/curl/easy.h +index 2dbfb26b5..e0bf86169 100644 +--- a/include/curl/easy.h ++++ b/include/curl/easy.h +@@ -41,6 +41,15 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); + CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); + CURL_EXTERN void curl_easy_cleanup(CURL *curl); + ++/* ++ * curl-impersonate: Tell libcurl to impersonate a browser. ++ * This is a wrapper function that calls curl_easy_setopt() ++ * multiple times with all the parameters required. That's also why it was ++ * created as a separate API function and not just as another option to ++ * curl_easy_setopt(). ++ */ ++CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target); ++ + /* + * NAME curl_easy_getinfo() + * +diff --git a/lib/easy.c b/lib/easy.c +index 20293a710..df3e66bc0 100644 +--- a/lib/easy.c ++++ b/lib/easy.c +@@ -80,6 +80,7 @@ + #include "dynbuf.h" + #include "altsvc.h" + #include "hsts.h" ++#include "strcase.h" + + /* The last 3 #include files should be in this order */ + #include "curl_printf.h" +@@ -282,6 +283,126 @@ void curl_global_cleanup(void) + init_flags = 0; + } + ++/* ++ * curl-impersonate: Options to be set for each supported target browser. ++ * Note: this does not include the HTTP headers, which are handled separately ++ * in Curl_http(). ++ */ ++#define IMPERSONATE_MAX_HEADERS 32 ++static const struct impersonate_opts { ++ const char *target; ++ int httpversion; ++ int ssl_version; ++ const char *ciphers; ++ const char *http_headers[IMPERSONATE_MAX_HEADERS]; ++ /* Other TLS options will come here in the future once they are ++ * configurable through curl_easy_setopt() */ ++} impersonations[] = { ++ { ++ .target = "chrome98", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "ECDHE-ECDSA-AES128-GCM-SHA256," ++ "ECDHE-RSA-AES128-GCM-SHA256," ++ "ECDHE-ECDSA-AES256-GCM-SHA384," ++ "ECDHE-RSA-AES256-GCM-SHA384," ++ "ECDHE-ECDSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-AES128-SHA," ++ "ECDHE-RSA-AES256-SHA," ++ "AES128-GCM-SHA256," ++ "AES256-GCM-SHA384," ++ "AES128-SHA," ++ "AES256-SHA", ++ .http_headers = { ++ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"98\", \"Google Chrome\";v=\"98\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"Windows\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br", ++ "Accept-Language: en-US,en;q=0.9" ++ } ++ } ++}; ++ ++#define NUM_IMPERSONATIONS \ ++ sizeof(impersonations) / sizeof(impersonations[0]) ++ ++/* ++ * curl-impersonate: ++ * Call curl_easy_setopt() with all the needed options as defined in the ++ * 'impersonations' array. ++ * */ ++CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target) ++{ ++ int i; ++ int ret; ++ const struct impersonate_opts *opts = NULL; ++ struct curl_slist *headers = NULL; ++ ++ for(i = 0; i < NUM_IMPERSONATIONS; i++) { ++ if (Curl_strncasecompare(target, ++ impersonations[i].target, ++ strlen(impersonations[i].target))) { ++ opts = &impersonations[i]; ++ break; ++ } ++ } ++ ++ if(!opts) { ++ DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n", ++ target)); ++ return CURLE_BAD_FUNCTION_ARGUMENT; ++ } ++ ++ if(opts->httpversion != CURL_HTTP_VERSION_NONE) { ++ ret = curl_easy_setopt(data, CURLOPT_HTTP_VERSION, opts->httpversion); ++ if(ret) ++ return ret; ++ } ++ ++ if (opts->ssl_version != CURL_SSLVERSION_DEFAULT) { ++ ret = curl_easy_setopt(data, CURLOPT_SSLVERSION, opts->ssl_version); ++ if(ret) ++ return ret; ++ } ++ ++ if(opts->ciphers) { ++ ret = curl_easy_setopt(data, CURLOPT_SSL_CIPHER_LIST, opts->ciphers); ++ if (ret) ++ return ret; ++ } ++ ++ /* Build a linked list out of the static array of headers. */ ++ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) { ++ if(opts->http_headers[i]) { ++ headers = curl_slist_append(headers, opts->http_headers[i]); ++ if(!headers) { ++ return CURLE_OUT_OF_MEMORY; ++ } ++ } ++ } ++ ++ if(headers) { ++ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers); ++ curl_slist_free_all(headers); ++ if(ret) ++ return ret; ++ } ++ ++ return CURLE_OK; ++} ++ + /* + * curl_easy_init() is the external interface to alloc, setup and init an + * easy handle that is returned. If anything goes wrong, NULL is returned. +@@ -290,6 +411,7 @@ struct Curl_easy *curl_easy_init(void) + { + CURLcode result; + struct Curl_easy *data; ++ char *target; + + /* Make sure we inited the global SSL stuff */ + if(!initialized) { +@@ -308,6 +430,22 @@ struct Curl_easy *curl_easy_init(void) + return NULL; + } + ++ /* ++ * curl-impersonate: Hook into curl_easy_init() to set the required options ++ * from an environment variable. ++ * This is a bit hacky but allows seamless integration of libcurl-impersonate ++ * without code modifications to the app. ++ */ ++ target = curl_getenv("CURL_IMPERSONATE"); ++ if(target) { ++ result = curl_easy_impersonate(data, target); ++ free(target); ++ if(result) { ++ Curl_close(&data); ++ return NULL; ++ } ++ } ++ + return data; + } + +@@ -878,6 +1016,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) + outcurl->state.referer_alloc = TRUE; + } + ++ if(data->state.base_headers) { ++ outcurl->state.base_headers = ++ Curl_slist_duplicate(data->state.base_headers); ++ if(!outcurl->state.base_headers) ++ goto fail; ++ } ++ + /* Reinitialize an SSL engine for the new handle + * note: the engine name has already been copied by dupset */ + if(outcurl->set.str[STRING_SSL_ENGINE]) { +diff --git a/lib/easyoptions.c b/lib/easyoptions.c +index 04871ad1e..cd5998146 100644 +--- a/lib/easyoptions.c ++++ b/lib/easyoptions.c +@@ -130,6 +130,7 @@ struct curl_easyoption Curl_easyopts[] = { + {"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0}, + {"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0}, + {"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0}, ++ {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0}, + {"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0}, + {"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0}, + {"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0}, +diff --git a/lib/http.c b/lib/http.c +index f08a343e3..879151dd2 100644 +--- a/lib/http.c ++++ b/lib/http.c +@@ -84,6 +84,7 @@ + #include "altsvc.h" + #include "hsts.h" + #include "c-hyper.h" ++#include "slist.h" + + /* The last 3 #include files should be in this order */ + #include "curl_printf.h" +@@ -1795,6 +1796,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, + int numlists = 1; /* by default */ + int i; + ++ /* ++ * curl-impersonate: Use the merged list of headers if it exists (i.e. when ++ * the CURLOPT_HTTPBASEHEADER option was set. ++ */ ++ struct curl_slist *noproxyheaders = ++ (data->state.merged_headers ? ++ data->state.merged_headers : ++ data->set.headers); ++ + #ifndef CURL_DISABLE_PROXY + enum proxy_use proxy; + +@@ -1806,10 +1816,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, + + switch(proxy) { + case HEADER_SERVER: +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + break; + case HEADER_PROXY: +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + if(data->set.sep_headers) { + h[1] = data->set.proxyheaders; + numlists++; +@@ -1819,12 +1829,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, + if(data->set.sep_headers) + h[0] = data->set.proxyheaders; + else +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + break; + } + #else + (void)is_connect; +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + #endif + + /* loop through one or two lists */ +@@ -2059,6 +2069,92 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn, + *reqp = httpreq; + } + ++/* ++ * curl-impersonate: ++ * Create a new linked list of headers. ++ * The new list is a merge between the "base" headers and the application given ++ * headers. The "base" headers contain curl-impersonate's list of headers ++ * used by default by the impersonated browser. ++ * ++ * The application given headers will override the "base" headers if supplied. ++ */ ++CURLcode Curl_http_merge_headers(struct Curl_easy *data) ++{ ++ int i; ++ int ret; ++ struct curl_slist *head; ++ struct curl_slist *dup = NULL; ++ struct curl_slist *new_list = NULL; ++ ++ if (!data->state.base_headers) ++ return CURLE_OK; ++ ++ /* Duplicate the list for temporary use. */ ++ if (data->set.headers) { ++ dup = Curl_slist_duplicate(data->set.headers); ++ if(!dup) ++ return CURLE_OUT_OF_MEMORY; ++ } ++ ++ for(head = data->state.base_headers; head; head = head->next) { ++ char *sep; ++ size_t prefix_len; ++ bool found = FALSE; ++ struct curl_slist *head2; ++ ++ sep = strchr(head->data, ':'); ++ if(!sep) ++ continue; ++ ++ prefix_len = sep - head->data; ++ ++ /* Check if this header was added by the application. */ ++ for(head2 = dup; head2; head2 = head2->next) { ++ if(head2->data && ++ strncasecompare(head2->data, head->data, prefix_len) && ++ Curl_headersep(head2->data[prefix_len]) ) { ++ new_list = curl_slist_append(new_list, head2->data); ++ /* Free and set to NULL to mark that it's been added. */ ++ Curl_safefree(head2->data); ++ found = TRUE; ++ break; ++ } ++ } ++ ++ if (!found) { ++ new_list = curl_slist_append(new_list, head->data); ++ } ++ ++ if (!new_list) { ++ ret = CURLE_OUT_OF_MEMORY; ++ goto fail; ++ } ++ } ++ ++ /* Now go over any additional application-supplied headers. */ ++ for(head = dup; head; head = head->next) { ++ if(head->data) { ++ new_list = curl_slist_append(new_list, head->data); ++ if(!new_list) { ++ ret = CURLE_OUT_OF_MEMORY; ++ goto fail; ++ } ++ } ++ } ++ ++ curl_slist_free_all(dup); ++ /* Save the new, merged list separately, so it can be freed later. */ ++ curl_slist_free_all(data->state.merged_headers); ++ data->state.merged_headers = new_list; ++ ++ return CURLE_OK; ++ ++fail: ++ Curl_safefree(dup); ++ curl_slist_free_all(new_list); ++ return ret; ++} ++ + CURLcode Curl_http_useragent(struct Curl_easy *data) + { + /* The User-Agent string might have been allocated in url.c already, because +@@ -3067,6 +3163,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done) + if(result) + return result; + ++ /* curl-impersonate: Add HTTP headers to impersonate real browsers. */ ++ result = Curl_http_merge_headers(data); ++ if (result) ++ return result; ++ + result = Curl_http_useragent(data); + if(result) + return result; diff --git a/lib/http.h b/lib/http.h index b4aaba2a2..1cf65c4b1 100644 --- a/lib/http.h @@ -160,6 +552,103 @@ index f8dcc63b4..e6b728592 100644 multi->ipv6_works = Curl_ipv6works(NULL); #ifdef USE_WINSOCK +diff --git a/lib/setopt.c b/lib/setopt.c +index 599ed5d99..1baa48e70 100644 +--- a/lib/setopt.c ++++ b/lib/setopt.c +@@ -48,6 +48,7 @@ + #include "multiif.h" + #include "altsvc.h" + #include "hsts.h" ++#include "slist.h" + + /* The last 3 #include files should be in this order */ + #include "curl_printf.h" +@@ -688,6 +689,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) + va_arg(param, char *)); + break; + ++ case CURLOPT_HTTPBASEHEADER: ++ /* ++ * curl-impersonate: ++ * Set a list of "base" headers. These will be merged with any headers ++ * set by CURLOPT_HTTPHEADER. curl-impersonate uses this option in order ++ * to set a list of default browser headers. ++ * ++ * Unlike CURLOPT_HTTPHEADER, ++ * the list is copied and can be immediately freed by the user. ++ */ ++ curl_slist_free_all(data->state.base_headers); ++ data->state.base_headers = \ ++ Curl_slist_duplicate(va_arg(param, struct curl_slist *)); ++ if (!data->state.base_headers) ++ result = CURLE_OUT_OF_MEMORY; ++ break; ++ + case CURLOPT_HTTPHEADER: + /* + * Set a list with HTTP headers to use (or replace internals with) +diff --git a/lib/transfer.c b/lib/transfer.c +index 22704fa15..1e100140c 100644 +--- a/lib/transfer.c ++++ b/lib/transfer.c +@@ -102,7 +102,15 @@ char *Curl_checkheaders(const struct Curl_easy *data, + DEBUGASSERT(thislen); + DEBUGASSERT(thisheader[thislen-1] != ':'); + +- for(head = data->set.headers; head; head = head->next) { ++ /* ++ * curl-impersonate: ++ * Check if we have overriden the user-supplied list of headers. ++ */ ++ head = data->set.headers; ++ if (data->state.merged_headers) ++ head = data->state.merged_headers; ++ ++ for(; head; head = head->next) { + if(strncasecompare(head->data, thisheader, thislen) && + Curl_headersep(head->data[thislen]) ) + return head->data; +diff --git a/lib/url.c b/lib/url.c +index 9f1013554..f0f266797 100644 +--- a/lib/url.c ++++ b/lib/url.c +@@ -469,6 +469,11 @@ CURLcode Curl_close(struct Curl_easy **datap) + Curl_safefree(data->state.aptr.proxyuser); + Curl_safefree(data->state.aptr.proxypasswd); + ++ /* curl-impersonate: Free the list set by CURLOPT_HTTPBASEHEADER. */ ++ curl_slist_free_all(data->state.base_headers); ++ /* curl-impersonate: Free the dynamic list of headers. */ ++ curl_slist_free_all(data->state.merged_headers); ++ + #ifndef CURL_DISABLE_DOH + if(data->req.doh) { + Curl_dyn_free(&data->req.doh->probe[0].serverdoh); +diff --git a/lib/urldata.h b/lib/urldata.h +index cc9c88870..a35a20e10 100644 +--- a/lib/urldata.h ++++ b/lib/urldata.h +@@ -1421,6 +1421,19 @@ struct UrlState { + CURLcode hresult; /* used to pass return codes back from hyper callbacks */ + #endif + ++ /* ++ * curl-impersonate: ++ * List of "base" headers set by CURLOPT_HTTPBASEHEADER. ++ */ ++ struct curl_slist *base_headers; ++ /* ++ * curl-impersonate: ++ * Dynamically-constructed list of HTTP headers. ++ * This list is a merge of the default HTTP headers needed to impersonate a ++ * browser, together with any user-supplied headers. ++ */ ++ struct curl_slist *merged_headers; ++ + /* Dynamically allocated strings, MUST be freed before this struct is + killed. */ + struct dynamically_allocated_data { diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c index f836c63b0..5c562549f 100644 --- a/lib/vtls/openssl.c From 8714c4631c28163e182555b4c6a728267ef2b54c Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sat, 26 Feb 2022 12:48:17 +0200 Subject: [PATCH 3/7] Update README.md about libcurl-impersonate --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 9ebc9ba..b4a0a02 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ docker build -t curl-impersonate-chrome chrome/ The resulting image contains: * `/build/out/curl-impersonate` - The curl binary that can impersonate Chrome. It is compiled statically against libcurl, BoringSSL, and libnghttp2 so that it won't conflict with any existing libraries on your system. You can use it from the container or copy it out. Tested to work on Ubuntu 20.04. * `/build/out/curl_chrome98` - A wrapper script that launches `curl-impersonate` with the needed headers and ciphers to impersonate Chrome 98. +* `/build/out/libcurl-impersonate.so` - libcurl compiled with impersonation support. See [Usage](#usage) below for more details. You can use them inside the docker, copy them out using `docker cp` or use them in a multi-stage docker build. If you use it outside this container: * Install dependencies: `sudo apt install libbrotli1` @@ -60,6 +61,27 @@ curl_chrome98 https://www.google.com ``` You can add command line flags and they will be passed on to curl. However, some flags change curl's TLS signature which may cause it to be detected. +### libcurl-impersonate +`libcurl-impersonate.so` is libcurl compiled with the same changes as the command line `curl-impersonate`. +It has an additional API function: +```c +CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target); +``` +You can call it with the target names, e.g. `"chrome98"`, and it will internally set all the options and headers that are otherwise set by the wrapper scripts. Specifically it sets: +* `CURLOPT_HTTP_VERSION` +* `CURLOPT_SSLVERSION` +* `CURLOPT_SSL_CIPHER_LIST` +* `CURLOPT_HTTPBASEHEADER` (non-standard option created for this project). +Note that if you call `curl_easy_setopt()` later it may override the options set by `curl_easy_impersonate()`. + +### Using CURL_IMPERSONATE env var +*Experimental*: +If your application uses `libcurl` already, you can rename `libcurl-impersonate.so` and replace the existing library at runtime with `LD_LIBRARY_PATH`. You can then set the `CURL_IMPERSONATE` env var, e.g. +```bash +LD_LIBRARY_PATH=/path/to/libcurl-impersonate/ CURL_IMPERSONATE=chrome98 my_app +``` +But note that doing so for `curl` itself will NOT WORK (it overrides the TLS options by default). Use the wrapper scripts instead. + ## Contents This repository contains two main folders: From 6dad23b4b8b31f90359c378803fdbfbec9ba5b37 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sun, 27 Feb 2022 23:21:00 +0200 Subject: [PATCH 4/7] Add tests for libcurl-impersonate Test that libcurl-impersonate produces the desired TLS signature when the CURL_IMPERSONATE env var is set. A small C program called "minicurl" is linked to libcurl, and libcurl-impersonate is loaded at runtime with LD_PRELOAD. --- tests/Dockerfile | 7 +- tests/minicurl.c | 194 ++++++++++++++++++++++++++++++++++++++ tests/test_impersonate.py | 33 ++++++- 3 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 tests/minicurl.c diff --git a/tests/Dockerfile b/tests/Dockerfile index 5676cab..741d0b5 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.10.1-slim-buster WORKDIR /tests RUN apt-get update && \ - apt-get install -y tcpdump libbrotli1 libnss3 + apt-get install -y tcpdump libbrotli1 libnss3 gcc libcurl4-openssl-dev COPY requirements.txt requirements.txt @@ -18,4 +18,9 @@ COPY --from=curl-impersonate-chrome /build/out/* /tests/chrome/ COPY . . +# Compile 'minicurl' which is used for testing libcurl-impersonate. +# 'minicurl' is compiled against the "regular" libcurl. +# libcurl-impersonate will replace it at runtime via LD_PRELOAD. +RUN gcc -Wall -Werror -o minicurl minicurl.c `curl-config --libs` + ENTRYPOINT ["pytest"] diff --git a/tests/minicurl.c b/tests/minicurl.c new file mode 100644 index 0000000..57887e2 --- /dev/null +++ b/tests/minicurl.c @@ -0,0 +1,194 @@ +/* + * A simple program that uses libcurl to fetch a URL and output to stdout. + * + * It is intended to be linked against the "regular" libcurl, with + * "libcurl-impersonate" loaded via LD_PRELOAD. It does the bare minimum + * to support the Python tests. + */ +#include +#include +#include +#include +#include +#include +#include + +#include + +/* Command line options. */ +struct opts { + char *outfile; + uint16_t local_port_start; + uint16_t local_port_end; + char *url; +}; + +int parse_ports_range(char *str, uint16_t *start, uint16_t *end) +{ + char port[32]; + char *sep; + unsigned long int tmp; + + if (strlen(str) >= sizeof(port)) { + return 1; + } + strncpy(port, str, sizeof(port) - 1); + sep = strchr(port, '-'); + if (!sep) { + return 1; + } + *sep = 0; + + errno = 0; + tmp = strtoul(port, NULL, 10); + if (errno || tmp == 0 || tmp > 0xffff) { + return 1; + } + *start = (uint16_t)tmp; + tmp = strtoul(sep + 1, NULL, 10); + if (errno || tmp == 0 || tmp > 0xffff || tmp < *start) { + return 1; + } + *end = (uint16_t)tmp; + + return 0; +} + +int parse_opts(int argc, char **argv, struct opts *opts) +{ + int c; + int r; + + memset(opts, 0, sizeof(*opts)); + + while (1) { + int option_index = 0; + static struct option long_options[] = { + {"local-port", required_argument, NULL, 'l'} + }; + + c = getopt_long(argc, argv, "o:", long_options, &option_index); + if (c == -1) { + break; + } + + switch (c) { + case 'l': + r = parse_ports_range(optarg, + &opts->local_port_start, + &opts->local_port_end); + if (r) { + return r; + } + break; + case 'o': + opts->outfile = optarg; + break; + } + } + + if (optind < argc) { + opts->url = argv[optind++]; + } else { + return 1; + } + + if (optind < argc) { + /* Too many arguments. */ + return 1; + } + + return 0; +} + +int main(int argc, char *argv[]) +{ + struct opts opts; + CURLcode c; + CURL *curl = NULL; + FILE *file; + + if (parse_opts(argc, argv, &opts)) { + fprintf(stderr, "Invalid arguments\n"); + exit(1); + } + + if (opts.outfile) { + file = fopen(opts.outfile, "w"); + if (!file) { + fprintf(stderr, "Failed opening %s for writing\n", opts.outfile); + exit(1); + } + } else { + file = stdout; + } + + c = curl_global_init(CURL_GLOBAL_DEFAULT); + if (c) { + fprintf(stderr, "curl_global_init() failed\n"); + goto out_close; + } + + curl = curl_easy_init(); + if (!curl) { + fprintf(stderr, "curl_easy_init() failed\n"); + c = 1; + goto out; + } + + c = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite); + if (c) { + fprintf(stderr, "curl_easy_setopt(CURLOPT_WRITEFUNCTION) failed\n"); + goto out; + } + + c = curl_easy_setopt(curl, CURLOPT_WRITEDATA, file); + if (c) { + fprintf(stderr, "curl_easy_setopt(CURLOPT_WRITEDATA) failed\n"); + goto out; + } + + if (opts.local_port_start && opts.local_port_end) { + c = curl_easy_setopt(curl, + CURLOPT_LOCALPORT, + opts.local_port_start); + if (c) { + fprintf(stderr, "curl_easy_setopt(CURLOPT_LOCALPORT) failed\n"); + goto out; + } + + c = curl_easy_setopt(curl, + CURLOPT_LOCALPORTRANGE, + opts.local_port_end - opts.local_port_start); + if (c) { + fprintf(stderr, + "curl_easy_setopt(CURLOPT_LOCALPORTRANGE) failed\n"); + goto out; + } + } + + c = curl_easy_setopt(curl, CURLOPT_URL, opts.url); + if (c) { + fprintf(stderr, "curl_easy_setopt(CURLOPT_URL) failed\n"); + goto out; + } + + c = curl_easy_perform(curl); + if (c) { + fprintf(stderr, "curl_easy_perform() failed\n"); + goto out; + } + + c = 0; + +out: + if (curl) { + curl_easy_cleanup(curl); + } + curl_global_cleanup(); +out_close: + if (file) { + fclose(file); + } + return c; +} diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index 3f2b108..f1e12e4 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -1,3 +1,4 @@ +import os import io import logging import subprocess @@ -176,16 +177,30 @@ class TestImpersonation: return None @pytest.mark.parametrize( - "curl_binary, expected_signature", + "curl_binary, env_vars, expected_signature", [ - ("chrome/curl_chrome98", "chrome_98.0.4758.102_win10"), - ("firefox/curl_ff91esr", "firefox_91.6.0esr_win10"), - ("firefox/curl_ff95", "firefox_95.0.2_win10") + # Test wrapper scripts + ("chrome/curl_chrome98", None, "chrome_98.0.4758.102_win10"), + ("firefox/curl_ff91esr", None, "firefox_91.6.0esr_win10"), + ("firefox/curl_ff95", None, "firefox_95.0.2_win10"), + + # Test libcurl-impersonate by loading it with LD_PRELOAD to an app + # linked against the regular libcurl and setting the + # CURL_IMPERSONATE env var. + ( + "./minicurl", + { + "LD_PRELOAD": "./chrome/libcurl-impersonate.so", + "CURL_IMPERSONATE": "chrome98" + }, + "chrome_98.0.4758.102_win10" + ) ] ) def test_impersonation(self, tcpdump, curl_binary, + env_vars, browser_signatures, expected_signature): """ @@ -196,13 +211,21 @@ class TestImpersonation: extract the Client Hello packet from the capture and compares its signature with the expected one defined in the YAML database. """ + env = os.environ.copy() + if env_vars: + env |= env_vars + logging.debug(f"Launching '{curl_binary}' to {self.TEST_URL}") + if env_vars: + logging.debug("Environment variables: {}".format( + " ".join([f"{k}={v}" for k, v in env_vars.items()]))) + curl = subprocess.Popen([ curl_binary, "-o", "/dev/null", "--local-port", f"{self.LOCAL_PORTS[0]}-{self.LOCAL_PORTS[1]}", self.TEST_URL - ]) + ], env=env) ret = curl.wait(timeout=10) assert ret == 0 From 4e2e782e5f44150b46747076e4c98eb07036dd75 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sun, 27 Feb 2022 23:48:17 +0200 Subject: [PATCH 5/7] Add Edge 98 signature to libcurl-impersonate --- chrome/patches/curl-impersonate.patch | 45 ++++++++++++++++++++++++--- tests/test_impersonate.py | 8 +++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/chrome/patches/curl-impersonate.patch b/chrome/patches/curl-impersonate.patch index d5f6df6..6d2e372 100644 --- a/chrome/patches/curl-impersonate.patch +++ b/chrome/patches/curl-impersonate.patch @@ -57,7 +57,7 @@ index 2dbfb26b5..e0bf86169 100644 * NAME curl_easy_getinfo() * diff --git a/lib/easy.c b/lib/easy.c -index 20293a710..df3e66bc0 100644 +index 20293a710..ec16aee23 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -80,6 +80,7 @@ @@ -68,7 +68,7 @@ index 20293a710..df3e66bc0 100644 /* The last 3 #include files should be in this order */ #include "curl_printf.h" -@@ -282,6 +283,126 @@ void curl_global_cleanup(void) +@@ -282,6 +283,161 @@ void curl_global_cleanup(void) init_flags = 0; } @@ -121,6 +121,41 @@ index 20293a710..df3e66bc0 100644 + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + } ++ }, ++ { ++ .target = "edge98", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "ECDHE-ECDSA-AES128-GCM-SHA256," ++ "ECDHE-RSA-AES128-GCM-SHA256," ++ "ECDHE-ECDSA-AES256-GCM-SHA384," ++ "ECDHE-RSA-AES256-GCM-SHA384," ++ "ECDHE-ECDSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-AES128-SHA," ++ "ECDHE-RSA-AES256-SHA," ++ "AES128-GCM-SHA256," ++ "AES256-GCM-SHA384," ++ "AES128-SHA," ++ "AES256-SHA", ++ .http_headers = { ++ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"98\", \"Microsoft Edge\";v=\"98\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"Windows\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.62", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br", ++ "Accept-Language: en-US,en;q=0.9" ++ } + } +}; + @@ -195,7 +230,7 @@ index 20293a710..df3e66bc0 100644 /* * curl_easy_init() is the external interface to alloc, setup and init an * easy handle that is returned. If anything goes wrong, NULL is returned. -@@ -290,6 +411,7 @@ struct Curl_easy *curl_easy_init(void) +@@ -290,6 +446,7 @@ struct Curl_easy *curl_easy_init(void) { CURLcode result; struct Curl_easy *data; @@ -203,7 +238,7 @@ index 20293a710..df3e66bc0 100644 /* Make sure we inited the global SSL stuff */ if(!initialized) { -@@ -308,6 +430,22 @@ struct Curl_easy *curl_easy_init(void) +@@ -308,6 +465,22 @@ struct Curl_easy *curl_easy_init(void) return NULL; } @@ -226,7 +261,7 @@ index 20293a710..df3e66bc0 100644 return data; } -@@ -878,6 +1016,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) +@@ -878,6 +1051,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) outcurl->state.referer_alloc = TRUE; } diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index 45a532f..db8efa8 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -195,6 +195,14 @@ class TestImpersonation: "CURL_IMPERSONATE": "chrome98" }, "chrome_98.0.4758.102_win10" + ), + ( + "./minicurl", + { + "LD_PRELOAD": "./chrome/libcurl-impersonate.so", + "CURL_IMPERSONATE": "edge98" + }, + "edge_98.0.1108.62_win10" ) ] ) From 00dc027185aebd49536d267b9da2f372368a6156 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sun, 27 Feb 2022 23:56:35 +0200 Subject: [PATCH 6/7] Update README.md --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ad08bea..834bce2 100644 --- a/README.md +++ b/README.md @@ -73,15 +73,17 @@ You can call it with the target names, e.g. `"chrome98"`, and it will internally * `CURLOPT_SSLVERSION` * `CURLOPT_SSL_CIPHER_LIST` * `CURLOPT_HTTPBASEHEADER` (non-standard option created for this project). -Note that if you call `curl_easy_setopt()` later it may override the options set by `curl_easy_impersonate()`. + +Note that if you call `curl_easy_setopt()` later with one of the above it will override the options set by `curl_easy_impersonate()`. ### Using CURL_IMPERSONATE env var -*Experimental*: -If your application uses `libcurl` already, you can rename `libcurl-impersonate.so` and replace the existing library at runtime with `LD_LIBRARY_PATH`. You can then set the `CURL_IMPERSONATE` env var, e.g. +*Experimental*: If your application uses `libcurl` already, you can replace the existing library at runtime with `LD_PRELOAD`. You can then set the `CURL_IMPERSONATE` env var. For example: ```bash -LD_LIBRARY_PATH=/path/to/libcurl-impersonate/ CURL_IMPERSONATE=chrome98 my_app +LD_PRELOAD=/path/to/libcurl-impersonate.so CURL_IMPERSONATE=chrome98 my_app ``` -But note that doing so for `curl` itself will NOT WORK (it overrides the TLS options by default). Use the wrapper scripts instead. +The `CURL_IMPERSONATE` env var will cause `curl_easy_impersonate()` to be called automatically for any new curl handle created by `curl_easy_init()`. + +Note that the above will NOT WORK for `curl` itself because the curl tool overrides the TLS settings. Use the wrapper scripts instead. ## Contents From a7cbfd9fed78460cb9fa9d01e1ba94f0393650f2 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Mon, 28 Feb 2022 10:18:04 +0200 Subject: [PATCH 7/7] Add libcurl impersonation support in Firefox build 48415a4b00354c5a9c95d35d78172b0cb28d15ff added impersonation capabilities to libcurl in the Chrome build. This adds the same capabilities to the Firefox build as well. curl-impersonate.patch generated from https://github.com/lwthiker/curl/commit/b30b245b722dbe9765615e3f316e9a7dab99bbf9 --- README.md | 1 + chrome/Dockerfile | 3 +- firefox/Dockerfile | 37 +- firefox/patches/curl-impersonate.patch | 525 +++++++++++++++++++++++++ tests/test_impersonate.py | 16 + 5 files changed, 568 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 834bce2..cccc152 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ The resulting image contains: * `/build/out/curl-impersonate` - The curl binary that can impersonate Firefox. It is compiled statically against libcurl, nss, and libnghttp2 so that it won't conflict with any existing libraries on your system. You can use it from the container or copy it out. Tested to work on Ubuntu 20.04. * `/build/out/curl_ff91esr` - A wrapper script that launches `curl-impersonate` with the needed headers and ciphers to impersonate Firefox 91 ESR (Extended Support Release). * `/build/out/curl_ff95` - Same but with Firefox 95. +* `/build/out/libcurl-impersonate.so` - libcurl compiled with impersonation support. See [Usage](#usage) below for more details. If you use it outside this container: * Install dependencies: `sudo apt install libbrotli1` diff --git a/chrome/Dockerfile b/chrome/Dockerfile index 23afa4f..c16865a 100644 --- a/chrome/Dockerfile +++ b/chrome/Dockerfile @@ -73,8 +73,7 @@ RUN mkdir out && \ # Re-compile libcurl dynamically RUN cd ${CURL_VERSION} && \ - ./configure --enable-versioned-symbols \ - --with-openssl=/build/boringssl/build \ + ./configure --with-openssl=/build/boringssl/build \ --with-nghttp2=/usr/local \ LIBS="-pthread" \ CFLAGS="-I/build/boringssl/build" \ diff --git a/firefox/Dockerfile b/firefox/Dockerfile index 9fc274c..1c5f9ca 100644 --- a/firefox/Dockerfile +++ b/firefox/Dockerfile @@ -40,7 +40,7 @@ RUN cd ${NGHTTP2_VERSION} && \ # Compile nghttp2 RUN cd ${NGHTTP2_VERSION} && \ - ./configure && \ + ./configure --with-pic && \ make && make install # Download curl. @@ -48,17 +48,15 @@ ARG CURL_VERSION=curl-7.81.0 RUN curl -o ${CURL_VERSION}.tar.xz https://curl.se/download/${CURL_VERSION}.tar.xz RUN tar xf ${CURL_VERSION}.tar.xz -# Patch Curl. +# Patch curl and re-generate the configure script COPY patches/curl-*.patch ${CURL_VERSION}/ - -# Re-generate the configure script RUN cd ${CURL_VERSION} && \ for p in $(ls curl-*.patch); do patch -p1 < $p; done && \ autoreconf -fi # Compile curl with nss RUN cd ${CURL_VERSION} && \ - ./configure --with-nss=/build/${NSS_VERSION}/dist/Release --enable-static --disable-shared CFLAGS="-I/build/${NSS_VERSION}/dist/public/nss -I/build/${NSS_VERSION}/dist/Release/include/nspr" --with-nghttp2=/usr/local && \ + ./configure --with-nss=/build/${NSS_VERSION}/dist/Release --enable-static --disable-shared CFLAGS="-I/build/${NSS_VERSION}/dist/public/nss -I/build/${NSS_VERSION}/dist/Release/include/nspr" --with-nghttp2=/usr/local USE_CURL_SSLKEYLOGFILE=true && \ make # curl tries to load the CA certificates for libnss. @@ -66,13 +64,28 @@ RUN cd ${CURL_VERSION} && \ # which is supplied by libnss3 on Debian/Ubuntu RUN apt-get install -y libnss3 -# 'xxd' is needed for the wrapper curl_ff95 script -RUN apt-get install -y xxd - RUN mkdir out && \ - cp ${CURL_VERSION}/src/curl out/curl-impersonate + cp ${CURL_VERSION}/src/curl out/curl-impersonate && \ + strip out/curl-impersonate + +# Re-compile libcurl dynamically +RUN cd ${CURL_VERSION} && \ + ./configure --with-nss=/build/${NSS_VERSION}/dist/Release \ + --with-nghttp2=/usr/local \ + CFLAGS="-I/build/${NSS_VERSION}/dist/public/nss -I/build/${NSS_VERSION}/dist/Release/include/nspr" \ + USE_CURL_SSLKEYLOGFILE=true && \ + make clean && make + +# Rename to 'libcurl-impersonate' to avoid confusion, and recreate the +# symbolic links. +RUN ver=$(readlink -f curl-7.81.0/lib/.libs/libcurl.so | sed 's/.*so\.//') && \ + major=$(echo -n $ver | cut -d'.' -f1) && \ + cp "${CURL_VERSION}/lib/.libs/libcurl.so.$ver" "out/libcurl-impersonate.so.$ver" && \ + ln -s "libcurl-impersonate.so.$ver" "out/libcurl-impersonate.so.$major" && \ + ln -s "libcurl-impersonate.so.$ver" "out/libcurl-impersonate.so" && \ + strip "out/libcurl-impersonate.so.$ver" + # Wrapper script -COPY curl_* out/ - -RUN chmod +x out/* +COPY curl_ff* out/ +RUN chmod +x out/curl_* diff --git a/firefox/patches/curl-impersonate.patch b/firefox/patches/curl-impersonate.patch index 83034db..10dce73 100644 --- a/firefox/patches/curl-impersonate.patch +++ b/firefox/patches/curl-impersonate.patch @@ -21,6 +21,434 @@ index 63e320236..deb054300 100644 AC_MSG_NOTICE([-L is $LD_H2]) LDFLAGS="$LDFLAGS $LD_H2" +diff --git a/include/curl/curl.h b/include/curl/curl.h +index 7b69ce2d6..fe4bb36b9 100644 +--- a/include/curl/curl.h ++++ b/include/curl/curl.h +@@ -2135,6 +2135,10 @@ typedef enum { + /* Set MIME option flags. */ + CURLOPT(CURLOPT_MIME_OPTIONS, CURLOPTTYPE_LONG, 315), + ++ /* curl-impersonate: A list of headers used by the impersonated browser. ++ * If given, merged with CURLOPT_HTTPHEADER. */ ++ CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 316), ++ + CURLOPT_LASTENTRY /* the last unused */ + } CURLoption; + +diff --git a/include/curl/easy.h b/include/curl/easy.h +index 2dbfb26b5..e0bf86169 100644 +--- a/include/curl/easy.h ++++ b/include/curl/easy.h +@@ -41,6 +41,15 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); + CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); + CURL_EXTERN void curl_easy_cleanup(CURL *curl); + ++/* ++ * curl-impersonate: Tell libcurl to impersonate a browser. ++ * This is a wrapper function that calls curl_easy_setopt() ++ * multiple times with all the parameters required. That's also why it was ++ * created as a separate API function and not just as another option to ++ * curl_easy_setopt(). ++ */ ++CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target); ++ + /* + * NAME curl_easy_getinfo() + * +diff --git a/lib/easy.c b/lib/easy.c +index 20293a710..b0b4c2751 100644 +--- a/lib/easy.c ++++ b/lib/easy.c +@@ -80,6 +80,7 @@ + #include "dynbuf.h" + #include "altsvc.h" + #include "hsts.h" ++#include "strcase.h" + + /* The last 3 #include files should be in this order */ + #include "curl_printf.h" +@@ -282,6 +283,162 @@ void curl_global_cleanup(void) + init_flags = 0; + } + ++/* ++ * curl-impersonate: Options to be set for each supported target browser. ++ * Note: this does not include the HTTP headers, which are handled separately ++ * in Curl_http(). ++ */ ++#define IMPERSONATE_MAX_HEADERS 32 ++static const struct impersonate_opts { ++ const char *target; ++ int httpversion; ++ int ssl_version; ++ const char *ciphers; ++ const char *http_headers[IMPERSONATE_MAX_HEADERS]; ++ /* Other TLS options will come here in the future once they are ++ * configurable through curl_easy_setopt() */ ++} impersonations[] = { ++ { ++ .target = "ff91esr", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "aes_128_gcm_sha_256," ++ "chacha20_poly1305_sha_256," ++ "aes_256_gcm_sha_384," ++ "ecdhe_ecdsa_aes_128_gcm_sha_256," ++ "ecdhe_rsa_aes_128_gcm_sha_256," ++ "ecdhe_ecdsa_chacha20_poly1305_sha_256," ++ "ecdhe_rsa_chacha20_poly1305_sha_256," ++ "ecdhe_ecdsa_aes_256_gcm_sha_384," ++ "ecdhe_rsa_aes_256_gcm_sha_384," ++ "ecdhe_ecdsa_aes_256_sha," ++ "ecdhe_ecdsa_aes_128_sha," ++ "ecdhe_rsa_aes_128_sha," ++ "ecdhe_rsa_aes_256_sha," ++ "rsa_aes_128_gcm_sha_256," ++ "rsa_aes_256_gcm_sha_384," ++ "rsa_aes_128_sha," ++ "rsa_aes_256_sha," ++ "rsa_3des_ede_cbc_sha", ++ .http_headers = { ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", ++ "Accept-Language: en-US,en;q=0.5", ++ "Accept-Encoding: gzip, deflate, br", ++ "Connection: keep-alive", ++ "Upgrade-Insecure-Requests: 1", ++ "Sec-Fetch-Dest: document", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-User: ?1" ++ } ++ }, ++ { ++ .target = "ff95", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "aes_128_gcm_sha_256," ++ "chacha20_poly1305_sha_256," ++ "aes_256_gcm_sha_384," ++ "ecdhe_ecdsa_aes_128_gcm_sha_256," ++ "ecdhe_rsa_aes_128_gcm_sha_256," ++ "ecdhe_ecdsa_chacha20_poly1305_sha_256," ++ "ecdhe_rsa_chacha20_poly1305_sha_256," ++ "ecdhe_ecdsa_aes_256_gcm_sha_384," ++ "ecdhe_rsa_aes_256_gcm_sha_384," ++ "ecdhe_ecdsa_aes_256_sha," ++ "ecdhe_ecdsa_aes_128_sha," ++ "ecdhe_rsa_aes_128_sha," ++ "ecdhe_rsa_aes_256_sha," ++ "rsa_aes_128_gcm_sha_256," ++ "rsa_aes_256_gcm_sha_384," ++ "rsa_aes_128_sha," ++ "rsa_aes_256_sha", ++ .http_headers = { ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", ++ "Accept-Language: en-US,en;q=0.5", ++ "Accept-Encoding: gzip, deflate, br", ++ "Connection: keep-alive", ++ "Upgrade-Insecure-Requests: 1", ++ "Sec-Fetch-Dest: document", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-User: ?1" ++ } ++ } ++}; ++ ++#define NUM_IMPERSONATIONS \ ++ sizeof(impersonations) / sizeof(impersonations[0]) ++ ++/* ++ * curl-impersonate: ++ * Call curl_easy_setopt() with all the needed options as defined in the ++ * 'impersonations' array. ++ * */ ++CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target) ++{ ++ int i; ++ int ret; ++ const struct impersonate_opts *opts = NULL; ++ struct curl_slist *headers = NULL; ++ ++ for(i = 0; i < NUM_IMPERSONATIONS; i++) { ++ if (Curl_strncasecompare(target, ++ impersonations[i].target, ++ strlen(impersonations[i].target))) { ++ opts = &impersonations[i]; ++ break; ++ } ++ } ++ ++ if(!opts) { ++ DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n", ++ target)); ++ return CURLE_BAD_FUNCTION_ARGUMENT; ++ } ++ ++ if(opts->httpversion != CURL_HTTP_VERSION_NONE) { ++ ret = curl_easy_setopt(data, CURLOPT_HTTP_VERSION, opts->httpversion); ++ if(ret) ++ return ret; ++ } ++ ++ if (opts->ssl_version != CURL_SSLVERSION_DEFAULT) { ++ ret = curl_easy_setopt(data, CURLOPT_SSLVERSION, opts->ssl_version); ++ if(ret) ++ return ret; ++ } ++ ++ if(opts->ciphers) { ++ ret = curl_easy_setopt(data, CURLOPT_SSL_CIPHER_LIST, opts->ciphers); ++ if (ret) ++ return ret; ++ } ++ ++ /* Build a linked list out of the static array of headers. */ ++ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) { ++ if(opts->http_headers[i]) { ++ headers = curl_slist_append(headers, opts->http_headers[i]); ++ if(!headers) { ++ return CURLE_OUT_OF_MEMORY; ++ } ++ } ++ } ++ ++ if(headers) { ++ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers); ++ curl_slist_free_all(headers); ++ if(ret) ++ return ret; ++ } ++ ++ return CURLE_OK; ++} ++ + /* + * curl_easy_init() is the external interface to alloc, setup and init an + * easy handle that is returned. If anything goes wrong, NULL is returned. +@@ -290,6 +447,7 @@ struct Curl_easy *curl_easy_init(void) + { + CURLcode result; + struct Curl_easy *data; ++ char *target; + + /* Make sure we inited the global SSL stuff */ + if(!initialized) { +@@ -308,6 +466,22 @@ struct Curl_easy *curl_easy_init(void) + return NULL; + } + ++ /* ++ * curl-impersonate: Hook into curl_easy_init() to set the required options ++ * from an environment variable. ++ * This is a bit hacky but allows seamless integration of libcurl-impersonate ++ * without code modifications to the app. ++ */ ++ target = curl_getenv("CURL_IMPERSONATE"); ++ if(target) { ++ result = curl_easy_impersonate(data, target); ++ free(target); ++ if(result) { ++ Curl_close(&data); ++ return NULL; ++ } ++ } ++ + return data; + } + +@@ -878,6 +1052,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) + outcurl->state.referer_alloc = TRUE; + } + ++ if(data->state.base_headers) { ++ outcurl->state.base_headers = ++ Curl_slist_duplicate(data->state.base_headers); ++ if(!outcurl->state.base_headers) ++ goto fail; ++ } ++ + /* Reinitialize an SSL engine for the new handle + * note: the engine name has already been copied by dupset */ + if(outcurl->set.str[STRING_SSL_ENGINE]) { +diff --git a/lib/easyoptions.c b/lib/easyoptions.c +index 04871ad1e..cd5998146 100644 +--- a/lib/easyoptions.c ++++ b/lib/easyoptions.c +@@ -130,6 +130,7 @@ struct curl_easyoption Curl_easyopts[] = { + {"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0}, + {"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0}, + {"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0}, ++ {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0}, + {"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0}, + {"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0}, + {"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0}, +diff --git a/lib/http.c b/lib/http.c +index f08a343e3..879151dd2 100644 +--- a/lib/http.c ++++ b/lib/http.c +@@ -84,6 +84,7 @@ + #include "altsvc.h" + #include "hsts.h" + #include "c-hyper.h" ++#include "slist.h" + + /* The last 3 #include files should be in this order */ + #include "curl_printf.h" +@@ -1795,6 +1796,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, + int numlists = 1; /* by default */ + int i; + ++ /* ++ * curl-impersonate: Use the merged list of headers if it exists (i.e. when ++ * the CURLOPT_HTTPBASEHEADER option was set. ++ */ ++ struct curl_slist *noproxyheaders = ++ (data->state.merged_headers ? ++ data->state.merged_headers : ++ data->set.headers); ++ + #ifndef CURL_DISABLE_PROXY + enum proxy_use proxy; + +@@ -1806,10 +1816,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, + + switch(proxy) { + case HEADER_SERVER: +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + break; + case HEADER_PROXY: +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + if(data->set.sep_headers) { + h[1] = data->set.proxyheaders; + numlists++; +@@ -1819,12 +1829,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, + if(data->set.sep_headers) + h[0] = data->set.proxyheaders; + else +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + break; + } + #else + (void)is_connect; +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + #endif + + /* loop through one or two lists */ +@@ -2059,6 +2069,92 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn, + *reqp = httpreq; + } + ++/* ++ * curl-impersonate: ++ * Create a new linked list of headers. ++ * The new list is a merge between the "base" headers and the application given ++ * headers. The "base" headers contain curl-impersonate's list of headers ++ * used by default by the impersonated browser. ++ * ++ * The application given headers will override the "base" headers if supplied. ++ */ ++CURLcode Curl_http_merge_headers(struct Curl_easy *data) ++{ ++ int i; ++ int ret; ++ struct curl_slist *head; ++ struct curl_slist *dup = NULL; ++ struct curl_slist *new_list = NULL; ++ ++ if (!data->state.base_headers) ++ return CURLE_OK; ++ ++ /* Duplicate the list for temporary use. */ ++ if (data->set.headers) { ++ dup = Curl_slist_duplicate(data->set.headers); ++ if(!dup) ++ return CURLE_OUT_OF_MEMORY; ++ } ++ ++ for(head = data->state.base_headers; head; head = head->next) { ++ char *sep; ++ size_t prefix_len; ++ bool found = FALSE; ++ struct curl_slist *head2; ++ ++ sep = strchr(head->data, ':'); ++ if(!sep) ++ continue; ++ ++ prefix_len = sep - head->data; ++ ++ /* Check if this header was added by the application. */ ++ for(head2 = dup; head2; head2 = head2->next) { ++ if(head2->data && ++ strncasecompare(head2->data, head->data, prefix_len) && ++ Curl_headersep(head2->data[prefix_len]) ) { ++ new_list = curl_slist_append(new_list, head2->data); ++ /* Free and set to NULL to mark that it's been added. */ ++ Curl_safefree(head2->data); ++ found = TRUE; ++ break; ++ } ++ } ++ ++ if (!found) { ++ new_list = curl_slist_append(new_list, head->data); ++ } ++ ++ if (!new_list) { ++ ret = CURLE_OUT_OF_MEMORY; ++ goto fail; ++ } ++ } ++ ++ /* Now go over any additional application-supplied headers. */ ++ for(head = dup; head; head = head->next) { ++ if(head->data) { ++ new_list = curl_slist_append(new_list, head->data); ++ if(!new_list) { ++ ret = CURLE_OUT_OF_MEMORY; ++ goto fail; ++ } ++ } ++ } ++ ++ curl_slist_free_all(dup); ++ /* Save the new, merged list separately, so it can be freed later. */ ++ curl_slist_free_all(data->state.merged_headers); ++ data->state.merged_headers = new_list; ++ ++ return CURLE_OK; ++ ++fail: ++ Curl_safefree(dup); ++ curl_slist_free_all(new_list); ++ return ret; ++} ++ + CURLcode Curl_http_useragent(struct Curl_easy *data) + { + /* The User-Agent string might have been allocated in url.c already, because +@@ -3067,6 +3163,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done) + if(result) + return result; + ++ /* curl-impersonate: Add HTTP headers to impersonate real browsers. */ ++ result = Curl_http_merge_headers(data); ++ if (result) ++ return result; ++ + result = Curl_http_useragent(data); + if(result) + return result; diff --git a/lib/http2.c b/lib/http2.c index e74400a4c..1f4d496f3 100644 --- a/lib/http2.c @@ -35,6 +463,103 @@ index e74400a4c..1f4d496f3 100644 /* USHRT_MAX is 65535 == 0xffff */ #define HEADER_OVERFLOW(x) \ +diff --git a/lib/setopt.c b/lib/setopt.c +index 599ed5d99..1baa48e70 100644 +--- a/lib/setopt.c ++++ b/lib/setopt.c +@@ -48,6 +48,7 @@ + #include "multiif.h" + #include "altsvc.h" + #include "hsts.h" ++#include "slist.h" + + /* The last 3 #include files should be in this order */ + #include "curl_printf.h" +@@ -688,6 +689,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) + va_arg(param, char *)); + break; + ++ case CURLOPT_HTTPBASEHEADER: ++ /* ++ * curl-impersonate: ++ * Set a list of "base" headers. These will be merged with any headers ++ * set by CURLOPT_HTTPHEADER. curl-impersonate uses this option in order ++ * to set a list of default browser headers. ++ * ++ * Unlike CURLOPT_HTTPHEADER, ++ * the list is copied and can be immediately freed by the user. ++ */ ++ curl_slist_free_all(data->state.base_headers); ++ data->state.base_headers = \ ++ Curl_slist_duplicate(va_arg(param, struct curl_slist *)); ++ if (!data->state.base_headers) ++ result = CURLE_OUT_OF_MEMORY; ++ break; ++ + case CURLOPT_HTTPHEADER: + /* + * Set a list with HTTP headers to use (or replace internals with) +diff --git a/lib/transfer.c b/lib/transfer.c +index 22704fa15..1e100140c 100644 +--- a/lib/transfer.c ++++ b/lib/transfer.c +@@ -102,7 +102,15 @@ char *Curl_checkheaders(const struct Curl_easy *data, + DEBUGASSERT(thislen); + DEBUGASSERT(thisheader[thislen-1] != ':'); + +- for(head = data->set.headers; head; head = head->next) { ++ /* ++ * curl-impersonate: ++ * Check if we have overriden the user-supplied list of headers. ++ */ ++ head = data->set.headers; ++ if (data->state.merged_headers) ++ head = data->state.merged_headers; ++ ++ for(; head; head = head->next) { + if(strncasecompare(head->data, thisheader, thislen) && + Curl_headersep(head->data[thislen]) ) + return head->data; +diff --git a/lib/url.c b/lib/url.c +index 9f1013554..f0f266797 100644 +--- a/lib/url.c ++++ b/lib/url.c +@@ -469,6 +469,11 @@ CURLcode Curl_close(struct Curl_easy **datap) + Curl_safefree(data->state.aptr.proxyuser); + Curl_safefree(data->state.aptr.proxypasswd); + ++ /* curl-impersonate: Free the list set by CURLOPT_HTTPBASEHEADER. */ ++ curl_slist_free_all(data->state.base_headers); ++ /* curl-impersonate: Free the dynamic list of headers. */ ++ curl_slist_free_all(data->state.merged_headers); ++ + #ifndef CURL_DISABLE_DOH + if(data->req.doh) { + Curl_dyn_free(&data->req.doh->probe[0].serverdoh); +diff --git a/lib/urldata.h b/lib/urldata.h +index cc9c88870..a35a20e10 100644 +--- a/lib/urldata.h ++++ b/lib/urldata.h +@@ -1421,6 +1421,19 @@ struct UrlState { + CURLcode hresult; /* used to pass return codes back from hyper callbacks */ + #endif + ++ /* ++ * curl-impersonate: ++ * List of "base" headers set by CURLOPT_HTTPBASEHEADER. ++ */ ++ struct curl_slist *base_headers; ++ /* ++ * curl-impersonate: ++ * Dynamically-constructed list of HTTP headers. ++ * This list is a merge of the default HTTP headers needed to impersonate a ++ * browser, together with any user-supplied headers. ++ */ ++ struct curl_slist *merged_headers; ++ + /* Dynamically allocated strings, MUST be freed before this struct is + killed. */ + struct dynamically_allocated_data { diff --git a/lib/vtls/nss.c b/lib/vtls/nss.c index 2b44f0512..4c60797c7 100644 --- a/lib/vtls/nss.c diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index db8efa8..672256c 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -203,6 +203,22 @@ class TestImpersonation: "CURL_IMPERSONATE": "edge98" }, "edge_98.0.1108.62_win10" + ), + ( + "./minicurl", + { + "LD_PRELOAD": "./firefox/libcurl-impersonate.so", + "CURL_IMPERSONATE": "ff91esr" + }, + "firefox_91.6.0esr_win10" + ), + ( + "./minicurl", + { + "LD_PRELOAD": "./firefox/libcurl-impersonate.so", + "CURL_IMPERSONATE": "ff95" + }, + "firefox_95.0.2_win10" ) ] )