From a7cbfd9fed78460cb9fa9d01e1ba94f0393650f2 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Mon, 28 Feb 2022 10:18:04 +0200 Subject: [PATCH] Add libcurl impersonation support in Firefox build 48415a4b00354c5a9c95d35d78172b0cb28d15ff added impersonation capabilities to libcurl in the Chrome build. This adds the same capabilities to the Firefox build as well. curl-impersonate.patch generated from https://github.com/lwthiker/curl/commit/b30b245b722dbe9765615e3f316e9a7dab99bbf9 --- README.md | 1 + chrome/Dockerfile | 3 +- firefox/Dockerfile | 37 +- firefox/patches/curl-impersonate.patch | 525 +++++++++++++++++++++++++ tests/test_impersonate.py | 16 + 5 files changed, 568 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 834bce2..cccc152 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ The resulting image contains: * `/build/out/curl-impersonate` - The curl binary that can impersonate Firefox. It is compiled statically against libcurl, nss, and libnghttp2 so that it won't conflict with any existing libraries on your system. You can use it from the container or copy it out. Tested to work on Ubuntu 20.04. * `/build/out/curl_ff91esr` - A wrapper script that launches `curl-impersonate` with the needed headers and ciphers to impersonate Firefox 91 ESR (Extended Support Release). * `/build/out/curl_ff95` - Same but with Firefox 95. +* `/build/out/libcurl-impersonate.so` - libcurl compiled with impersonation support. See [Usage](#usage) below for more details. If you use it outside this container: * Install dependencies: `sudo apt install libbrotli1` diff --git a/chrome/Dockerfile b/chrome/Dockerfile index 23afa4f..c16865a 100644 --- a/chrome/Dockerfile +++ b/chrome/Dockerfile @@ -73,8 +73,7 @@ RUN mkdir out && \ # Re-compile libcurl dynamically RUN cd ${CURL_VERSION} && \ - ./configure --enable-versioned-symbols \ - --with-openssl=/build/boringssl/build \ + ./configure --with-openssl=/build/boringssl/build \ --with-nghttp2=/usr/local \ LIBS="-pthread" \ CFLAGS="-I/build/boringssl/build" \ diff --git a/firefox/Dockerfile b/firefox/Dockerfile index 9fc274c..1c5f9ca 100644 --- a/firefox/Dockerfile +++ b/firefox/Dockerfile @@ -40,7 +40,7 @@ RUN cd ${NGHTTP2_VERSION} && \ # Compile nghttp2 RUN cd ${NGHTTP2_VERSION} && \ - ./configure && \ + ./configure --with-pic && \ make && make install # Download curl. @@ -48,17 +48,15 @@ ARG CURL_VERSION=curl-7.81.0 RUN curl -o ${CURL_VERSION}.tar.xz https://curl.se/download/${CURL_VERSION}.tar.xz RUN tar xf ${CURL_VERSION}.tar.xz -# Patch Curl. +# Patch curl and re-generate the configure script COPY patches/curl-*.patch ${CURL_VERSION}/ - -# Re-generate the configure script RUN cd ${CURL_VERSION} && \ for p in $(ls curl-*.patch); do patch -p1 < $p; done && \ autoreconf -fi # Compile curl with nss RUN cd ${CURL_VERSION} && \ - ./configure --with-nss=/build/${NSS_VERSION}/dist/Release --enable-static --disable-shared CFLAGS="-I/build/${NSS_VERSION}/dist/public/nss -I/build/${NSS_VERSION}/dist/Release/include/nspr" --with-nghttp2=/usr/local && \ + ./configure --with-nss=/build/${NSS_VERSION}/dist/Release --enable-static --disable-shared CFLAGS="-I/build/${NSS_VERSION}/dist/public/nss -I/build/${NSS_VERSION}/dist/Release/include/nspr" --with-nghttp2=/usr/local USE_CURL_SSLKEYLOGFILE=true && \ make # curl tries to load the CA certificates for libnss. @@ -66,13 +64,28 @@ RUN cd ${CURL_VERSION} && \ # which is supplied by libnss3 on Debian/Ubuntu RUN apt-get install -y libnss3 -# 'xxd' is needed for the wrapper curl_ff95 script -RUN apt-get install -y xxd - RUN mkdir out && \ - cp ${CURL_VERSION}/src/curl out/curl-impersonate + cp ${CURL_VERSION}/src/curl out/curl-impersonate && \ + strip out/curl-impersonate + +# Re-compile libcurl dynamically +RUN cd ${CURL_VERSION} && \ + ./configure --with-nss=/build/${NSS_VERSION}/dist/Release \ + --with-nghttp2=/usr/local \ + CFLAGS="-I/build/${NSS_VERSION}/dist/public/nss -I/build/${NSS_VERSION}/dist/Release/include/nspr" \ + USE_CURL_SSLKEYLOGFILE=true && \ + make clean && make + +# Rename to 'libcurl-impersonate' to avoid confusion, and recreate the +# symbolic links. +RUN ver=$(readlink -f curl-7.81.0/lib/.libs/libcurl.so | sed 's/.*so\.//') && \ + major=$(echo -n $ver | cut -d'.' -f1) && \ + cp "${CURL_VERSION}/lib/.libs/libcurl.so.$ver" "out/libcurl-impersonate.so.$ver" && \ + ln -s "libcurl-impersonate.so.$ver" "out/libcurl-impersonate.so.$major" && \ + ln -s "libcurl-impersonate.so.$ver" "out/libcurl-impersonate.so" && \ + strip "out/libcurl-impersonate.so.$ver" + # Wrapper script -COPY curl_* out/ - -RUN chmod +x out/* +COPY curl_ff* out/ +RUN chmod +x out/curl_* diff --git a/firefox/patches/curl-impersonate.patch b/firefox/patches/curl-impersonate.patch index 83034db..10dce73 100644 --- a/firefox/patches/curl-impersonate.patch +++ b/firefox/patches/curl-impersonate.patch @@ -21,6 +21,434 @@ index 63e320236..deb054300 100644 AC_MSG_NOTICE([-L is $LD_H2]) LDFLAGS="$LDFLAGS $LD_H2" +diff --git a/include/curl/curl.h b/include/curl/curl.h +index 7b69ce2d6..fe4bb36b9 100644 +--- a/include/curl/curl.h ++++ b/include/curl/curl.h +@@ -2135,6 +2135,10 @@ typedef enum { + /* Set MIME option flags. */ + CURLOPT(CURLOPT_MIME_OPTIONS, CURLOPTTYPE_LONG, 315), + ++ /* curl-impersonate: A list of headers used by the impersonated browser. ++ * If given, merged with CURLOPT_HTTPHEADER. */ ++ CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 316), ++ + CURLOPT_LASTENTRY /* the last unused */ + } CURLoption; + +diff --git a/include/curl/easy.h b/include/curl/easy.h +index 2dbfb26b5..e0bf86169 100644 +--- a/include/curl/easy.h ++++ b/include/curl/easy.h +@@ -41,6 +41,15 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); + CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); + CURL_EXTERN void curl_easy_cleanup(CURL *curl); + ++/* ++ * curl-impersonate: Tell libcurl to impersonate a browser. ++ * This is a wrapper function that calls curl_easy_setopt() ++ * multiple times with all the parameters required. That's also why it was ++ * created as a separate API function and not just as another option to ++ * curl_easy_setopt(). ++ */ ++CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target); ++ + /* + * NAME curl_easy_getinfo() + * +diff --git a/lib/easy.c b/lib/easy.c +index 20293a710..b0b4c2751 100644 +--- a/lib/easy.c ++++ b/lib/easy.c +@@ -80,6 +80,7 @@ + #include "dynbuf.h" + #include "altsvc.h" + #include "hsts.h" ++#include "strcase.h" + + /* The last 3 #include files should be in this order */ + #include "curl_printf.h" +@@ -282,6 +283,162 @@ void curl_global_cleanup(void) + init_flags = 0; + } + ++/* ++ * curl-impersonate: Options to be set for each supported target browser. ++ * Note: this does not include the HTTP headers, which are handled separately ++ * in Curl_http(). ++ */ ++#define IMPERSONATE_MAX_HEADERS 32 ++static const struct impersonate_opts { ++ const char *target; ++ int httpversion; ++ int ssl_version; ++ const char *ciphers; ++ const char *http_headers[IMPERSONATE_MAX_HEADERS]; ++ /* Other TLS options will come here in the future once they are ++ * configurable through curl_easy_setopt() */ ++} impersonations[] = { ++ { ++ .target = "ff91esr", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "aes_128_gcm_sha_256," ++ "chacha20_poly1305_sha_256," ++ "aes_256_gcm_sha_384," ++ "ecdhe_ecdsa_aes_128_gcm_sha_256," ++ "ecdhe_rsa_aes_128_gcm_sha_256," ++ "ecdhe_ecdsa_chacha20_poly1305_sha_256," ++ "ecdhe_rsa_chacha20_poly1305_sha_256," ++ "ecdhe_ecdsa_aes_256_gcm_sha_384," ++ "ecdhe_rsa_aes_256_gcm_sha_384," ++ "ecdhe_ecdsa_aes_256_sha," ++ "ecdhe_ecdsa_aes_128_sha," ++ "ecdhe_rsa_aes_128_sha," ++ "ecdhe_rsa_aes_256_sha," ++ "rsa_aes_128_gcm_sha_256," ++ "rsa_aes_256_gcm_sha_384," ++ "rsa_aes_128_sha," ++ "rsa_aes_256_sha," ++ "rsa_3des_ede_cbc_sha", ++ .http_headers = { ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", ++ "Accept-Language: en-US,en;q=0.5", ++ "Accept-Encoding: gzip, deflate, br", ++ "Connection: keep-alive", ++ "Upgrade-Insecure-Requests: 1", ++ "Sec-Fetch-Dest: document", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-User: ?1" ++ } ++ }, ++ { ++ .target = "ff95", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "aes_128_gcm_sha_256," ++ "chacha20_poly1305_sha_256," ++ "aes_256_gcm_sha_384," ++ "ecdhe_ecdsa_aes_128_gcm_sha_256," ++ "ecdhe_rsa_aes_128_gcm_sha_256," ++ "ecdhe_ecdsa_chacha20_poly1305_sha_256," ++ "ecdhe_rsa_chacha20_poly1305_sha_256," ++ "ecdhe_ecdsa_aes_256_gcm_sha_384," ++ "ecdhe_rsa_aes_256_gcm_sha_384," ++ "ecdhe_ecdsa_aes_256_sha," ++ "ecdhe_ecdsa_aes_128_sha," ++ "ecdhe_rsa_aes_128_sha," ++ "ecdhe_rsa_aes_256_sha," ++ "rsa_aes_128_gcm_sha_256," ++ "rsa_aes_256_gcm_sha_384," ++ "rsa_aes_128_sha," ++ "rsa_aes_256_sha", ++ .http_headers = { ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", ++ "Accept-Language: en-US,en;q=0.5", ++ "Accept-Encoding: gzip, deflate, br", ++ "Connection: keep-alive", ++ "Upgrade-Insecure-Requests: 1", ++ "Sec-Fetch-Dest: document", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-User: ?1" ++ } ++ } ++}; ++ ++#define NUM_IMPERSONATIONS \ ++ sizeof(impersonations) / sizeof(impersonations[0]) ++ ++/* ++ * curl-impersonate: ++ * Call curl_easy_setopt() with all the needed options as defined in the ++ * 'impersonations' array. ++ * */ ++CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target) ++{ ++ int i; ++ int ret; ++ const struct impersonate_opts *opts = NULL; ++ struct curl_slist *headers = NULL; ++ ++ for(i = 0; i < NUM_IMPERSONATIONS; i++) { ++ if (Curl_strncasecompare(target, ++ impersonations[i].target, ++ strlen(impersonations[i].target))) { ++ opts = &impersonations[i]; ++ break; ++ } ++ } ++ ++ if(!opts) { ++ DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n", ++ target)); ++ return CURLE_BAD_FUNCTION_ARGUMENT; ++ } ++ ++ if(opts->httpversion != CURL_HTTP_VERSION_NONE) { ++ ret = curl_easy_setopt(data, CURLOPT_HTTP_VERSION, opts->httpversion); ++ if(ret) ++ return ret; ++ } ++ ++ if (opts->ssl_version != CURL_SSLVERSION_DEFAULT) { ++ ret = curl_easy_setopt(data, CURLOPT_SSLVERSION, opts->ssl_version); ++ if(ret) ++ return ret; ++ } ++ ++ if(opts->ciphers) { ++ ret = curl_easy_setopt(data, CURLOPT_SSL_CIPHER_LIST, opts->ciphers); ++ if (ret) ++ return ret; ++ } ++ ++ /* Build a linked list out of the static array of headers. */ ++ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) { ++ if(opts->http_headers[i]) { ++ headers = curl_slist_append(headers, opts->http_headers[i]); ++ if(!headers) { ++ return CURLE_OUT_OF_MEMORY; ++ } ++ } ++ } ++ ++ if(headers) { ++ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers); ++ curl_slist_free_all(headers); ++ if(ret) ++ return ret; ++ } ++ ++ return CURLE_OK; ++} ++ + /* + * curl_easy_init() is the external interface to alloc, setup and init an + * easy handle that is returned. If anything goes wrong, NULL is returned. +@@ -290,6 +447,7 @@ struct Curl_easy *curl_easy_init(void) + { + CURLcode result; + struct Curl_easy *data; ++ char *target; + + /* Make sure we inited the global SSL stuff */ + if(!initialized) { +@@ -308,6 +466,22 @@ struct Curl_easy *curl_easy_init(void) + return NULL; + } + ++ /* ++ * curl-impersonate: Hook into curl_easy_init() to set the required options ++ * from an environment variable. ++ * This is a bit hacky but allows seamless integration of libcurl-impersonate ++ * without code modifications to the app. ++ */ ++ target = curl_getenv("CURL_IMPERSONATE"); ++ if(target) { ++ result = curl_easy_impersonate(data, target); ++ free(target); ++ if(result) { ++ Curl_close(&data); ++ return NULL; ++ } ++ } ++ + return data; + } + +@@ -878,6 +1052,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) + outcurl->state.referer_alloc = TRUE; + } + ++ if(data->state.base_headers) { ++ outcurl->state.base_headers = ++ Curl_slist_duplicate(data->state.base_headers); ++ if(!outcurl->state.base_headers) ++ goto fail; ++ } ++ + /* Reinitialize an SSL engine for the new handle + * note: the engine name has already been copied by dupset */ + if(outcurl->set.str[STRING_SSL_ENGINE]) { +diff --git a/lib/easyoptions.c b/lib/easyoptions.c +index 04871ad1e..cd5998146 100644 +--- a/lib/easyoptions.c ++++ b/lib/easyoptions.c +@@ -130,6 +130,7 @@ struct curl_easyoption Curl_easyopts[] = { + {"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0}, + {"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0}, + {"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0}, ++ {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0}, + {"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0}, + {"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0}, + {"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0}, +diff --git a/lib/http.c b/lib/http.c +index f08a343e3..879151dd2 100644 +--- a/lib/http.c ++++ b/lib/http.c +@@ -84,6 +84,7 @@ + #include "altsvc.h" + #include "hsts.h" + #include "c-hyper.h" ++#include "slist.h" + + /* The last 3 #include files should be in this order */ + #include "curl_printf.h" +@@ -1795,6 +1796,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, + int numlists = 1; /* by default */ + int i; + ++ /* ++ * curl-impersonate: Use the merged list of headers if it exists (i.e. when ++ * the CURLOPT_HTTPBASEHEADER option was set. ++ */ ++ struct curl_slist *noproxyheaders = ++ (data->state.merged_headers ? ++ data->state.merged_headers : ++ data->set.headers); ++ + #ifndef CURL_DISABLE_PROXY + enum proxy_use proxy; + +@@ -1806,10 +1816,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, + + switch(proxy) { + case HEADER_SERVER: +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + break; + case HEADER_PROXY: +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + if(data->set.sep_headers) { + h[1] = data->set.proxyheaders; + numlists++; +@@ -1819,12 +1829,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, + if(data->set.sep_headers) + h[0] = data->set.proxyheaders; + else +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + break; + } + #else + (void)is_connect; +- h[0] = data->set.headers; ++ h[0] = noproxyheaders; + #endif + + /* loop through one or two lists */ +@@ -2059,6 +2069,92 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn, + *reqp = httpreq; + } + ++/* ++ * curl-impersonate: ++ * Create a new linked list of headers. ++ * The new list is a merge between the "base" headers and the application given ++ * headers. The "base" headers contain curl-impersonate's list of headers ++ * used by default by the impersonated browser. ++ * ++ * The application given headers will override the "base" headers if supplied. ++ */ ++CURLcode Curl_http_merge_headers(struct Curl_easy *data) ++{ ++ int i; ++ int ret; ++ struct curl_slist *head; ++ struct curl_slist *dup = NULL; ++ struct curl_slist *new_list = NULL; ++ ++ if (!data->state.base_headers) ++ return CURLE_OK; ++ ++ /* Duplicate the list for temporary use. */ ++ if (data->set.headers) { ++ dup = Curl_slist_duplicate(data->set.headers); ++ if(!dup) ++ return CURLE_OUT_OF_MEMORY; ++ } ++ ++ for(head = data->state.base_headers; head; head = head->next) { ++ char *sep; ++ size_t prefix_len; ++ bool found = FALSE; ++ struct curl_slist *head2; ++ ++ sep = strchr(head->data, ':'); ++ if(!sep) ++ continue; ++ ++ prefix_len = sep - head->data; ++ ++ /* Check if this header was added by the application. */ ++ for(head2 = dup; head2; head2 = head2->next) { ++ if(head2->data && ++ strncasecompare(head2->data, head->data, prefix_len) && ++ Curl_headersep(head2->data[prefix_len]) ) { ++ new_list = curl_slist_append(new_list, head2->data); ++ /* Free and set to NULL to mark that it's been added. */ ++ Curl_safefree(head2->data); ++ found = TRUE; ++ break; ++ } ++ } ++ ++ if (!found) { ++ new_list = curl_slist_append(new_list, head->data); ++ } ++ ++ if (!new_list) { ++ ret = CURLE_OUT_OF_MEMORY; ++ goto fail; ++ } ++ } ++ ++ /* Now go over any additional application-supplied headers. */ ++ for(head = dup; head; head = head->next) { ++ if(head->data) { ++ new_list = curl_slist_append(new_list, head->data); ++ if(!new_list) { ++ ret = CURLE_OUT_OF_MEMORY; ++ goto fail; ++ } ++ } ++ } ++ ++ curl_slist_free_all(dup); ++ /* Save the new, merged list separately, so it can be freed later. */ ++ curl_slist_free_all(data->state.merged_headers); ++ data->state.merged_headers = new_list; ++ ++ return CURLE_OK; ++ ++fail: ++ Curl_safefree(dup); ++ curl_slist_free_all(new_list); ++ return ret; ++} ++ + CURLcode Curl_http_useragent(struct Curl_easy *data) + { + /* The User-Agent string might have been allocated in url.c already, because +@@ -3067,6 +3163,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done) + if(result) + return result; + ++ /* curl-impersonate: Add HTTP headers to impersonate real browsers. */ ++ result = Curl_http_merge_headers(data); ++ if (result) ++ return result; ++ + result = Curl_http_useragent(data); + if(result) + return result; diff --git a/lib/http2.c b/lib/http2.c index e74400a4c..1f4d496f3 100644 --- a/lib/http2.c @@ -35,6 +463,103 @@ index e74400a4c..1f4d496f3 100644 /* USHRT_MAX is 65535 == 0xffff */ #define HEADER_OVERFLOW(x) \ +diff --git a/lib/setopt.c b/lib/setopt.c +index 599ed5d99..1baa48e70 100644 +--- a/lib/setopt.c ++++ b/lib/setopt.c +@@ -48,6 +48,7 @@ + #include "multiif.h" + #include "altsvc.h" + #include "hsts.h" ++#include "slist.h" + + /* The last 3 #include files should be in this order */ + #include "curl_printf.h" +@@ -688,6 +689,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) + va_arg(param, char *)); + break; + ++ case CURLOPT_HTTPBASEHEADER: ++ /* ++ * curl-impersonate: ++ * Set a list of "base" headers. These will be merged with any headers ++ * set by CURLOPT_HTTPHEADER. curl-impersonate uses this option in order ++ * to set a list of default browser headers. ++ * ++ * Unlike CURLOPT_HTTPHEADER, ++ * the list is copied and can be immediately freed by the user. ++ */ ++ curl_slist_free_all(data->state.base_headers); ++ data->state.base_headers = \ ++ Curl_slist_duplicate(va_arg(param, struct curl_slist *)); ++ if (!data->state.base_headers) ++ result = CURLE_OUT_OF_MEMORY; ++ break; ++ + case CURLOPT_HTTPHEADER: + /* + * Set a list with HTTP headers to use (or replace internals with) +diff --git a/lib/transfer.c b/lib/transfer.c +index 22704fa15..1e100140c 100644 +--- a/lib/transfer.c ++++ b/lib/transfer.c +@@ -102,7 +102,15 @@ char *Curl_checkheaders(const struct Curl_easy *data, + DEBUGASSERT(thislen); + DEBUGASSERT(thisheader[thislen-1] != ':'); + +- for(head = data->set.headers; head; head = head->next) { ++ /* ++ * curl-impersonate: ++ * Check if we have overriden the user-supplied list of headers. ++ */ ++ head = data->set.headers; ++ if (data->state.merged_headers) ++ head = data->state.merged_headers; ++ ++ for(; head; head = head->next) { + if(strncasecompare(head->data, thisheader, thislen) && + Curl_headersep(head->data[thislen]) ) + return head->data; +diff --git a/lib/url.c b/lib/url.c +index 9f1013554..f0f266797 100644 +--- a/lib/url.c ++++ b/lib/url.c +@@ -469,6 +469,11 @@ CURLcode Curl_close(struct Curl_easy **datap) + Curl_safefree(data->state.aptr.proxyuser); + Curl_safefree(data->state.aptr.proxypasswd); + ++ /* curl-impersonate: Free the list set by CURLOPT_HTTPBASEHEADER. */ ++ curl_slist_free_all(data->state.base_headers); ++ /* curl-impersonate: Free the dynamic list of headers. */ ++ curl_slist_free_all(data->state.merged_headers); ++ + #ifndef CURL_DISABLE_DOH + if(data->req.doh) { + Curl_dyn_free(&data->req.doh->probe[0].serverdoh); +diff --git a/lib/urldata.h b/lib/urldata.h +index cc9c88870..a35a20e10 100644 +--- a/lib/urldata.h ++++ b/lib/urldata.h +@@ -1421,6 +1421,19 @@ struct UrlState { + CURLcode hresult; /* used to pass return codes back from hyper callbacks */ + #endif + ++ /* ++ * curl-impersonate: ++ * List of "base" headers set by CURLOPT_HTTPBASEHEADER. ++ */ ++ struct curl_slist *base_headers; ++ /* ++ * curl-impersonate: ++ * Dynamically-constructed list of HTTP headers. ++ * This list is a merge of the default HTTP headers needed to impersonate a ++ * browser, together with any user-supplied headers. ++ */ ++ struct curl_slist *merged_headers; ++ + /* Dynamically allocated strings, MUST be freed before this struct is + killed. */ + struct dynamically_allocated_data { diff --git a/lib/vtls/nss.c b/lib/vtls/nss.c index 2b44f0512..4c60797c7 100644 --- a/lib/vtls/nss.c diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index db8efa8..672256c 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -203,6 +203,22 @@ class TestImpersonation: "CURL_IMPERSONATE": "edge98" }, "edge_98.0.1108.62_win10" + ), + ( + "./minicurl", + { + "LD_PRELOAD": "./firefox/libcurl-impersonate.so", + "CURL_IMPERSONATE": "ff91esr" + }, + "firefox_91.6.0esr_win10" + ), + ( + "./minicurl", + { + "LD_PRELOAD": "./firefox/libcurl-impersonate.so", + "CURL_IMPERSONATE": "ff95" + }, + "firefox_95.0.2_win10" ) ] )