hoelee/curl-impersonate
1
0
Fork 0
mirror of https://github.com/lwthiker/curl-impersonate.git synced 2025-08-08 04:42:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
Files
4e2e782e5f44150b46747076e4c98eb07036dd75
curl-impersonate/chrome/patches/curl-impersonate.patch
lwthiker 4e2e782e5f Add Edge 98 signature to libcurl-impersonate
2022-02-27 23:48:17 +02:00

818 lines
26 KiB
Diff
Raw Blame History

diff --git a/configure.ac b/configure.ac
index 63e320236..deb054300 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2573,15 +2573,15 @@ if test X"$want_nghttp2" != Xno; then
if test "$PKGCONFIG" != "no" ; then
LIB_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path])
- $PKGCONFIG --libs-only-l libnghttp2`
+ $PKGCONFIG --static --libs-only-l libnghttp2`
AC_MSG_NOTICE([-l is $LIB_H2])
CPP_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path]) dnl
- $PKGCONFIG --cflags-only-I libnghttp2`
+ $PKGCONFIG --static --cflags-only-I libnghttp2`
AC_MSG_NOTICE([-I is $CPP_H2])
LD_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path])
- $PKGCONFIG --libs-only-L libnghttp2`
+ $PKGCONFIG --static --libs-only-L libnghttp2`
AC_MSG_NOTICE([-L is $LD_H2])
LDFLAGS="$LDFLAGS $LD_H2"
diff --git a/include/curl/curl.h b/include/curl/curl.h
index 7b69ce2d6..fe4bb36b9 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -2135,6 +2135,10 @@ typedef enum {
/* Set MIME option flags. */
CURLOPT(CURLOPT_MIME_OPTIONS, CURLOPTTYPE_LONG, 315),
+ /* curl-impersonate: A list of headers used by the impersonated browser.
+ * If given, merged with CURLOPT_HTTPHEADER. */
+ CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 316),
+
CURLOPT_LASTENTRY /* the last unused */
} CURLoption;
diff --git a/include/curl/easy.h b/include/curl/easy.h
index 2dbfb26b5..e0bf86169 100644
--- a/include/curl/easy.h
+++ b/include/curl/easy.h
@@ -41,6 +41,15 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...);
CURL_EXTERN CURLcode curl_easy_perform(CURL *curl);
CURL_EXTERN void curl_easy_cleanup(CURL *curl);
+/*
+ * curl-impersonate: Tell libcurl to impersonate a browser.
+ * This is a wrapper function that calls curl_easy_setopt()
+ * multiple times with all the parameters required. That's also why it was
+ * created as a separate API function and not just as another option to
+ * curl_easy_setopt().
+ */
+CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target);
+
/*
* NAME curl_easy_getinfo()
*
diff --git a/lib/easy.c b/lib/easy.c
index 20293a710..ec16aee23 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -80,6 +80,7 @@
#include "dynbuf.h"
#include "altsvc.h"
#include "hsts.h"
+#include "strcase.h"
/* The last 3 #include files should be in this order */
#include "curl_printf.h"
@@ -282,6 +283,161 @@ void curl_global_cleanup(void)
init_flags = 0;
}
+/*
+ * curl-impersonate: Options to be set for each supported target browser.
+ * Note: this does not include the HTTP headers, which are handled separately
+ * in Curl_http().
+ */
+#define IMPERSONATE_MAX_HEADERS 32
+static const struct impersonate_opts {
+ const char *target;
+ int httpversion;
+ int ssl_version;
+ const char *ciphers;
+ const char *http_headers[IMPERSONATE_MAX_HEADERS];
+ /* Other TLS options will come here in the future once they are
+ * configurable through curl_easy_setopt() */
+} impersonations[] = {
+ {
+ .target = "chrome98",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "TLS_AES_128_GCM_SHA256,"
+ "TLS_AES_256_GCM_SHA384,"
+ "TLS_CHACHA20_POLY1305_SHA256,"
+ "ECDHE-ECDSA-AES128-GCM-SHA256,"
+ "ECDHE-RSA-AES128-GCM-SHA256,"
+ "ECDHE-ECDSA-AES256-GCM-SHA384,"
+ "ECDHE-RSA-AES256-GCM-SHA384,"
+ "ECDHE-ECDSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-AES128-SHA,"
+ "ECDHE-RSA-AES256-SHA,"
+ "AES128-GCM-SHA256,"
+ "AES256-GCM-SHA384,"
+ "AES128-SHA,"
+ "AES256-SHA",
+ .http_headers = {
+ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"98\", \"Google Chrome\";v=\"98\"",
+ "sec-ch-ua-mobile: ?0",
+ "sec-ch-ua-platform: \"Windows\"",
+ "Upgrade-Insecure-Requests: 1",
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-User: ?1",
+ "Sec-Fetch-Dest: document",
+ "Accept-Encoding: gzip, deflate, br",
+ "Accept-Language: en-US,en;q=0.9"
+ }
+ },
+ {
+ .target = "edge98",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "TLS_AES_128_GCM_SHA256,"
+ "TLS_AES_256_GCM_SHA384,"
+ "TLS_CHACHA20_POLY1305_SHA256,"
+ "ECDHE-ECDSA-AES128-GCM-SHA256,"
+ "ECDHE-RSA-AES128-GCM-SHA256,"
+ "ECDHE-ECDSA-AES256-GCM-SHA384,"
+ "ECDHE-RSA-AES256-GCM-SHA384,"
+ "ECDHE-ECDSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-AES128-SHA,"
+ "ECDHE-RSA-AES256-SHA,"
+ "AES128-GCM-SHA256,"
+ "AES256-GCM-SHA384,"
+ "AES128-SHA,"
+ "AES256-SHA",
+ .http_headers = {
+ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"98\", \"Microsoft Edge\";v=\"98\"",
+ "sec-ch-ua-mobile: ?0",
+ "sec-ch-ua-platform: \"Windows\"",
+ "Upgrade-Insecure-Requests: 1",
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.62",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-User: ?1",
+ "Sec-Fetch-Dest: document",
+ "Accept-Encoding: gzip, deflate, br",
+ "Accept-Language: en-US,en;q=0.9"
+ }
+ }
+};
+
+#define NUM_IMPERSONATIONS \
+ sizeof(impersonations) / sizeof(impersonations[0])
+
+/*
+ * curl-impersonate:
+ * Call curl_easy_setopt() with all the needed options as defined in the
+ * 'impersonations' array.
+ * */
+CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target)
+{
+ int i;
+ int ret;
+ const struct impersonate_opts *opts = NULL;
+ struct curl_slist *headers = NULL;
+
+ for(i = 0; i < NUM_IMPERSONATIONS; i++) {
+ if (Curl_strncasecompare(target,
+ impersonations[i].target,
+ strlen(impersonations[i].target))) {
+ opts = &impersonations[i];
+ break;
+ }
+ }
+
+ if(!opts) {
+ DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n",
+ target));
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+ }
+
+ if(opts->httpversion != CURL_HTTP_VERSION_NONE) {
+ ret = curl_easy_setopt(data, CURLOPT_HTTP_VERSION, opts->httpversion);
+ if(ret)
+ return ret;
+ }
+
+ if (opts->ssl_version != CURL_SSLVERSION_DEFAULT) {
+ ret = curl_easy_setopt(data, CURLOPT_SSLVERSION, opts->ssl_version);
+ if(ret)
+ return ret;
+ }
+
+ if(opts->ciphers) {
+ ret = curl_easy_setopt(data, CURLOPT_SSL_CIPHER_LIST, opts->ciphers);
+ if (ret)
+ return ret;
+ }
+
+ /* Build a linked list out of the static array of headers. */
+ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) {
+ if(opts->http_headers[i]) {
+ headers = curl_slist_append(headers, opts->http_headers[i]);
+ if(!headers) {
+ return CURLE_OUT_OF_MEMORY;
+ }
+ }
+ }
+
+ if(headers) {
+ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers);
+ curl_slist_free_all(headers);
+ if(ret)
+ return ret;
+ }
+
+ return CURLE_OK;
+}
+
/*
* curl_easy_init() is the external interface to alloc, setup and init an
* easy handle that is returned. If anything goes wrong, NULL is returned.
@@ -290,6 +446,7 @@ struct Curl_easy *curl_easy_init(void)
{
CURLcode result;
struct Curl_easy *data;
+ char *target;
/* Make sure we inited the global SSL stuff */
if(!initialized) {
@@ -308,6 +465,22 @@ struct Curl_easy *curl_easy_init(void)
return NULL;
}
+ /*
+ * curl-impersonate: Hook into curl_easy_init() to set the required options
+ * from an environment variable.
+ * This is a bit hacky but allows seamless integration of libcurl-impersonate
+ * without code modifications to the app.
+ */
+ target = curl_getenv("CURL_IMPERSONATE");
+ if(target) {
+ result = curl_easy_impersonate(data, target);
+ free(target);
+ if(result) {
+ Curl_close(&data);
+ return NULL;
+ }
+ }
+
return data;
}
@@ -878,6 +1051,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
outcurl->state.referer_alloc = TRUE;
}
+ if(data->state.base_headers) {
+ outcurl->state.base_headers =
+ Curl_slist_duplicate(data->state.base_headers);
+ if(!outcurl->state.base_headers)
+ goto fail;
+ }
+
/* Reinitialize an SSL engine for the new handle
* note: the engine name has already been copied by dupset */
if(outcurl->set.str[STRING_SSL_ENGINE]) {
diff --git a/lib/easyoptions.c b/lib/easyoptions.c
index 04871ad1e..cd5998146 100644
--- a/lib/easyoptions.c
+++ b/lib/easyoptions.c
@@ -130,6 +130,7 @@ struct curl_easyoption Curl_easyopts[] = {
{"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0},
{"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0},
{"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0},
+ {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0},
{"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0},
{"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0},
{"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0},
diff --git a/lib/http.c b/lib/http.c
index f08a343e3..879151dd2 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -84,6 +84,7 @@
#include "altsvc.h"
#include "hsts.h"
#include "c-hyper.h"
+#include "slist.h"
/* The last 3 #include files should be in this order */
#include "curl_printf.h"
@@ -1795,6 +1796,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
int numlists = 1; /* by default */
int i;
+ /*
+ * curl-impersonate: Use the merged list of headers if it exists (i.e. when
+ * the CURLOPT_HTTPBASEHEADER option was set.
+ */
+ struct curl_slist *noproxyheaders =
+ (data->state.merged_headers ?
+ data->state.merged_headers :
+ data->set.headers);
+
#ifndef CURL_DISABLE_PROXY
enum proxy_use proxy;
@@ -1806,10 +1816,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
switch(proxy) {
case HEADER_SERVER:
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
break;
case HEADER_PROXY:
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
if(data->set.sep_headers) {
h[1] = data->set.proxyheaders;
numlists++;
@@ -1819,12 +1829,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
if(data->set.sep_headers)
h[0] = data->set.proxyheaders;
else
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
break;
}
#else
(void)is_connect;
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
#endif
/* loop through one or two lists */
@@ -2059,6 +2069,92 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn,
*reqp = httpreq;
}
+/*
+ * curl-impersonate:
+ * Create a new linked list of headers.
+ * The new list is a merge between the "base" headers and the application given
+ * headers. The "base" headers contain curl-impersonate's list of headers
+ * used by default by the impersonated browser.
+ *
+ * The application given headers will override the "base" headers if supplied.
+ */
+CURLcode Curl_http_merge_headers(struct Curl_easy *data)
+{
+ int i;
+ int ret;
+ struct curl_slist *head;
+ struct curl_slist *dup = NULL;
+ struct curl_slist *new_list = NULL;
+
+ if (!data->state.base_headers)
+ return CURLE_OK;
+
+ /* Duplicate the list for temporary use. */
+ if (data->set.headers) {
+ dup = Curl_slist_duplicate(data->set.headers);
+ if(!dup)
+ return CURLE_OUT_OF_MEMORY;
+ }
+
+ for(head = data->state.base_headers; head; head = head->next) {
+ char *sep;
+ size_t prefix_len;
+ bool found = FALSE;
+ struct curl_slist *head2;
+
+ sep = strchr(head->data, ':');
+ if(!sep)
+ continue;
+
+ prefix_len = sep - head->data;
+
+ /* Check if this header was added by the application. */
+ for(head2 = dup; head2; head2 = head2->next) {
+ if(head2->data &&
+ strncasecompare(head2->data, head->data, prefix_len) &&
+ Curl_headersep(head2->data[prefix_len]) ) {
+ new_list = curl_slist_append(new_list, head2->data);
+ /* Free and set to NULL to mark that it's been added. */
+ Curl_safefree(head2->data);
+ found = TRUE;
+ break;
+ }
+ }
+
+ if (!found) {
+ new_list = curl_slist_append(new_list, head->data);
+ }
+
+ if (!new_list) {
+ ret = CURLE_OUT_OF_MEMORY;
+ goto fail;
+ }
+ }
+
+ /* Now go over any additional application-supplied headers. */
+ for(head = dup; head; head = head->next) {
+ if(head->data) {
+ new_list = curl_slist_append(new_list, head->data);
+ if(!new_list) {
+ ret = CURLE_OUT_OF_MEMORY;
+ goto fail;
+ }
+ }
+ }
+
+ curl_slist_free_all(dup);
+ /* Save the new, merged list separately, so it can be freed later. */
+ curl_slist_free_all(data->state.merged_headers);
+ data->state.merged_headers = new_list;
+
+ return CURLE_OK;
+
+fail:
+ Curl_safefree(dup);
+ curl_slist_free_all(new_list);
+ return ret;
+}
+
CURLcode Curl_http_useragent(struct Curl_easy *data)
{
/* The User-Agent string might have been allocated in url.c already, because
@@ -3067,6 +3163,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
if(result)
return result;
+ /* curl-impersonate: Add HTTP headers to impersonate real browsers. */
+ result = Curl_http_merge_headers(data);
+ if (result)
+ return result;
+
result = Curl_http_useragent(data);
if(result)
return result;
diff --git a/lib/http.h b/lib/http.h
index b4aaba2a2..1cf65c4b1 100644
--- a/lib/http.h
+++ b/lib/http.h
@@ -278,7 +278,8 @@ struct http_conn {
struct h2settings settings;
/* list of settings that will be sent */
- nghttp2_settings_entry local_settings[3];
+ /* curl-impersonate: Align HTTP/2 settings to Chrome's */
+ nghttp2_settings_entry local_settings[5];
size_t local_settings_num;
#else
int unused; /* prevent a compiler warning */
diff --git a/lib/http2.c b/lib/http2.c
index e74400a4c..33197df20 100644
--- a/lib/http2.c
+++ b/lib/http2.c
@@ -41,6 +41,7 @@
#include "curl_printf.h"
#include "curl_memory.h"
#include "memdebug.h"
+#include "rand.h"
#define H2_BUFSIZE 32768
@@ -1193,16 +1194,27 @@ static void populate_settings(struct Curl_easy *data,
{
nghttp2_settings_entry *iv = httpc->local_settings;
- iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
- iv[0].value = Curl_multi_max_concurrent_streams(data->multi);
+ /* curl-impersonate: Align HTTP/2 settings to Chrome's */
+ iv[0].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE;
+ iv[0].value = 0x10000;
- iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
- iv[1].value = HTTP2_HUGE_WINDOW_SIZE;
+ iv[1].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
+ iv[1].value = Curl_multi_max_concurrent_streams(data->multi);
- iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
- iv[2].value = data->multi->push_cb != NULL;
+ iv[2].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
+ iv[2].value = 0x600000;
- httpc->local_settings_num = 3;
+ iv[3].settings_id = NGHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE;
+ iv[3].value = 0x40000;
+
+ // iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
+ // iv[2].value = data->multi->push_cb != NULL;
+
+ // Looks like random setting set by Chrome, maybe similar to TLS GREASE. */
+ Curl_rand(data, (unsigned char *)&iv[4].settings_id, sizeof(iv[4].settings_id));
+ Curl_rand(data, (unsigned char *)&iv[4].value, sizeof(iv[4].value));
+
+ httpc->local_settings_num = 5;
}
void Curl_http2_done(struct Curl_easy *data, bool premature)
@@ -1818,7 +1830,8 @@ static ssize_t http2_recv(struct Curl_easy *data, int sockindex,
/* Index where :authority header field will appear in request header
field list. */
-#define AUTHORITY_DST_IDX 3
+/* curl-impersonate: Put the ":authority" header in the first place. */
+#define AUTHORITY_DST_IDX 1
/* USHRT_MAX is 65535 == 0xffff */
#define HEADER_OVERFLOW(x) \
@@ -2032,25 +2045,26 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
}
if(!end || end == hdbuf)
goto fail;
- nva[1].name = (unsigned char *)":path";
- nva[1].namelen = strlen((char *)nva[1].name);
- nva[1].value = (unsigned char *)hdbuf;
- nva[1].valuelen = (size_t)(end - hdbuf);
- nva[1].flags = NGHTTP2_NV_FLAG_NONE;
- if(HEADER_OVERFLOW(nva[1])) {
+ /* curl-impersonate: Switch the places of ":path" and ":scheme". */
+ nva[2].name = (unsigned char *)":path";
+ nva[2].namelen = strlen((char *)nva[2].name);
+ nva[2].value = (unsigned char *)hdbuf;
+ nva[2].valuelen = (size_t)(end - hdbuf);
+ nva[2].flags = NGHTTP2_NV_FLAG_NONE;
+ if(HEADER_OVERFLOW(nva[2])) {
failf(data, "Failed sending HTTP request: Header overflow");
goto fail;
}
- nva[2].name = (unsigned char *)":scheme";
- nva[2].namelen = strlen((char *)nva[2].name);
+ nva[1].name = (unsigned char *)":scheme";
+ nva[1].namelen = strlen((char *)nva[1].name);
if(conn->handler->flags & PROTOPT_SSL)
- nva[2].value = (unsigned char *)"https";
+ nva[1].value = (unsigned char *)"https";
else
- nva[2].value = (unsigned char *)"http";
- nva[2].valuelen = strlen((char *)nva[2].value);
- nva[2].flags = NGHTTP2_NV_FLAG_NONE;
- if(HEADER_OVERFLOW(nva[2])) {
+ nva[1].value = (unsigned char *)"http";
+ nva[1].valuelen = strlen((char *)nva[1].value);
+ nva[1].flags = NGHTTP2_NV_FLAG_NONE;
+ if(HEADER_OVERFLOW(nva[1])) {
failf(data, "Failed sending HTTP request: Header overflow");
goto fail;
}
diff --git a/lib/http2.h b/lib/http2.h
index d6986d97f..fa5c90e7f 100644
--- a/lib/http2.h
+++ b/lib/http2.h
@@ -29,7 +29,8 @@
/* value for MAX_CONCURRENT_STREAMS we use until we get an updated setting
from the peer */
-#define DEFAULT_MAX_CONCURRENT_STREAMS 100
+/* curl-impersonate: Use 1000 concurrent streams like Chrome. */
+#define DEFAULT_MAX_CONCURRENT_STREAMS 1000
/*
* Store nghttp2 version info in this buffer.
diff --git a/lib/multi.c b/lib/multi.c
index f8dcc63b4..e6b728592 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -393,7 +393,8 @@ struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */
/* -1 means it not set by user, use the default value */
multi->maxconnects = -1;
- multi->max_concurrent_streams = 100;
+ /* curl-impersonate: Use 1000 concurrent streams like Chrome. */
+ multi->max_concurrent_streams = 1000;
multi->ipv6_works = Curl_ipv6works(NULL);
#ifdef USE_WINSOCK
diff --git a/lib/setopt.c b/lib/setopt.c
index 599ed5d99..1baa48e70 100644
--- a/lib/setopt.c
+++ b/lib/setopt.c
@@ -48,6 +48,7 @@
#include "multiif.h"
#include "altsvc.h"
#include "hsts.h"
+#include "slist.h"
/* The last 3 #include files should be in this order */
#include "curl_printf.h"
@@ -688,6 +689,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
va_arg(param, char *));
break;
+ case CURLOPT_HTTPBASEHEADER:
+ /*
+ * curl-impersonate:
+ * Set a list of "base" headers. These will be merged with any headers
+ * set by CURLOPT_HTTPHEADER. curl-impersonate uses this option in order
+ * to set a list of default browser headers.
+ *
+ * Unlike CURLOPT_HTTPHEADER,
+ * the list is copied and can be immediately freed by the user.
+ */
+ curl_slist_free_all(data->state.base_headers);
+ data->state.base_headers = \
+ Curl_slist_duplicate(va_arg(param, struct curl_slist *));
+ if (!data->state.base_headers)
+ result = CURLE_OUT_OF_MEMORY;
+ break;
+
case CURLOPT_HTTPHEADER:
/*
* Set a list with HTTP headers to use (or replace internals with)
diff --git a/lib/transfer.c b/lib/transfer.c
index 22704fa15..1e100140c 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -102,7 +102,15 @@ char *Curl_checkheaders(const struct Curl_easy *data,
DEBUGASSERT(thislen);
DEBUGASSERT(thisheader[thislen-1] != ':');
- for(head = data->set.headers; head; head = head->next) {
+ /*
+ * curl-impersonate:
+ * Check if we have overriden the user-supplied list of headers.
+ */
+ head = data->set.headers;
+ if (data->state.merged_headers)
+ head = data->state.merged_headers;
+
+ for(; head; head = head->next) {
if(strncasecompare(head->data, thisheader, thislen) &&
Curl_headersep(head->data[thislen]) )
return head->data;
diff --git a/lib/url.c b/lib/url.c
index 9f1013554..f0f266797 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -469,6 +469,11 @@ CURLcode Curl_close(struct Curl_easy **datap)
Curl_safefree(data->state.aptr.proxyuser);
Curl_safefree(data->state.aptr.proxypasswd);
+ /* curl-impersonate: Free the list set by CURLOPT_HTTPBASEHEADER. */
+ curl_slist_free_all(data->state.base_headers);
+ /* curl-impersonate: Free the dynamic list of headers. */
+ curl_slist_free_all(data->state.merged_headers);
+
#ifndef CURL_DISABLE_DOH
if(data->req.doh) {
Curl_dyn_free(&data->req.doh->probe[0].serverdoh);
diff --git a/lib/urldata.h b/lib/urldata.h
index cc9c88870..a35a20e10 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1421,6 +1421,19 @@ struct UrlState {
CURLcode hresult; /* used to pass return codes back from hyper callbacks */
#endif
+ /*
+ * curl-impersonate:
+ * List of "base" headers set by CURLOPT_HTTPBASEHEADER.
+ */
+ struct curl_slist *base_headers;
+ /*
+ * curl-impersonate:
+ * Dynamically-constructed list of HTTP headers.
+ * This list is a merge of the default HTTP headers needed to impersonate a
+ * browser, together with any user-supplied headers.
+ */
+ struct curl_slist *merged_headers;
+
/* Dynamically allocated strings, MUST be freed before this struct is
killed. */
struct dynamically_allocated_data {
diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c
index f836c63b0..5c562549f 100644
--- a/lib/vtls/openssl.c
+++ b/lib/vtls/openssl.c
@@ -76,6 +76,8 @@
#include <openssl/buffer.h>
#include <openssl/pkcs12.h>
+#include <brotli/decode.h>
+
#ifdef USE_AMISSL
#include "amigaos.h"
#endif
@@ -2629,6 +2631,31 @@ static CURLcode load_cacert_from_memory(SSL_CTX *ctx,
return (count > 0 ? CURLE_OK : CURLE_SSL_CACERT_BADFILE);
}
+/* Taken from Chromium and adapted to C,
+ * see net/ssl/cert_compression.cc
+ */
+int DecompressBrotliCert(SSL* ssl,
+ CRYPTO_BUFFER** out,
+ size_t uncompressed_len,
+ const uint8_t* in,
+ size_t in_len) {
+ uint8_t* data;
+ CRYPTO_BUFFER* decompressed = CRYPTO_BUFFER_alloc(&data, uncompressed_len);
+ if (!decompressed) {
+ return 0;
+ }
+
+ size_t output_size = uncompressed_len;
+ if (BrotliDecoderDecompress(in_len, in, &output_size, data) !=
+ BROTLI_DECODER_RESULT_SUCCESS ||
+ output_size != uncompressed_len) {
+ return 0;
+ }
+
+ *out = decompressed;
+ return 1;
+}
+
static CURLcode ossl_connect_step1(struct Curl_easy *data,
struct connectdata *conn, int sockindex)
{
@@ -2767,7 +2794,10 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
ctx_options = SSL_OP_ALL;
#ifdef SSL_OP_NO_TICKET
- ctx_options |= SSL_OP_NO_TICKET;
+ /* curl-impersonate patch.
+ * Turn off SSL_OP_NO_TICKET, we want TLS extension 35 (session_ticket)
+ * to be sent. */
+ ctx_options &= ~SSL_OP_NO_TICKET;
#endif
#ifdef SSL_OP_NO_COMPRESSION
@@ -2821,8 +2851,11 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
SSL_CTX_set_options(backend->ctx, ctx_options);
#ifdef HAS_NPN
+ /* curl-impersonate: Do not enable the NPN extension. */
+ /*
if(conn->bits.tls_enable_npn)
SSL_CTX_set_next_proto_select_cb(backend->ctx, select_next_proto_cb, data);
+ */
#endif
#ifdef HAS_ALPN
@@ -2937,6 +2970,19 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
}
#endif
+ /* curl-impersonate:
+ * Configure BoringSSL to behave like Chrome.
+ * See Constructor of SSLContext at net/socket/ssl_client_socket_impl.cc
+ * and SSLClientSocketImpl::Init()
+ * in the Chromium's source code. */
+
+ /* Enable TLS GREASE. */
+ SSL_CTX_set_grease_enabled(backend->ctx, 1);
+
+ /* Add support for TLS extension 27 - compress_certificate.
+ * Add Brotli decompression. See Chromium net/ssl/cert_compression.cc */
+ SSL_CTX_add_cert_compression_alg(backend->ctx,
+ TLSEXT_cert_compression_brotli, NULL, DecompressBrotliCert);
#if defined(USE_WIN32_CRYPTO)
/* Import certificates from the Windows root certificate store if requested.
@@ -3236,6 +3282,41 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
SSL_set_connect_state(backend->handle);
+#ifdef USE_HTTP2
+ /* curl-impersonate: This adds the ALPS extension (17513).
+ * Chromium calls this function as well in SSLClientSocketImpl::Init().
+ * The 4th parameter is called "settings", and I don't know what it
+ * should contain. For now, use an empty string. */
+ SSL_add_application_settings(backend->handle, "h2", 2, NULL, 0);
+#endif
+
+ SSL_set_options(backend->handle,
+ SSL_OP_LEGACY_SERVER_CONNECT);
+ SSL_set_mode(backend->handle,
+ SSL_MODE_CBC_RECORD_SPLITTING | SSL_MODE_ENABLE_FALSE_START);
+
+ /* curl-impersonate: Enable TLS extensions 5 - status_request and
+ * 18 - signed_certificate_timestamp. */
+ SSL_enable_signed_cert_timestamps(backend->handle);
+ SSL_enable_ocsp_stapling(backend->handle);
+
+ /* curl-impersonate: Some SSL settings copied over from Chrome. */
+ SSL_set_shed_handshake_config(backend->handle, 1);
+
+ /* curl-impersonate: Set the signature algorithms.
+ * (TLS extension 13).
+ * See net/socket/ssl_client_socket_impl.cc in Chromium's source. */
+ static const uint16_t kVerifyPrefs[] = {
+ SSL_SIGN_ECDSA_SECP256R1_SHA256, SSL_SIGN_RSA_PSS_RSAE_SHA256,
+ SSL_SIGN_RSA_PKCS1_SHA256, SSL_SIGN_ECDSA_SECP384R1_SHA384,
+ SSL_SIGN_RSA_PSS_RSAE_SHA384, SSL_SIGN_RSA_PKCS1_SHA384,
+ SSL_SIGN_RSA_PSS_RSAE_SHA512, SSL_SIGN_RSA_PKCS1_SHA512,
+ };
+ if (!SSL_set_verify_algorithm_prefs(backend->handle, kVerifyPrefs,
+ sizeof(kVerifyPrefs) / sizeof(kVerifyPrefs[0]))) {
+ return CURLE_SSL_CIPHER;
+ }
+
backend->server_cert = 0x0;
#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
if((0 == Curl_inet_pton(AF_INET, hostname, &addr)) &&
Reference in New Issue View Git Blame Copy Permalink