hoelee/curl-impersonate
1
0
Fork 0
mirror of https://github.com/lwthiker/curl-impersonate.git synced 2025-04-27 07:36:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
main
curl-impersonate/firefox/patches/curl-impersonate.patch

1603 lines
53 KiB
Diff
Raw Permalink Blame History

diff --git a/Makefile.am b/Makefile.am
index f25e4e2f0..9a8026915 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -156,13 +156,13 @@ CLEANFILES = $(VC10_LIBVCXPROJ) $(VC10_SRCVCXPROJ) $(VC11_LIBVCXPROJ) \
$(VC14_SRCVCXPROJ) $(VC14_10_LIBVCXPROJ) $(VC14_10_SRCVCXPROJ) \
$(VC14_30_LIBVCXPROJ) $(VC14_30_SRCVCXPROJ)
-bin_SCRIPTS = curl-config
+bin_SCRIPTS = curl-impersonate-ff-config
SUBDIRS = lib src
DIST_SUBDIRS = $(SUBDIRS) tests packages scripts include docs
pkgconfigdir = $(libdir)/pkgconfig
-pkgconfig_DATA = libcurl.pc
+pkgconfig_DATA = libcurl-impersonate-ff.pc
# List of files required to generate VC IDE .dsp, .vcproj and .vcxproj files
include lib/Makefile.inc
diff --git a/configure.ac b/configure.ac
index 75a882b12..9310059b9 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1493,7 +1493,8 @@ if test X"$OPT_BROTLI" != Xno; then
dnl if given with a prefix, we set -L and -I based on that
if test -n "$PREFIX_BROTLI"; then
- LIB_BROTLI="-lbrotlidec"
+ # curl-impersonate: Use static libbrotli
+ LIB_BROTLI="-lbrotlidec-static -lbrotlicommon-static"
LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff
CPP_BROTLI=-I${PREFIX_BROTLI}/include
DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff
@@ -1503,7 +1504,11 @@ if test X"$OPT_BROTLI" != Xno; then
CPPFLAGS="$CPPFLAGS $CPP_BROTLI"
LIBS="$LIB_BROTLI $LIBS"
- AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress)
+ AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress,
+ # curl-impersonate: Define 'action-if-found' explicitly to prevent
+ # -lbrotlidec from being added to LIBS (already added before)
+ AC_DEFINE(HAVE_LIBBROTLI, 1, [Define to 1 if libbrotli exists])
+ )
AC_CHECK_HEADERS(brotli/decode.h,
curl_brotli_msg="enabled (libbrotlidec)"
@@ -4706,8 +4711,8 @@ AC_CONFIG_FILES([Makefile \
tests/http/clients/Makefile \
packages/Makefile \
packages/vms/Makefile \
- curl-config \
- libcurl.pc
+ curl-impersonate-ff-config:curl-config.in \
+ libcurl-impersonate-ff.pc:libcurl.pc.in
])
AC_OUTPUT
diff --git a/curl-config.in b/curl-config.in
index 54f92d931..531bec977 100644
--- a/curl-config.in
+++ b/curl-config.in
@@ -163,9 +163,9 @@ while test $# -gt 0; do
CURLLIBDIR=""
fi
if test "X@ENABLE_SHARED@" = "Xno"; then
- echo ${CURLLIBDIR}-lcurl @LIBCURL_LIBS@
+ echo ${CURLLIBDIR}-lcurl-impersonate-ff @LIBCURL_LIBS@
else
- echo ${CURLLIBDIR}-lcurl
+ echo ${CURLLIBDIR}-lcurl-impersonate-ff
fi
;;
--ssl-backends)
@@ -174,7 +174,7 @@ while test $# -gt 0; do
--static-libs)
if test "X@ENABLE_STATIC@" != "Xno" ; then
- echo "@libdir@/libcurl.@libext@" @LDFLAGS@ @LIBCURL_LIBS@
+ echo "@libdir@/libcurl-impersonate-ff.@libext@" @LDFLAGS@ @LIBCURL_LIBS@
else
echo "curl was built with static libraries disabled" >&2
exit 1
diff --git a/include/curl/curl.h b/include/curl/curl.h
index 944352421..07d08a0bb 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -2207,6 +2207,10 @@ typedef enum {
/* Can leak things, gonna exit() soon */
CURLOPT(CURLOPT_QUICK_EXIT, CURLOPTTYPE_LONG, 322),
+ /* curl-impersonate: A list of headers used by the impersonated browser.
+ * If given, merged with CURLOPT_HTTPHEADER. */
+ CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 323),
+
CURLOPT_LASTENTRY /* the last unused */
} CURLoption;
diff --git a/include/curl/easy.h b/include/curl/easy.h
index 1285101c5..c620065dc 100644
--- a/include/curl/easy.h
+++ b/include/curl/easy.h
@@ -43,6 +43,16 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...);
CURL_EXTERN CURLcode curl_easy_perform(CURL *curl);
CURL_EXTERN void curl_easy_cleanup(CURL *curl);
+/*
+ * curl-impersonate: Tell libcurl to impersonate a browser.
+ * This is a wrapper function that calls curl_easy_setopt()
+ * multiple times with all the parameters required. That's also why it was
+ * created as a separate API function and not just as another option to
+ * curl_easy_setopt().
+ */
+CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target,
+ int default_headers);
+
/*
* NAME curl_easy_getinfo()
*
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 3c0a70912..e4a77a0e8 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -31,7 +31,7 @@ EXTRA_DIST = Makefile.mk config-win32.h config-win32ce.h config-plan9.h \
config-os400.h setup-os400.h $(CMAKE_DIST) setup-win32.h .checksrc \
Makefile.soname
-lib_LTLIBRARIES = libcurl.la
+lib_LTLIBRARIES = libcurl-impersonate-ff.la
if BUILD_UNITTESTS
noinst_LTLIBRARIES = libcurlu.la
@@ -67,51 +67,51 @@ AM_CFLAGS =
# Makefile.inc provides the CSOURCES and HHEADERS defines
include Makefile.inc
-libcurl_la_SOURCES = $(CSOURCES) $(HHEADERS)
-libcurlu_la_SOURCES = $(CSOURCES) $(HHEADERS)
+libcurl_impersonate_ff_la_SOURCES = $(CSOURCES) $(HHEADERS)
+libcurlu_impersonate_ff_la_SOURCES = $(CSOURCES) $(HHEADERS)
-libcurl_la_CPPFLAGS_EXTRA =
-libcurl_la_LDFLAGS_EXTRA =
-libcurl_la_CFLAGS_EXTRA =
+libcurl_impersonate_ff_la_CPPFLAGS_EXTRA =
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA =
+libcurl_impersonate_ff_la_CFLAGS_EXTRA =
if CURL_LT_SHLIB_USE_VERSION_INFO
-libcurl_la_LDFLAGS_EXTRA += $(VERSIONINFO)
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += $(VERSIONINFO)
endif
if CURL_LT_SHLIB_USE_NO_UNDEFINED
-libcurl_la_LDFLAGS_EXTRA += -no-undefined
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -no-undefined
endif
if CURL_LT_SHLIB_USE_MIMPURE_TEXT
-libcurl_la_LDFLAGS_EXTRA += -mimpure-text
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -mimpure-text
endif
if CURL_LT_SHLIB_USE_VERSIONED_SYMBOLS
-libcurl_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers
else
# if symbol-hiding is enabled, hide them!
if DOING_CURL_SYMBOL_HIDING
-libcurl_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*'
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*'
endif
endif
if USE_CPPFLAG_CURL_STATICLIB
-libcurl_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB
+libcurl_impersonate_ff_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB
else
if HAVE_WINDRES
-libcurl_la_SOURCES += $(LIB_RCFILES)
+libcurl_impersonate_ff_la_SOURCES += $(LIB_RCFILES)
$(LIB_RCFILES): $(top_srcdir)/include/curl/curlver.h
endif
endif
if DOING_CURL_SYMBOL_HIDING
-libcurl_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS
-libcurl_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING)
+libcurl_impersonate_ff_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS
+libcurl_impersonate_ff_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING)
endif
-libcurl_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_la_CPPFLAGS_EXTRA)
-libcurl_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS)
-libcurl_la_CFLAGS = $(AM_CFLAGS) $(libcurl_la_CFLAGS_EXTRA)
+libcurl_impersonate_ff_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_impersonate_ff_la_CPPFLAGS_EXTRA)
+libcurl_impersonate_ff_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_impersonate_ff_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS)
+libcurl_impersonate_ff_la_CFLAGS = $(AM_CFLAGS) $(libcurl_impersonate_ff_la_CFLAGS_EXTRA)
libcurlu_la_CPPFLAGS = $(AM_CPPFLAGS) -DCURL_STATICLIB -DUNITTESTS
libcurlu_la_LDFLAGS = $(AM_LDFLAGS) -static $(LIBCURL_LIBS)
diff --git a/lib/Makefile.inc b/lib/Makefile.inc
index f815170a7..9d9417edc 100644
--- a/lib/Makefile.inc
+++ b/lib/Makefile.inc
@@ -174,6 +174,7 @@ LIB_CFILES = \
idn.c \
if2ip.c \
imap.c \
+ impersonate.c \
inet_ntop.c \
inet_pton.c \
krb5.c \
diff --git a/lib/dynhds.c b/lib/dynhds.c
index b325e0060..4c8a73bab 100644
--- a/lib/dynhds.c
+++ b/lib/dynhds.c
@@ -52,6 +52,8 @@ entry_new(const char *name, size_t namelen,
e->valuelen = valuelen;
if(opts & DYNHDS_OPT_LOWERCASE)
Curl_strntolower(e->name, e->name, e->namelen);
+ if(opts & DYNHDS_OPT_LOWERCASE_VAL)
+ Curl_strntolower(e->value, e->value, e->valuelen);
return e;
}
@@ -134,6 +136,16 @@ void Curl_dynhds_set_opts(struct dynhds *dynhds, int opts)
dynhds->opts = opts;
}
+void Curl_dynhds_set_opt(struct dynhds *dynhds, int opt)
+{
+ dynhds->opts |= opt;
+}
+
+void Curl_dynhds_del_opt(struct dynhds *dynhds, int opt)
+{
+ dynhds->opts &= ~opt;
+}
+
struct dynhds_entry *Curl_dynhds_getn(struct dynhds *dynhds, size_t n)
{
DEBUGASSERT(dynhds);
diff --git a/lib/dynhds.h b/lib/dynhds.h
index 777baa58a..2d542dfd6 100644
--- a/lib/dynhds.h
+++ b/lib/dynhds.h
@@ -53,6 +53,7 @@ struct dynhds {
#define DYNHDS_OPT_NONE (0)
#define DYNHDS_OPT_LOWERCASE (1 << 0)
+#define DYNHDS_OPT_LOWERCASE_VAL (1 << 1)
/**
* Init for use on first time or after a reset.
@@ -82,6 +83,8 @@ size_t Curl_dynhds_count(struct dynhds *dynhds);
* This will not have an effect on already existing headers.
*/
void Curl_dynhds_set_opts(struct dynhds *dynhds, int opts);
+void Curl_dynhds_set_opt(struct dynhds *dynhds, int opt);
+void Curl_dynhds_del_opt(struct dynhds *dynhds, int opt);
/**
* Return the n-th header entry or NULL if it does not exist.
diff --git a/lib/easy.c b/lib/easy.c
index d36cc03d1..c5a21e4e3 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -73,6 +73,8 @@
#include "dynbuf.h"
#include "altsvc.h"
#include "hsts.h"
+#include "strcase.h"
+#include "impersonate.h"
#include "easy_lock.h"
@@ -330,6 +332,76 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name,
return rc;
}
+/*
+ * curl-impersonate:
+ * Call curl_easy_setopt() with all the needed options as defined in the
+ * 'impersonations' array.
+ * */
+CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target,
+ int default_headers)
+{
+ int i;
+ int ret;
+ const struct impersonate_opts *opts;
+ struct curl_slist *headers = NULL;
+
+ for (opts = impersonations; opts->target != NULL; opts++) {
+ if (strncasecompare(target, opts->target, strlen(opts->target))) {
+ break;
+ }
+ }
+
+ if(opts->target == NULL) {
+ DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n",
+ target));
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+ }
+
+ if(opts->httpversion != CURL_HTTP_VERSION_NONE) {
+ ret = curl_easy_setopt(data, CURLOPT_HTTP_VERSION, opts->httpversion);
+ if(ret)
+ return ret;
+ }
+
+ if (opts->ssl_version != CURL_SSLVERSION_DEFAULT) {
+ ret = curl_easy_setopt(data, CURLOPT_SSLVERSION, opts->ssl_version);
+ if(ret)
+ return ret;
+ }
+
+ if(opts->ciphers) {
+ ret = curl_easy_setopt(data, CURLOPT_SSL_CIPHER_LIST, opts->ciphers);
+ if (ret)
+ return ret;
+ }
+
+ if(default_headers) {
+ /* Build a linked list out of the static array of headers. */
+ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) {
+ if(opts->http_headers[i]) {
+ headers = curl_slist_append(headers, opts->http_headers[i]);
+ if(!headers) {
+ return CURLE_OUT_OF_MEMORY;
+ }
+ }
+ }
+
+ if(headers) {
+ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers);
+ curl_slist_free_all(headers);
+ if(ret)
+ return ret;
+ }
+ }
+
+ /* Always enable all supported compressions. */
+ ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, "");
+ if(ret)
+ return ret;
+
+ return CURLE_OK;
+}
+
/*
* curl_easy_init() is the external interface to alloc, setup and init an
* easy handle that is returned. If anything goes wrong, NULL is returned.
@@ -338,6 +410,8 @@ struct Curl_easy *curl_easy_init(void)
{
CURLcode result;
struct Curl_easy *data;
+ char *env_target;
+ char *env_headers;
/* Make sure we inited the global SSL stuff */
global_init_lock();
@@ -360,6 +434,29 @@ struct Curl_easy *curl_easy_init(void)
return NULL;
}
+ /*
+ * curl-impersonate: Hook into curl_easy_init() to set the required options
+ * from an environment variable.
+ * This is a bit hacky but allows seamless integration of libcurl-impersonate
+ * without code modifications to the app.
+ */
+ env_target = curl_getenv("CURL_IMPERSONATE");
+ if(env_target) {
+ env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS");
+ if(env_headers) {
+ result = curl_easy_impersonate(data, env_target,
+ !strcasecompare(env_headers, "no"));
+ free(env_headers);
+ } else {
+ result = curl_easy_impersonate(data, env_target, true);
+ }
+ free(env_target);
+ if(result) {
+ Curl_close(&data);
+ return NULL;
+ }
+ }
+
return data;
}
@@ -930,6 +1027,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
outcurl->state.referer_alloc = TRUE;
}
+ if(data->state.base_headers) {
+ outcurl->state.base_headers =
+ Curl_slist_duplicate(data->state.base_headers);
+ if(!outcurl->state.base_headers)
+ goto fail;
+ }
+
/* Reinitialize an SSL engine for the new handle
* note: the engine name has already been copied by dupset */
if(outcurl->set.str[STRING_SSL_ENGINE]) {
@@ -1019,6 +1123,9 @@ fail:
*/
void curl_easy_reset(struct Curl_easy *data)
{
+ char *env_target;
+ char *env_headers;
+
Curl_free_request_state(data);
/* zero out UserDefined data: */
@@ -1043,6 +1150,23 @@ void curl_easy_reset(struct Curl_easy *data)
#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH)
Curl_http_auth_cleanup_digest(data);
#endif
+
+ /*
+ * curl-impersonate: Hook into curl_easy_reset() to set the required options
+ * from an environment variable, just like in curl_easy_init().
+ */
+ env_target = curl_getenv("CURL_IMPERSONATE");
+ if(env_target) {
+ env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS");
+ if(env_headers) {
+ curl_easy_impersonate(data, env_target,
+ !strcasecompare(env_headers, "no"));
+ free(env_headers);
+ } else {
+ curl_easy_impersonate(data, env_target, true);
+ }
+ free(env_target);
+ }
}
/*
diff --git a/lib/easyoptions.c b/lib/easyoptions.c
index a9c1efd00..e04b42381 100644
--- a/lib/easyoptions.c
+++ b/lib/easyoptions.c
@@ -134,6 +134,7 @@ struct curl_easyoption Curl_easyopts[] = {
{"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0},
{"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0},
{"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0},
+ {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0},
{"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0},
{"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0},
{"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0},
diff --git a/lib/http.c b/lib/http.c
index 219dcc2c0..19ddd1d36 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -89,6 +89,7 @@
#include "hsts.h"
#include "ws.h"
#include "c-hyper.h"
+#include "slist.h"
#include "curl_ctype.h"
/* The last 3 #include files should be in this order */
@@ -1881,6 +1882,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
int numlists = 1; /* by default */
int i;
+ /*
+ * curl-impersonate: Use the merged list of headers if it exists (i.e. when
+ * the CURLOPT_HTTPBASEHEADER option was set.
+ */
+ struct curl_slist *noproxyheaders =
+ (data->state.merged_headers ?
+ data->state.merged_headers :
+ data->set.headers);
+
#ifndef CURL_DISABLE_PROXY
enum proxy_use proxy;
@@ -1892,10 +1902,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
switch(proxy) {
case HEADER_SERVER:
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
break;
case HEADER_PROXY:
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
if(data->set.sep_headers) {
h[1] = data->set.proxyheaders;
numlists++;
@@ -1905,12 +1915,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
if(data->set.sep_headers)
h[0] = data->set.proxyheaders;
else
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
break;
}
#else
(void)is_connect;
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
#endif
/* loop through one or two lists */
@@ -2146,6 +2156,108 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn,
*reqp = httpreq;
}
+/*
+ * curl-impersonate:
+ * Create a new linked list of headers.
+ * The new list is a merge between the "base" headers and the application given
+ * headers. The "base" headers contain curl-impersonate's list of headers
+ * used by default by the impersonated browser.
+ *
+ * The application given headers will override the "base" headers if supplied.
+ */
+CURLcode Curl_http_merge_headers(struct Curl_easy *data)
+{
+ int i;
+ int ret;
+ struct curl_slist *head;
+ struct curl_slist *dup = NULL;
+ struct curl_slist *new_list = NULL;
+ char *uagent;
+
+ if (!data->state.base_headers)
+ return CURLE_OK;
+
+ /* Duplicate the list for temporary use. */
+ if (data->set.headers) {
+ dup = Curl_slist_duplicate(data->set.headers);
+ if(!dup)
+ return CURLE_OUT_OF_MEMORY;
+ }
+
+ for(head = data->state.base_headers; head; head = head->next) {
+ char *sep;
+ size_t prefix_len;
+ bool found = FALSE;
+ struct curl_slist *head2;
+
+ sep = strchr(head->data, ':');
+ if(!sep)
+ continue;
+
+ prefix_len = sep - head->data;
+
+ /* Check if this header was added by the application. */
+ for(head2 = dup; head2; head2 = head2->next) {
+ if(head2->data &&
+ strncasecompare(head2->data, head->data, prefix_len) &&
+ Curl_headersep(head2->data[prefix_len]) ) {
+ new_list = curl_slist_append(new_list, head2->data);
+ /* Free and set to NULL to mark that it's been added. */
+ Curl_safefree(head2->data);
+ found = TRUE;
+ break;
+ }
+ }
+
+ /* If the user agent was set with CURLOPT_USERAGENT, but not with
+ * CURLOPT_HTTPHEADER, take it from there instead. */
+ if(!found &&
+ strncasecompare(head->data, "User-Agent", prefix_len) &&
+ data->set.str[STRING_USERAGENT] &&
+ *data->set.str[STRING_USERAGENT]) {
+ uagent = aprintf("User-Agent: %s", data->set.str[STRING_USERAGENT]);
+ if(!uagent) {
+ ret = CURLE_OUT_OF_MEMORY;
+ goto fail;
+ }
+ new_list = Curl_slist_append_nodup(new_list, uagent);
+ found = TRUE;
+ }
+
+ if (!found) {
+ new_list = curl_slist_append(new_list, head->data);
+ }
+
+ if (!new_list) {
+ ret = CURLE_OUT_OF_MEMORY;
+ goto fail;
+ }
+ }
+
+ /* Now go over any additional application-supplied headers. */
+ for(head = dup; head; head = head->next) {
+ if(head->data) {
+ new_list = curl_slist_append(new_list, head->data);
+ if(!new_list) {
+ ret = CURLE_OUT_OF_MEMORY;
+ goto fail;
+ }
+ }
+ }
+
+ curl_slist_free_all(dup);
+ /* Save the new, merged list separately, so it can be freed later. */
+ curl_slist_free_all(data->state.merged_headers);
+ data->state.merged_headers = new_list;
+
+ return CURLE_OK;
+
+fail:
+ Curl_safefree(dup);
+ curl_slist_free_all(new_list);
+ return ret;
+}
+
CURLcode Curl_http_useragent(struct Curl_easy *data)
{
/* The User-Agent string might have been allocated in url.c already, because
@@ -3165,6 +3277,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
http = data->req.p.http;
DEBUGASSERT(http);
+ /* curl-impersonate: Add HTTP headers to impersonate real browsers. */
+ result = Curl_http_merge_headers(data);
+ if (result)
+ return result;
+
result = Curl_http_host(data, conn);
if(result)
return result;
@@ -4818,23 +4935,32 @@ CURLcode Curl_http_req_to_h2(struct dynhds *h2_headers,
Curl_dynhds_set_opts(h2_headers, DYNHDS_OPT_LOWERCASE);
result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_METHOD),
req->method, strlen(req->method));
- if(!result && scheme) {
- result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_SCHEME),
- scheme, strlen(scheme));
+ if(!result && req->path) {
+ result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_PATH),
+ req->path, strlen(req->path));
}
if(!result && authority) {
result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_AUTHORITY),
authority, strlen(authority));
}
- if(!result && req->path) {
- result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_PATH),
- req->path, strlen(req->path));
+ if(!result && scheme) {
+ result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_SCHEME),
+ scheme, strlen(scheme));
}
for(i = 0; !result && i < Curl_dynhds_count(&req->headers); ++i) {
e = Curl_dynhds_getn(&req->headers, i);
if(!h2_non_field(e->name, e->namelen)) {
+ /* curl-impersonate:
+ * Some HTTP/2 servers reject 'te' header value that is not lowercase (e.g. 'Trailers).
+ * Convert to lowercase explicitly.
+ */
+ if(e->namelen == 2 && strcasecompare(e->name, "te"))
+ Curl_dynhds_set_opt(h2_headers, DYNHDS_OPT_LOWERCASE_VAL);
+
result = Curl_dynhds_add(h2_headers, e->name, e->namelen,
e->value, e->valuelen);
+
+ Curl_dynhds_del_opt(h2_headers, DYNHDS_OPT_LOWERCASE_VAL);
}
}
diff --git a/lib/http2.c b/lib/http2.c
index c666192fc..e926eb3f5 100644
--- a/lib/http2.c
+++ b/lib/http2.c
@@ -63,12 +63,13 @@
#define NGHTTP2_HAS_SET_LOCAL_WINDOW_SIZE 1
#endif
-
/* buffer dimensioning:
* use 16K as chunk size, as that fits H2 DATA frames well */
#define H2_CHUNK_SIZE (16 * 1024)
/* this is how much we want "in flight" for a stream */
-#define H2_STREAM_WINDOW_SIZE (10 * 1024 * 1024)
+/* curl-impersonate: Set H2_STREAM_WINDOW_SIZE to 128kb so that the initial
+ * window size matches what Firefox sends. */
+#define H2_STREAM_WINDOW_SIZE (128 * 1024)
/* on receving from TLS, we prep for holding a full stream window */
#define H2_NW_RECV_CHUNKS (H2_STREAM_WINDOW_SIZE / H2_CHUNK_SIZE)
/* on send into TLS, we just want to accumulate small frames */
@@ -86,7 +87,8 @@
* will block their received QUOTA in the connection window. And if we
* run out of space, the server is blocked from sending us any data.
* See #10988 for an issue with this. */
-#define HTTP2_HUGE_WINDOW_SIZE (100 * H2_STREAM_WINDOW_SIZE)
+/* curl-impersonate: set HTTP2_HUGE_WINDOW_SIZE to 12MB to match Firefox. */
+#define HTTP2_HUGE_WINDOW_SIZE (96 * H2_STREAM_WINDOW_SIZE)
#define H2_SETTINGS_IV_LEN 3
#define H2_BINSETTINGS_LEN 80
@@ -94,14 +96,18 @@
static int populate_settings(nghttp2_settings_entry *iv,
struct Curl_easy *data)
{
- iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
- iv[0].value = Curl_multi_max_concurrent_streams(data->multi);
+ /* curl-impersonate: Align HTTP/2 settings to Firefox's */
+ iv[0].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE;
+ iv[0].value = 0x10000;
iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
iv[1].value = H2_STREAM_WINDOW_SIZE;
- iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
- iv[2].value = data->multi->push_cb != NULL;
+ iv[2].settings_id = NGHTTP2_SETTINGS_MAX_FRAME_SIZE;
+ iv[2].value = 0x4000;
+
+ // iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
+ // iv[2].value = data->multi->push_cb != NULL;
return 3;
}
@@ -392,6 +398,71 @@ static void multi_connchanged(struct Curl_multi *multi)
multi->recheckstate = TRUE;
}
+/*
+ * curl-impersonate: Start with stream id 15 as Firefox does.
+ */
+#define FIREFOX_DEFAULT_STREAM_ID (15)
+
+static CURLcode http2_set_stream_priority(struct Curl_cfilter *cf,
+ struct Curl_easy *data,
+ int32_t stream_id,
+ int32_t dep_stream_id,
+ int32_t weight)
+{
+ int rv;
+ struct cf_h2_ctx *ctx = cf->ctx;
+ nghttp2_priority_spec pri_spec;
+
+ nghttp2_priority_spec_init(&pri_spec, dep_stream_id, weight, 0);
+ rv = nghttp2_submit_priority(ctx->h2, NGHTTP2_FLAG_NONE,
+ stream_id, &pri_spec);
+ if(rv) {
+ failf(data, "nghttp2_submit_priority() failed: %s(%d)",
+ nghttp2_strerror(rv), rv);
+ return CURLE_HTTP2;
+ }
+
+ return CURLE_OK;
+}
+
+/*
+ * curl-impersonate: Firefox uses an elaborate scheme of http/2 streams to
+ * split the load for html/js/css/images. It builds a tree of streams with
+ * different weights (priorities) by default and communicates this to the
+ * server. Imitate that behavior.
+ */
+static CURLcode http2_set_stream_priorities(struct Curl_cfilter *cf,
+ struct Curl_easy *data)
+{
+ CURLcode result;
+
+ result = http2_set_stream_priority(cf, data, 3, 0, 201);
+ if(result)
+ return result;
+
+ result = http2_set_stream_priority(cf, data, 5, 0, 101);
+ if(result)
+ return result;
+
+ result = http2_set_stream_priority(cf, data, 7, 0, 0);
+ if(result)
+ return result;
+
+ result = http2_set_stream_priority(cf, data, 9, 7, 0);
+ if(result)
+ return result;
+
+ result = http2_set_stream_priority(cf, data, 11, 3, 0);
+ if(result)
+ return result;
+
+ result = http2_set_stream_priority(cf, data, 13, 0, 241);
+ if(result)
+ return result;
+
+ return CURLE_OK;
+}
+
/*
* Initialize the cfilter context
*/
@@ -492,6 +563,13 @@ static CURLcode cf_h2_ctx_init(struct Curl_cfilter *cf,
goto out;
}
+ result = http2_set_stream_priorities(cf, data);
+ if(result)
+ goto out;
+
+ /* Best effort to set the request's stream id to 15, like Firefox does. */
+ nghttp2_session_set_next_stream_id(ctx->h2, FIREFOX_DEFAULT_STREAM_ID);
+
/* all set, traffic will be send on connect */
result = CURLE_OK;
@@ -1616,18 +1694,25 @@ out:
return rv;
}
+
+/*
+ * curl-impersonate: Set the HTTP/2 stream weight to the one used by Firefox
+ * by default to fetch html resources.
+ */
+#define FIREFOX_DEFAULT_STREAM_WEIGHT (42)
+
static int sweight_wanted(const struct Curl_easy *data)
{
/* 0 weight is not set by user and we take the nghttp2 default one */
return data->set.priority.weight?
- data->set.priority.weight : NGHTTP2_DEFAULT_WEIGHT;
+ data->set.priority.weight : FIREFOX_DEFAULT_STREAM_WEIGHT;
}
static int sweight_in_effect(const struct Curl_easy *data)
{
/* 0 weight is not set by user and we take the nghttp2 default one */
return data->state.priority.weight?
- data->state.priority.weight : NGHTTP2_DEFAULT_WEIGHT;
+ data->state.priority.weight : FIREFOX_DEFAULT_STREAM_WEIGHT;
}
/*
@@ -1636,12 +1721,18 @@ static int sweight_in_effect(const struct Curl_easy *data)
* struct.
*/
+/*
+ * curl-impersonate: By default Firefox uses stream 13 as the "parent" of the
+ * stream that fetches the main html resource of the web page.
+ */
+#define FIREFOX_DEFAULT_STREAM_DEP (13)
+
static void h2_pri_spec(struct Curl_easy *data,
nghttp2_priority_spec *pri_spec)
{
struct Curl_data_priority *prio = &data->set.priority;
struct stream_ctx *depstream = H2_STREAM_CTX(prio->parent);
- int32_t depstream_id = depstream? depstream->id:0;
+ int32_t depstream_id = depstream? depstream->id:FIREFOX_DEFAULT_STREAM_DEP;
nghttp2_priority_spec_init(pri_spec, depstream_id,
sweight_wanted(data),
data->set.priority.exclusive);
diff --git a/lib/impersonate.c b/lib/impersonate.c
new file mode 100644
index 000000000..e8ae6a722
--- /dev/null
+++ b/lib/impersonate.c
@@ -0,0 +1,257 @@
+#include "curl_setup.h"
+
+#include <curl/curl.h>
+#include "impersonate.h"
+
+const struct impersonate_opts impersonations[] = {
+ {
+ .target = "ff91esr",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "aes_128_gcm_sha_256,"
+ "chacha20_poly1305_sha_256,"
+ "aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_256_sha,"
+ "ecdhe_ecdsa_aes_128_sha,"
+ "ecdhe_rsa_aes_128_sha,"
+ "ecdhe_rsa_aes_256_sha,"
+ "rsa_aes_128_gcm_sha_256,"
+ "rsa_aes_256_gcm_sha_384,"
+ "rsa_aes_128_sha,"
+ "rsa_aes_256_sha,"
+ "rsa_3des_ede_cbc_sha",
+ .http_headers = {
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+ "Accept-Language: en-US,en;q=0.5",
+ "Accept-Encoding: gzip, deflate, br",
+ "Upgrade-Insecure-Requests: 1",
+ "Sec-Fetch-Dest: document",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-User: ?1",
+ "TE: Trailers"
+ }
+ },
+ {
+ .target = "ff95",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "aes_128_gcm_sha_256,"
+ "chacha20_poly1305_sha_256,"
+ "aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_256_sha,"
+ "ecdhe_ecdsa_aes_128_sha,"
+ "ecdhe_rsa_aes_128_sha,"
+ "ecdhe_rsa_aes_256_sha,"
+ "rsa_aes_128_gcm_sha_256,"
+ "rsa_aes_256_gcm_sha_384,"
+ "rsa_aes_128_sha,"
+ "rsa_aes_256_sha",
+ .http_headers = {
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+ "Accept-Language: en-US,en;q=0.5",
+ "Accept-Encoding: gzip, deflate, br",
+ "Upgrade-Insecure-Requests: 1",
+ "Sec-Fetch-Dest: document",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-User: ?1",
+ "TE: Trailers"
+ }
+ },
+ {
+ .target = "ff98",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "aes_128_gcm_sha_256,"
+ "chacha20_poly1305_sha_256,"
+ "aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_256_sha,"
+ "ecdhe_ecdsa_aes_128_sha,"
+ "ecdhe_rsa_aes_128_sha,"
+ "ecdhe_rsa_aes_256_sha,"
+ "rsa_aes_128_gcm_sha_256,"
+ "rsa_aes_256_gcm_sha_384,"
+ "rsa_aes_128_sha,"
+ "rsa_aes_256_sha",
+ .http_headers = {
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:98.0) Gecko/20100101 Firefox/98.0",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+ "Accept-Language: en-US,en;q=0.5",
+ "Accept-Encoding: gzip, deflate, br",
+ "Upgrade-Insecure-Requests: 1",
+ "Sec-Fetch-Dest: document",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-User: ?1",
+ "TE: Trailers"
+ }
+ },
+ {
+ .target = "ff100",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "aes_128_gcm_sha_256,"
+ "chacha20_poly1305_sha_256,"
+ "aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_256_sha,"
+ "ecdhe_ecdsa_aes_128_sha,"
+ "ecdhe_rsa_aes_128_sha,"
+ "ecdhe_rsa_aes_256_sha,"
+ "rsa_aes_128_gcm_sha_256,"
+ "rsa_aes_256_gcm_sha_384,"
+ "rsa_aes_128_sha,"
+ "rsa_aes_256_sha",
+ .http_headers = {
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+ "Accept-Language: en-US,en;q=0.5",
+ "Accept-Encoding: gzip, deflate, br",
+ "Upgrade-Insecure-Requests: 1",
+ "Sec-Fetch-Dest: document",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-User: ?1",
+ "TE: Trailers"
+ }
+ },
+ {
+ .target = "ff102",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "aes_128_gcm_sha_256,"
+ "chacha20_poly1305_sha_256,"
+ "aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_256_sha,"
+ "ecdhe_ecdsa_aes_128_sha,"
+ "ecdhe_rsa_aes_128_sha,"
+ "ecdhe_rsa_aes_256_sha,"
+ "rsa_aes_128_gcm_sha_256,"
+ "rsa_aes_256_gcm_sha_384,"
+ "rsa_aes_128_sha,"
+ "rsa_aes_256_sha",
+ .http_headers = {
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+ "Accept-Language: en-US,en;q=0.5",
+ "Accept-Encoding: gzip, deflate, br",
+ "Upgrade-Insecure-Requests: 1",
+ "Sec-Fetch-Dest: document",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-User: ?1",
+ "TE: Trailers"
+ }
+ },
+ {
+ .target = "ff109",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "aes_128_gcm_sha_256,"
+ "chacha20_poly1305_sha_256,"
+ "aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_256_sha,"
+ "ecdhe_ecdsa_aes_128_sha,"
+ "ecdhe_rsa_aes_128_sha,"
+ "ecdhe_rsa_aes_256_sha,"
+ "rsa_aes_128_gcm_sha_256,"
+ "rsa_aes_256_gcm_sha_384,"
+ "rsa_aes_128_sha,"
+ "rsa_aes_256_sha",
+ .http_headers = {
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/109.0",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+ "Accept-Language: en-US,en;q=0.5",
+ "Accept-Encoding: gzip, deflate, br",
+ "Upgrade-Insecure-Requests: 1",
+ "Sec-Fetch-Dest: document",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-User: ?1",
+ "TE: Trailers"
+ }
+ },
+ {
+ .target = "ff117",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "aes_128_gcm_sha_256,"
+ "chacha20_poly1305_sha_256,"
+ "aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
+ "ecdhe_ecdsa_aes_256_sha,"
+ "ecdhe_ecdsa_aes_128_sha,"
+ "ecdhe_rsa_aes_128_sha,"
+ "ecdhe_rsa_aes_256_sha,"
+ "rsa_aes_128_gcm_sha_256,"
+ "rsa_aes_256_gcm_sha_384,"
+ "rsa_aes_128_sha,"
+ "rsa_aes_256_sha",
+ .http_headers = {
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/117.0",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+ "Accept-Language: en-US,en;q=0.5",
+ "Accept-Encoding: gzip, deflate, br",
+ "Upgrade-Insecure-Requests: 1",
+ "Sec-Fetch-Dest: document",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-User: ?1",
+ "TE: Trailers"
+ }
+ },
+ {
+ /* Last one must be NULL. */
+ .target = NULL
+ }
+};
diff --git a/lib/impersonate.h b/lib/impersonate.h
new file mode 100644
index 000000000..964b81f2e
--- /dev/null
+++ b/lib/impersonate.h
@@ -0,0 +1,25 @@
+#ifndef HEADER_CURL_IMPERSONATE_H
+#define HEADER_CURL_IMPERSONATE_H
+
+#define IMPERSONATE_MAX_HEADERS 32
+
+/*
+ * curl-impersonate: Options to be set for each supported target browser.
+ */
+struct impersonate_opts {
+ const char *target;
+ int httpversion;
+ int ssl_version;
+ const char *ciphers;
+ const char *http_headers[IMPERSONATE_MAX_HEADERS];
+ /* Other TLS options will come here in the future once they are
+ * configurable through curl_easy_setopt() */
+};
+
+/*
+ * curl-impersonate: Global array of supported browsers and their
+ * impersonation options.
+ */
+extern const struct impersonate_opts impersonations[];
+
+#endif /* HEADER_CURL_IMPERSONATE_H */
diff --git a/lib/setopt.c b/lib/setopt.c
index 0c3b9634d..bd7c6746d 100644
--- a/lib/setopt.c
+++ b/lib/setopt.c
@@ -50,6 +50,7 @@
#include "multiif.h"
#include "altsvc.h"
#include "hsts.h"
+#include "slist.h"
/* The last 3 #include files should be in this order */
#include "curl_printf.h"
@@ -967,6 +968,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
#if !defined(CURL_DISABLE_HTTP) || !defined(CURL_DISABLE_SMTP) || \
!defined(CURL_DISABLE_IMAP)
# if !defined(CURL_DISABLE_HTTP) || !defined(CURL_DISABLE_MIME)
+ case CURLOPT_HTTPBASEHEADER:
+ /*
+ * curl-impersonate:
+ * Set a list of "base" headers. These will be merged with any headers
+ * set by CURLOPT_HTTPHEADER. curl-impersonate uses this option in order
+ * to set a list of default browser headers.
+ *
+ * Unlike CURLOPT_HTTPHEADER,
+ * the list is copied and can be immediately freed by the user.
+ */
+ curl_slist_free_all(data->state.base_headers);
+ data->state.base_headers = \
+ Curl_slist_duplicate(va_arg(param, struct curl_slist *));
+ if (!data->state.base_headers)
+ result = CURLE_OUT_OF_MEMORY;
+ break;
+
case CURLOPT_HTTPHEADER:
/*
* Set a list with HTTP headers to use (or replace internals with)
diff --git a/lib/transfer.c b/lib/transfer.c
index d2ff0c24c..56e2090b6 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -106,7 +106,15 @@ char *Curl_checkheaders(const struct Curl_easy *data,
DEBUGASSERT(thislen);
DEBUGASSERT(thisheader[thislen-1] != ':');
- for(head = data->set.headers; head; head = head->next) {
+ /*
+ * curl-impersonate:
+ * Check if we have overriden the user-supplied list of headers.
+ */
+ head = data->set.headers;
+ if (data->state.merged_headers)
+ head = data->state.merged_headers;
+
+ for(; head; head = head->next) {
if(strncasecompare(head->data, thisheader, thislen) &&
Curl_headersep(head->data[thislen]) )
return head->data;
diff --git a/lib/url.c b/lib/url.c
index b37d13f8f..ffb74cc4f 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -444,6 +444,11 @@ CURLcode Curl_close(struct Curl_easy **datap)
Curl_safefree(data->state.aptr.proxyuser);
Curl_safefree(data->state.aptr.proxypasswd);
+ /* curl-impersonate: Free the list set by CURLOPT_HTTPBASEHEADER. */
+ curl_slist_free_all(data->state.base_headers);
+ /* curl-impersonate: Free the dynamic list of headers. */
+ curl_slist_free_all(data->state.merged_headers);
+
#ifndef CURL_DISABLE_DOH
if(data->req.doh) {
Curl_dyn_free(&data->req.doh->probe[0].serverdoh);
diff --git a/lib/urldata.h b/lib/urldata.h
index f02e66541..d6162a80e 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1395,6 +1395,19 @@ struct UrlState {
CURLcode hresult; /* used to pass return codes back from hyper callbacks */
#endif
+ /*
+ * curl-impersonate:
+ * List of "base" headers set by CURLOPT_HTTPBASEHEADER.
+ */
+ struct curl_slist *base_headers;
+ /*
+ * curl-impersonate:
+ * Dynamically-constructed list of HTTP headers.
+ * This list is a merge of the default HTTP headers needed to impersonate a
+ * browser, together with any user-supplied headers.
+ */
+ struct curl_slist *merged_headers;
+
/* Dynamically allocated strings, MUST be freed before this struct is
killed. */
struct dynamically_allocated_data {
diff --git a/lib/vtls/nss.c b/lib/vtls/nss.c
index 5e5dbb744..2604fc755 100644
--- a/lib/vtls/nss.c
+++ b/lib/vtls/nss.c
@@ -145,6 +145,7 @@ static const struct cipher_s cipherlist[] = {
{"dhe_dss_3des_sha", SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA},
{"dhe_rsa_des_sha", SSL_DHE_RSA_WITH_DES_CBC_SHA},
{"dhe_dss_des_sha", SSL_DHE_DSS_WITH_DES_CBC_SHA},
+ {"rsa_3des_ede_cbc_sha", TLS_RSA_WITH_3DES_EDE_CBC_SHA},
/* TLS 1.0: Exportable 56-bit Cipher Suites. */
{"rsa_des_56_sha", TLS_RSA_EXPORT1024_WITH_DES_CBC_SHA},
{"rsa_rc4_56_sha", TLS_RSA_EXPORT1024_WITH_RC4_56_SHA},
@@ -380,6 +381,95 @@ static SECStatus set_ciphers(struct Curl_easy *data, PRFileDesc *model,
return SECSuccess;
}
+/* See nsSSLIOLayerSetOptions@nsNSSIOLayer.cpp, Firefox source code */
+const SSLNamedGroup named_groups[] = {
+ ssl_grp_ec_curve25519, ssl_grp_ec_secp256r1, ssl_grp_ec_secp384r1,
+ ssl_grp_ec_secp521r1, ssl_grp_ffdhe_2048, ssl_grp_ffdhe_3072};
+
+#define NUM_OF_NAMED_GROUPS sizeof(named_groups)/sizeof(named_groups[0])
+
+static SECStatus set_named_groups(PRFileDesc *model)
+{
+ /* This aligns TLS extension 10 (supported_groups) to what Firefox does. */
+ return SSL_NamedGroupConfig(model, named_groups, NUM_OF_NAMED_GROUPS);
+}
+
+static const SSLSignatureScheme signatures[] = {
+ ssl_sig_ecdsa_secp256r1_sha256, ssl_sig_ecdsa_secp384r1_sha384,
+ ssl_sig_ecdsa_secp521r1_sha512, ssl_sig_rsa_pss_sha256,
+ ssl_sig_rsa_pss_sha384, ssl_sig_rsa_pss_sha512,
+ ssl_sig_rsa_pkcs1_sha256, ssl_sig_rsa_pkcs1_sha384,
+ ssl_sig_rsa_pkcs1_sha512, ssl_sig_ecdsa_sha1,
+ ssl_sig_rsa_pkcs1_sha1
+};
+
+#define NUM_OF_SIGNATURES sizeof(signatures)/sizeof(signatures[0])
+
+static SECStatus set_additional_key_shares(PRFileDesc *model)
+{
+ /* This aligns TLS extension 51 (key_share) to what Firefox does. */
+ return SSL_SendAdditionalKeyShares(model, 1);
+}
+
+static SECStatus set_signatures(PRFileDesc *model)
+{
+ /* Align TLS extension 13 (signature_algorithms) to what Firefox does. */
+ return SSL_SignatureSchemePrefSet(model, signatures, NUM_OF_SIGNATURES);
+}
+
+static SECStatus set_ssl_options(PRFileDesc *model)
+{
+ SECStatus s;
+
+ /* Enable TLS 1.3 compat mode. Firefox does this, as can be seen at
+ * nsSSLIOLayerSetOptions()@nsNSSIOLayer.cpp.
+ * This has the side effect of NSS faking a TLS session ID.
+ * See ssl3_CreateClientHelloPreamble()@ssl3con.c
+ */
+ s = SSL_OptionSet(model, SSL_ENABLE_TLS13_COMPAT_MODE, PR_TRUE);
+ if (s != SECSuccess) {
+ return s;
+ }
+
+ /* Firefox sets the following options. I don't know what they do. */
+ s = SSL_OptionSet(model, SSL_REQUIRE_SAFE_NEGOTIATION, false);
+ if (s != SECSuccess) {
+ return s;
+ }
+ s = SSL_OptionSet(model, SSL_ENABLE_EXTENDED_MASTER_SECRET, true);
+ if (s != SECSuccess) {
+ return s;
+ }
+ s = SSL_OptionSet(model, SSL_ENABLE_HELLO_DOWNGRADE_CHECK, true);
+ if (s != SECSuccess) {
+ return s;
+ }
+ s = SSL_OptionSet(model, SSL_ENABLE_0RTT_DATA, true);
+ if (s != SECSuccess) {
+ return s;
+ }
+
+ /* This adds TLS extension 34 to the Client Hello. */
+ s = SSL_OptionSet(model, SSL_ENABLE_DELEGATED_CREDENTIALS, true);
+ if (s != SECSuccess) {
+ return s;
+ }
+
+ /* This adds TLS extension 5 (status_request) to the Client Hello. */
+ s = SSL_OptionSet(model, SSL_ENABLE_OCSP_STAPLING, true);
+ if (s != SECSuccess) {
+ return s;
+ }
+
+ /* Remove TLS extension 18 (signed_certificate_timestamp) */
+ s = SSL_OptionSet(model, SSL_ENABLE_SIGNED_CERT_TIMESTAMPS, false);
+ if (s != SECSuccess) {
+ return s;
+ }
+
+ return SSL_OptionSet(model, SSL_HANDSHAKE_AS_CLIENT, true);
+}
+
/*
* Return true if at least one cipher-suite is enabled. Used to determine
* if we need to call NSS_SetDomesticPolicy() to enable the default ciphers.
@@ -1341,6 +1431,7 @@ static CURLcode nss_load_module(SECMODModule **pmod, const char *library,
if(module)
SECMOD_DestroyModule(module);
+
return CURLE_FAILED_INIT;
}
@@ -1923,6 +2014,12 @@ static CURLcode nss_setup_connect(struct Curl_cfilter *cf,
if(SSL_OptionSet(model, SSL_NO_CACHE, ssl_no_cache) != SECSuccess)
goto error;
+ if(ssl_config->primary.sessionid) {
+ if(SSL_OptionSet(model, SSL_ENABLE_SESSION_TICKETS,
+ PR_TRUE) != SECSuccess)
+ goto error;
+ }
+
/* enable/disable the requested SSL version(s) */
if(nss_init_sslver(&sslver, cf, data) != CURLE_OK)
goto error;
@@ -1962,6 +2059,14 @@ static CURLcode nss_setup_connect(struct Curl_cfilter *cf,
}
}
+ if (set_named_groups(model) != SECSuccess ||
+ set_additional_key_shares(model) != SECSuccess ||
+ set_signatures(model) != SECSuccess ||
+ set_ssl_options(model) != SECSuccess) {
+ result = CURLE_SSL_CIPHER;
+ goto error;
+ }
+
if(!conn_config->verifypeer && conn_config->verifyhost)
infof(data, "WARNING: ignoring value of ssl.verifyhost");
diff --git a/lib/vtls/vtls.c b/lib/vtls/vtls.c
index 32334016b..a9b6299b2 100644
--- a/lib/vtls/vtls.c
+++ b/lib/vtls/vtls.c
@@ -139,7 +139,7 @@ static const struct alpn_spec ALPN_SPEC_H11 = {
};
#ifdef USE_HTTP2
static const struct alpn_spec ALPN_SPEC_H2_H11 = {
- { ALPN_H2, ALPN_HTTP_1_1 }, 2
+ { ALPN_HTTP_1_1, ALPN_H2 }, 2
};
#endif
diff --git a/libcurl.pc.in b/libcurl.pc.in
index 9db6b0f89..9e2f19af9 100644
--- a/libcurl.pc.in
+++ b/libcurl.pc.in
@@ -36,6 +36,6 @@ Name: libcurl
URL: https://curl.se/
Description: Library to transfer files with ftp, http, etc.
Version: @CURLVERSION@
-Libs: -L${libdir} -lcurl @LIBCURL_NO_SHARED@
+Libs: -L${libdir} -lcurl-impersonate-ff @LIBCURL_NO_SHARED@
Libs.private: @LIBCURL_LIBS@
Cflags: -I${includedir} @CPPFLAG_CURL_STATICLIB@
diff --git a/m4/curl-nss.m4 b/m4/curl-nss.m4
index 2916c3613..f07fa30ac 100644
--- a/m4/curl-nss.m4
+++ b/m4/curl-nss.m4
@@ -76,7 +76,123 @@ if test "x$OPT_NSS" != xno; then
# Without pkg-config, we'll kludge in some defaults
AC_MSG_WARN([Using hard-wired libraries and compilation flags for NSS.])
addld="-L$OPT_NSS/lib"
- addlib="-lssl3 -lsmime3 -lnss3 -lplds4 -lplc4 -lnspr4"
+
+ # curl-impersonate: Link NSS statically.
+ # NSS is poorly documented in this regard and a lot of trial and error
+ # was made to come up with the correct list of linking flags. The
+ # libraries have circular dependencies which makes their order extremely
+ # difficult to find out.
+
+ # Some references:
+ # https://github.com/mozilla/application-services/blob/b2690fd2e4cc3e8e10b6868ab0de8b79c89d3a93/components/support/rc_crypto/nss/nss_build_common/src/lib.rs#L94
+ # and
+ # https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/freebl/freebl.gyp
+
+ # On Linux we can use special linker flags to force static linking
+ # (-l:libplc4.a etc.), otherwise the linker will prefer to use
+ # libplc4.so. On other systems the dynamic libraries would have to be
+ # removed manually from the NSS directory before building curl.
+ case $host_os in
+ linux*)
+ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -l:libplc4.a -l:libplds4.a -l:libnspr4.a -lsqlite"
+ ;;
+ darwin*)
+ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4"
+ ;;
+ *)
+ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4 -lsqlite"
+ ;;
+ esac
+
+ case $host_cpu in
+ arm)
+ addlib="$addlib -larmv8_c_lib -lgcm-aes-arm32-neon_c_lib"
+ ;;
+ aarch64)
+ addlib="$addlib -larmv8_c_lib -lgcm-aes-aarch64_c_lib"
+ ;;
+ x86)
+ addlib="$addlib -lgcm-aes-x86_c_lib"
+ ;;
+ x86_64)
+ addlib="$addlib -lgcm-aes-x86_c_lib -lhw-acc-crypto-avx -lhw-acc-crypto-avx2 -lsha-x86_c_lib"
+ case $host_os in
+ linux*)
+ addlib="$addlib -lintel-gcm-wrap_c_lib -lintel-gcm-s_lib"
+ ;;
+ esac
+ ;;
+ esac
+
+ # curl-impersonate:
+ # On Linux these linker flags are necessary to resolve
+ # the symbol mess and circular dependencies of NSS .a libraries
+ # to make the AC_CHECK_LIB test below pass.
+ case $host_os in
+ linux*)
+ addlib="-Wl,--start-group $addlib -Wl,--end-group"
+ ;;
+ esac
+
+ # External dependencies for nss
+ case $host_os in
+ linux*)
+ addlib="$addlib -pthread -ldl"
+ ;;
+ darwin*)
+ addlib="$addlib -lsqlite3"
+ ;;
+ esac
+
+ # Attempt to locate libnssckbi.
+ # This library file contains the trusted certificates and nss loads it
+ # at runtime using dlopen. If it's not in a path findable by dlopen
+ # we have to add that path explicitly using -rpath so it may find it.
+ # On Ubuntu and Mac M1 it is in a non-standard location.
+ AC_ARG_WITH(libnssckbi,
+ [AS_HELP_STRING([--with-libnssckbi=DIRECTORY],
+ [Path where libnssckbi can be found when using NSS])],
+ [AS_IF(
+ [test x"$withval" = xyes],
+ [nssckbi_path="check"],
+ [nssckbi_path="$withval"])],
+ [nssckbi_path="check"])
+
+ AS_IF(
+ [test "x$nssckbi_path" = xno],
+ [],
+ [test "x$nssckbi_path" != xcheck],
+ [addld="$addld -Wl,-rpath,$nssckbi_path"],
+ [
+ AC_MSG_CHECKING([if libnssckbi is in a non-standard location])
+ case $host_os in
+ linux*)
+ search_paths="/usr/lib/$host /usr/lib/$host/nss"
+ search_paths="$search_paths /usr/lib/$host_cpu-$host_os"
+ search_paths="$search_paths /usr/lib/$host_cpu-$host_os/nss"
+ search_ext="so"
+ ;;
+ darwin*)
+ search_paths="/opt/homebrew/lib"
+ search_ext="dylib"
+ ;;
+ esac
+
+ found="no"
+ for path in $search_paths; do
+ if test -f "$path/libnssckbi.$search_ext"; then
+ AC_MSG_RESULT([$path])
+ addld="$addld -Wl,-rpath,$path"
+ found="yes"
+ break
+ fi
+ done
+
+ if test "$found" = "no"; then
+ AC_MSG_RESULT([no])
+ fi
+ ])
+
addcflags="-I$OPT_NSS/include"
version="unknown"
nssprefix=$OPT_NSS
@@ -93,7 +209,7 @@ if test "x$OPT_NSS" != xno; then
fi
dnl The function SSL_VersionRangeSet() is needed to enable TLS > 1.0
- AC_CHECK_LIB(nss3, SSL_VersionRangeSet,
+ AC_CHECK_LIB(nss_static, SSL_VersionRangeSet,
[
AC_DEFINE(USE_NSS, 1, [if NSS is enabled])
AC_SUBST(USE_NSS, [1])
@@ -103,9 +219,7 @@ if test "x$OPT_NSS" != xno; then
test nss != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes
],
[
- LDFLAGS="$CLEANLDFLAGS"
- LIBS="$CLEANLIBS"
- CPPFLAGS="$CLEANCPPFLAGS"
+ AC_MSG_ERROR([Failed linking NSS statically])
])
if test "x$USE_NSS" = "xyes"; then
diff --git a/src/Makefile.am b/src/Makefile.am
index f24cb6924..5750eaba8 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -43,7 +43,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/include \
-I$(top_srcdir)/lib \
-I$(top_srcdir)/src
-bin_PROGRAMS = curl
+bin_PROGRAMS = curl-impersonate-ff
SUBDIRS = ../docs
@@ -54,9 +54,9 @@ endif
include Makefile.inc
# CURL_FILES comes from Makefile.inc
-curl_SOURCES = $(CURL_FILES)
+curl_impersonate_ff_SOURCES = $(CURL_FILES)
if HAVE_WINDRES
-curl_SOURCES += $(CURL_RCFILES)
+curl_impersonate_ff_SOURCES += $(CURL_RCFILES)
$(CURL_RCFILES): tool_version.h
endif
@@ -67,9 +67,9 @@ CFLAGS += @CURL_CFLAG_EXTRAS@
LIBS = $(BLANK_AT_MAKETIME)
if USE_EXPLICIT_LIB_DEPS
-curl_LDADD = $(top_builddir)/lib/libcurl.la @LIBCURL_LIBS@
+curl_impersonate_ff_LDADD = $(top_builddir)/lib/libcurl-impersonate-ff.la @LIBCURL_LIBS@
else
-curl_LDADD = $(top_builddir)/lib/libcurl.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@
+curl_impersonate_ff_LDADD = $(top_builddir)/lib/libcurl-impersonate-ff.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@
endif
# if unit tests are enabled, build a static library to link them with
Reference in New Issue View Git Blame Copy Permalink