diff --git a/Makefile.am b/Makefile.am index 40771ed38..5de6f11b6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -155,13 +155,13 @@ CLEANFILES = $(VC10_LIBVCXPROJ) $(VC10_SRCVCXPROJ) $(VC11_LIBVCXPROJ) \ $(VC14_SRCVCXPROJ) $(VC14_10_LIBVCXPROJ) $(VC14_10_SRCVCXPROJ) \ $(VC14_30_LIBVCXPROJ) $(VC14_30_SRCVCXPROJ) -bin_SCRIPTS = curl-config +bin_SCRIPTS = curl-impersonate-ff-config SUBDIRS = lib src DIST_SUBDIRS = $(SUBDIRS) tests packages scripts include docs pkgconfigdir = $(libdir)/pkgconfig -pkgconfig_DATA = libcurl.pc +pkgconfig_DATA = libcurl-impersonate-ff.pc # List of files required to generate VC IDE .dsp, .vcproj and .vcxproj files include lib/Makefile.inc diff --git a/configure.ac b/configure.ac index de2dee5a4..d2cbe4ee1 100644 --- a/configure.ac +++ b/configure.ac @@ -1331,7 +1331,8 @@ if test X"$OPT_BROTLI" != Xno; then dnl if given with a prefix, we set -L and -I based on that if test -n "$PREFIX_BROTLI"; then - LIB_BROTLI="-lbrotlidec" + # curl-impersonate: Use static libbrotli + LIB_BROTLI="-lbrotlidec-static -lbrotlicommon-static" LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff CPP_BROTLI=-I${PREFIX_BROTLI}/include DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff @@ -1341,7 +1342,11 @@ if test X"$OPT_BROTLI" != Xno; then CPPFLAGS="$CPPFLAGS $CPP_BROTLI" LIBS="$LIB_BROTLI $LIBS" - AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress) + AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress, + # curl-impersonate: Define 'action-if-found' explicitly to prevent + # -lbrotlidec from being added to LIBS (already added before) + AC_DEFINE(HAVE_LIBBROTLI, 1, [Define to 1 if libbrotli exists]) + ) AC_CHECK_HEADERS(brotli/decode.h, curl_brotli_msg="enabled (libbrotlidec)" @@ -4426,8 +4431,8 @@ AC_CONFIG_FILES([Makefile \ tests/unit/Makefile \ packages/Makefile \ packages/vms/Makefile \ - curl-config \ - libcurl.pc + curl-impersonate-ff-config:curl-config.in \ + libcurl-impersonate-ff.pc:libcurl.pc.in ]) AC_OUTPUT diff --git a/curl-config.in b/curl-config.in index aaf2b8a43..47eff151b 100644 --- a/curl-config.in +++ b/curl-config.in @@ -163,9 +163,9 @@ while test $# -gt 0; do CURLLIBDIR="" fi if test "X@ENABLE_SHARED@" = "Xno"; then - echo ${CURLLIBDIR}-lcurl @LIBCURL_LIBS@ + echo ${CURLLIBDIR}-lcurl-impersonate-ff @LIBCURL_LIBS@ else - echo ${CURLLIBDIR}-lcurl + echo ${CURLLIBDIR}-lcurl-impersonate-ff fi ;; --ssl-backends) @@ -174,7 +174,7 @@ while test $# -gt 0; do --static-libs) if test "X@ENABLE_STATIC@" != "Xno" ; then - echo @libdir@/libcurl.@libext@ @LDFLAGS@ @LIBCURL_LIBS@ + echo @libdir@/libcurl-impersonate-ff.@libext@ @LDFLAGS@ @LIBCURL_LIBS@ else echo "curl was built with static libraries disabled" >&2 exit 1 diff --git a/include/curl/curl.h b/include/curl/curl.h index b00648e79..eefa36f2e 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -2143,6 +2143,10 @@ typedef enum { /* set the SSH host key callback custom pointer */ CURLOPT(CURLOPT_SSH_HOSTKEYDATA, CURLOPTTYPE_CBPOINT, 317), + /* curl-impersonate: A list of headers used by the impersonated browser. + * If given, merged with CURLOPT_HTTPHEADER. */ + CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 318), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; diff --git a/include/curl/easy.h b/include/curl/easy.h index 9c7e63ada..a3c54c4af 100644 --- a/include/curl/easy.h +++ b/include/curl/easy.h @@ -43,6 +43,16 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); CURL_EXTERN void curl_easy_cleanup(CURL *curl); +/* + * curl-impersonate: Tell libcurl to impersonate a browser. + * This is a wrapper function that calls curl_easy_setopt() + * multiple times with all the parameters required. That's also why it was + * created as a separate API function and not just as another option to + * curl_easy_setopt(). + */ +CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target, + int default_headers); + /* * NAME curl_easy_getinfo() * diff --git a/lib/Makefile.am b/lib/Makefile.am index 18ce47ea9..ea403a105 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -30,7 +30,7 @@ EXTRA_DIST = Makefile.m32 config-win32.h config-win32ce.h config-plan9.h \ libcurl.plist libcurl.rc config-amigaos.h makefile.amiga config-win32ce.h \ config-os400.h setup-os400.h $(CMAKE_DIST) setup-win32.h .checksrc -lib_LTLIBRARIES = libcurl.la +lib_LTLIBRARIES = libcurl-impersonate-ff.la if BUILD_UNITTESTS noinst_LTLIBRARIES = libcurlu.la @@ -75,43 +75,43 @@ AM_CPPFLAGS += -DBUILDING_LIBCURL AM_LDFLAGS = AM_CFLAGS = -libcurl_la_CPPFLAGS_EXTRA = -libcurl_la_LDFLAGS_EXTRA = -libcurl_la_CFLAGS_EXTRA = +libcurl_impersonate_ff_la_CPPFLAGS_EXTRA = +libcurl_impersonate_ff_la_LDFLAGS_EXTRA = +libcurl_impersonate_ff_la_CFLAGS_EXTRA = if CURL_LT_SHLIB_USE_VERSION_INFO -libcurl_la_LDFLAGS_EXTRA += $(VERSIONINFO) +libcurl_impersonate_ff_la_LDFLAGS_EXTRA += $(VERSIONINFO) endif if CURL_LT_SHLIB_USE_NO_UNDEFINED -libcurl_la_LDFLAGS_EXTRA += -no-undefined +libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -no-undefined endif if CURL_LT_SHLIB_USE_MIMPURE_TEXT -libcurl_la_LDFLAGS_EXTRA += -mimpure-text +libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -mimpure-text endif if CURL_LT_SHLIB_USE_VERSIONED_SYMBOLS -libcurl_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers +libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers else # if symbol-hiding is enabled, hide them! if DOING_CURL_SYMBOL_HIDING -libcurl_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*' +libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*' endif endif if USE_CPPFLAG_CURL_STATICLIB -libcurl_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB +libcurl_impersonate_ff_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB endif if DOING_CURL_SYMBOL_HIDING -libcurl_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS -libcurl_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING) +libcurl_impersonate_ff_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS +libcurl_impersonate_ff_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING) endif -libcurl_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_la_CPPFLAGS_EXTRA) -libcurl_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS) -libcurl_la_CFLAGS = $(AM_CFLAGS) $(libcurl_la_CFLAGS_EXTRA) +libcurl_impersonate_ff_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_impersonate_ff_la_CPPFLAGS_EXTRA) +libcurl_impersonate_ff_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_impersonate_ff_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS) +libcurl_impersonate_ff_la_CFLAGS = $(AM_CFLAGS) $(libcurl_impersonate_ff_la_CFLAGS_EXTRA) libcurlu_la_CPPFLAGS = $(AM_CPPFLAGS) -DCURL_STATICLIB -DUNITTESTS libcurlu_la_LDFLAGS = $(AM_LDFLAGS) -static $(LIBCURL_LIBS) @@ -120,7 +120,7 @@ libcurlu_la_CFLAGS = $(AM_CFLAGS) # Makefile.inc provides the CSOURCES and HHEADERS defines include Makefile.inc -libcurl_la_SOURCES = $(CSOURCES) $(HHEADERS) +libcurl_impersonate_ff_la_SOURCES = $(CSOURCES) $(HHEADERS) libcurlu_la_SOURCES = $(CSOURCES) $(HHEADERS) CHECKSRC = $(CS_$(V)) diff --git a/lib/Makefile.inc b/lib/Makefile.inc index 9bd8e324b..bfd5e90e2 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -165,6 +165,7 @@ LIB_CFILES = \ idn_win32.c \ if2ip.c \ imap.c \ + impersonate.c \ inet_ntop.c \ inet_pton.c \ krb5.c \ diff --git a/lib/easy.c b/lib/easy.c index 704a59df6..349d03933 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -81,6 +81,8 @@ #include "dynbuf.h" #include "altsvc.h" #include "hsts.h" +#include "strcase.h" +#include "impersonate.h" /* The last 3 #include files should be in this order */ #include "curl_printf.h" @@ -332,6 +334,76 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, return rc; } +/* + * curl-impersonate: + * Call curl_easy_setopt() with all the needed options as defined in the + * 'impersonations' array. + * */ +CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target, + int default_headers) +{ + int i; + int ret; + const struct impersonate_opts *opts; + struct curl_slist *headers = NULL; + + for (opts = impersonations; opts->target != NULL; opts++) { + if (Curl_strncasecompare(target, opts->target, strlen(opts->target))) { + break; + } + } + + if(opts->target == NULL) { + DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n", + target)); + return CURLE_BAD_FUNCTION_ARGUMENT; + } + + if(opts->httpversion != CURL_HTTP_VERSION_NONE) { + ret = curl_easy_setopt(data, CURLOPT_HTTP_VERSION, opts->httpversion); + if(ret) + return ret; + } + + if (opts->ssl_version != CURL_SSLVERSION_DEFAULT) { + ret = curl_easy_setopt(data, CURLOPT_SSLVERSION, opts->ssl_version); + if(ret) + return ret; + } + + if(opts->ciphers) { + ret = curl_easy_setopt(data, CURLOPT_SSL_CIPHER_LIST, opts->ciphers); + if (ret) + return ret; + } + + if(default_headers) { + /* Build a linked list out of the static array of headers. */ + for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) { + if(opts->http_headers[i]) { + headers = curl_slist_append(headers, opts->http_headers[i]); + if(!headers) { + return CURLE_OUT_OF_MEMORY; + } + } + } + + if(headers) { + ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers); + curl_slist_free_all(headers); + if(ret) + return ret; + } + } + + /* Always enable all supported compressions. */ + ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, ""); + if(ret) + return ret; + + return CURLE_OK; +} + /* * curl_easy_init() is the external interface to alloc, setup and init an * easy handle that is returned. If anything goes wrong, NULL is returned. @@ -340,6 +412,8 @@ struct Curl_easy *curl_easy_init(void) { CURLcode result; struct Curl_easy *data; + char *env_target; + char *env_headers; /* Make sure we inited the global SSL stuff */ global_init_lock(); @@ -362,6 +436,29 @@ struct Curl_easy *curl_easy_init(void) return NULL; } + /* + * curl-impersonate: Hook into curl_easy_init() to set the required options + * from an environment variable. + * This is a bit hacky but allows seamless integration of libcurl-impersonate + * without code modifications to the app. + */ + env_target = curl_getenv("CURL_IMPERSONATE"); + if(env_target) { + env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS"); + if(env_headers) { + result = curl_easy_impersonate(data, env_target, + !Curl_strcasecompare(env_headers, "no")); + free(env_headers); + } else { + result = curl_easy_impersonate(data, env_target, true); + } + free(env_target); + if(result) { + Curl_close(&data); + return NULL; + } + } + return data; } @@ -936,6 +1033,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) outcurl->state.referer_alloc = TRUE; } + if(data->state.base_headers) { + outcurl->state.base_headers = + Curl_slist_duplicate(data->state.base_headers); + if(!outcurl->state.base_headers) + goto fail; + } + /* Reinitialize an SSL engine for the new handle * note: the engine name has already been copied by dupset */ if(outcurl->set.str[STRING_SSL_ENGINE]) { @@ -1025,6 +1129,9 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) */ void curl_easy_reset(struct Curl_easy *data) { + char *env_target; + char *env_headers; + Curl_free_request_state(data); /* zero out UserDefined data: */ @@ -1049,6 +1156,23 @@ void curl_easy_reset(struct Curl_easy *data) #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH) Curl_http_auth_cleanup_digest(data); #endif + + /* + * curl-impersonate: Hook into curl_easy_reset() to set the required options + * from an environment variable, just like in curl_easy_init(). + */ + env_target = curl_getenv("CURL_IMPERSONATE"); + if(env_target) { + env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS"); + if(env_headers) { + curl_easy_impersonate(data, env_target, + !Curl_strcasecompare(env_headers, "no")); + free(env_headers); + } else { + curl_easy_impersonate(data, env_target, true); + } + free(env_target); + } } /* diff --git a/lib/easyoptions.c b/lib/easyoptions.c index c99f135ff..d70ff8c51 100644 --- a/lib/easyoptions.c +++ b/lib/easyoptions.c @@ -132,6 +132,7 @@ struct curl_easyoption Curl_easyopts[] = { {"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0}, {"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0}, {"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0}, + {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0}, {"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0}, {"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0}, {"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0}, diff --git a/lib/h2h3.c b/lib/h2h3.c index 9453cf55b..ba9433cb9 100644 --- a/lib/h2h3.c +++ b/lib/h2h3.c @@ -43,7 +43,8 @@ /* Index where :authority header field will appear in request header field list. */ -#define AUTHORITY_DST_IDX 3 +/* curl-impersonate: Put the ":authority" header in the second place. */ +#define AUTHORITY_DST_IDX 2 /* USHRT_MAX is 65535 == 0xffff */ #define HEADER_OVERFLOW(x) \ @@ -258,9 +259,6 @@ CURLcode Curl_pseudo_headers(struct Curl_easy *data, nva[i].valuelen = (end - hdbuf); } - nva[i].value = hdbuf; - nva[i].valuelen = (end - hdbuf); - ++i; } diff --git a/lib/http.c b/lib/http.c index 258722a60..9a06e281a 100644 --- a/lib/http.c +++ b/lib/http.c @@ -85,6 +85,7 @@ #include "altsvc.h" #include "hsts.h" #include "c-hyper.h" +#include "slist.h" /* The last 3 #include files should be in this order */ #include "curl_printf.h" @@ -1804,6 +1805,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, int numlists = 1; /* by default */ int i; + /* + * curl-impersonate: Use the merged list of headers if it exists (i.e. when + * the CURLOPT_HTTPBASEHEADER option was set. + */ + struct curl_slist *noproxyheaders = + (data->state.merged_headers ? + data->state.merged_headers : + data->set.headers); + #ifndef CURL_DISABLE_PROXY enum proxy_use proxy; @@ -1815,10 +1825,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, switch(proxy) { case HEADER_SERVER: - h[0] = data->set.headers; + h[0] = noproxyheaders; break; case HEADER_PROXY: - h[0] = data->set.headers; + h[0] = noproxyheaders; if(data->set.sep_headers) { h[1] = data->set.proxyheaders; numlists++; @@ -1828,12 +1838,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, if(data->set.sep_headers) h[0] = data->set.proxyheaders; else - h[0] = data->set.headers; + h[0] = noproxyheaders; break; } #else (void)is_connect; - h[0] = data->set.headers; + h[0] = noproxyheaders; #endif /* loop through one or two lists */ @@ -2069,6 +2079,108 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn, *reqp = httpreq; } +/* + * curl-impersonate: + * Create a new linked list of headers. + * The new list is a merge between the "base" headers and the application given + * headers. The "base" headers contain curl-impersonate's list of headers + * used by default by the impersonated browser. + * + * The application given headers will override the "base" headers if supplied. + */ +CURLcode Curl_http_merge_headers(struct Curl_easy *data) +{ + int i; + int ret; + struct curl_slist *head; + struct curl_slist *dup = NULL; + struct curl_slist *new_list = NULL; + char *uagent; + + if (!data->state.base_headers) + return CURLE_OK; + + /* Duplicate the list for temporary use. */ + if (data->set.headers) { + dup = Curl_slist_duplicate(data->set.headers); + if(!dup) + return CURLE_OUT_OF_MEMORY; + } + + for(head = data->state.base_headers; head; head = head->next) { + char *sep; + size_t prefix_len; + bool found = FALSE; + struct curl_slist *head2; + + sep = strchr(head->data, ':'); + if(!sep) + continue; + + prefix_len = sep - head->data; + + /* Check if this header was added by the application. */ + for(head2 = dup; head2; head2 = head2->next) { + if(head2->data && + strncasecompare(head2->data, head->data, prefix_len) && + Curl_headersep(head2->data[prefix_len]) ) { + new_list = curl_slist_append(new_list, head2->data); + /* Free and set to NULL to mark that it's been added. */ + Curl_safefree(head2->data); + found = TRUE; + break; + } + } + + /* If the user agent was set with CURLOPT_USERAGENT, but not with + * CURLOPT_HTTPHEADER, take it from there instead. */ + if(!found && + strncasecompare(head->data, "User-Agent", prefix_len) && + data->set.str[STRING_USERAGENT] && + *data->set.str[STRING_USERAGENT]) { + uagent = aprintf("User-Agent: %s", data->set.str[STRING_USERAGENT]); + if(!uagent) { + ret = CURLE_OUT_OF_MEMORY; + goto fail; + } + new_list = Curl_slist_append_nodup(new_list, uagent); + found = TRUE; + } + + if (!found) { + new_list = curl_slist_append(new_list, head->data); + } + + if (!new_list) { + ret = CURLE_OUT_OF_MEMORY; + goto fail; + } + } + + /* Now go over any additional application-supplied headers. */ + for(head = dup; head; head = head->next) { + if(head->data) { + new_list = curl_slist_append(new_list, head->data); + if(!new_list) { + ret = CURLE_OUT_OF_MEMORY; + goto fail; + } + } + } + + curl_slist_free_all(dup); + /* Save the new, merged list separately, so it can be freed later. */ + curl_slist_free_all(data->state.merged_headers); + data->state.merged_headers = new_list; + + return CURLE_OK; + +fail: + Curl_safefree(dup); + curl_slist_free_all(new_list); + return ret; +} + CURLcode Curl_http_useragent(struct Curl_easy *data) { /* The User-Agent string might have been allocated in url.c already, because @@ -3088,6 +3200,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done) http = data->req.p.http; DEBUGASSERT(http); + /* curl-impersonate: Add HTTP headers to impersonate real browsers. */ + result = Curl_http_merge_headers(data); + if (result) + return result; + result = Curl_http_host(data, conn); if(result) return result; diff --git a/lib/http2.c b/lib/http2.c index f6364d0e0..b5cb05e7e 100644 --- a/lib/http2.c +++ b/lib/http2.c @@ -61,7 +61,7 @@ #define NGHTTP2_HAS_SET_LOCAL_WINDOW_SIZE 1 #endif -#define HTTP2_HUGE_WINDOW_SIZE (32 * 1024 * 1024) /* 32 MB */ +#define HTTP2_HUGE_WINDOW_SIZE (12 * 1024 * 1024) /* 12 MB */ #ifdef DEBUG_HTTP2 #define H2BUGF(x) x @@ -79,13 +79,20 @@ static int h2_process_pending_input(struct Curl_easy *data, struct http_conn *httpc, CURLcode *err); + +/* + * curl-impersonate: Set the HTTP/2 stream weight to the one used by Firefox + * by default to fetch html resources. + */ +#define FIREFOX_DEFAULT_STREAM_WEIGHT (42) + /* * Curl_http2_init_state() is called when the easy handle is created and * allows for HTTP/2 specific init of state. */ void Curl_http2_init_state(struct UrlState *state) { - state->stream_weight = NGHTTP2_DEFAULT_WEIGHT; + state->stream_weight = FIREFOX_DEFAULT_STREAM_WEIGHT; } /* @@ -94,7 +101,7 @@ void Curl_http2_init_state(struct UrlState *state) */ void Curl_http2_init_userset(struct UserDefined *set) { - set->stream_weight = NGHTTP2_DEFAULT_WEIGHT; + set->stream_weight = FIREFOX_DEFAULT_STREAM_WEIGHT; } static int http2_getsock(struct Curl_easy *data, @@ -1212,14 +1219,18 @@ static void populate_settings(struct Curl_easy *data, { nghttp2_settings_entry *iv = httpc->local_settings; - iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; - iv[0].value = Curl_multi_max_concurrent_streams(data->multi); + /* curl-impersonate: Align HTTP/2 settings to Firefox's */ + iv[0].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE; + iv[0].value = 0x10000; iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; - iv[1].value = HTTP2_HUGE_WINDOW_SIZE; + iv[1].value = 0x20000; - iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; - iv[2].value = data->multi->push_cb != NULL; + iv[2].settings_id = NGHTTP2_SETTINGS_MAX_FRAME_SIZE; + iv[2].value = 0x4000; + + // iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; + // iv[2].value = data->multi->push_cb != NULL; httpc->local_settings_num = 3; } @@ -1586,12 +1597,18 @@ static ssize_t http2_handle_stream_close(struct connectdata *conn, * struct. */ +/* + * curl-impersonate: By default Firefox uses stream 13 as the "parent" of the + * stream that fetches the main html resource of the web page. + */ +#define FIREFOX_DEFAULT_STREAM_DEP (13) + static void h2_pri_spec(struct Curl_easy *data, nghttp2_priority_spec *pri_spec) { struct HTTP *depstream = (data->set.stream_depends_on? data->set.stream_depends_on->req.p.http:NULL); - int32_t depstream_id = depstream? depstream->stream_id:0; + int32_t depstream_id = depstream? depstream->stream_id:FIREFOX_DEFAULT_STREAM_DEP; nghttp2_priority_spec_init(pri_spec, depstream_id, data->set.stream_weight, data->set.stream_depends_e); data->state.stream_weight = data->set.stream_weight; @@ -2068,6 +2085,73 @@ CURLcode Curl_http2_setup(struct Curl_easy *data, return CURLE_OK; } +/* + * curl-impersonate: Start with stream id 15 as Firefox does. + */ +#define FIREFOX_DEFAULT_STREAM_ID (15) + +static CURLcode http2_set_stream_priority(struct Curl_easy *data, + int32_t stream_id, + int32_t dep_stream_id, + int32_t weight) +{ + int rv; + nghttp2_priority_spec pri_spec; + struct connectdata *conn = data->conn; + struct http_conn *httpc = &conn->proto.httpc; + + nghttp2_priority_spec_init(&pri_spec, dep_stream_id, weight, 0); + rv = nghttp2_submit_priority(httpc->h2, NGHTTP2_FLAG_NONE, + stream_id, &pri_spec); + if(rv) { + failf(data, "nghttp2_submit_priority() failed: %s(%d)", + nghttp2_strerror(rv), rv); + return CURLE_HTTP2; + } + + return CURLE_OK; +} + + +/* + * curl-impersonate: Firefox uses an elaborate scheme of http/2 streams to + * split the load for html/js/css/images. It builds a tree of streams with + * different weights (priorities) by default and communicates this to the + * server. Imitate that behavior. + */ +static CURLcode http2_set_stream_priorities(struct Curl_easy *data) +{ + CURLcode result; + struct connectdata *conn = data->conn; + struct http_conn *httpc = &conn->proto.httpc; + + result = http2_set_stream_priority(data, 3, 0, 201); + if(result) + return result; + + result = http2_set_stream_priority(data, 5, 0, 101); + if(result) + return result; + + result = http2_set_stream_priority(data, 7, 0, 0); + if(result) + return result; + + result = http2_set_stream_priority(data, 9, 7, 0); + if(result) + return result; + + result = http2_set_stream_priority(data, 11, 3, 0); + if(result) + return result; + + result = http2_set_stream_priority(data, 13, 0, 241); + if(result) + return result; + + return CURLE_OK; +} + CURLcode Curl_http2_switched(struct Curl_easy *data, const char *mem, size_t nread) { @@ -2076,6 +2160,7 @@ CURLcode Curl_http2_switched(struct Curl_easy *data, struct http_conn *httpc = &conn->proto.httpc; int rv; struct HTTP *stream = data->req.p.http; + nghttp2_priority_spec pri_spec; result = Curl_http2_setup(data, conn); if(result) @@ -2130,6 +2215,13 @@ CURLcode Curl_http2_switched(struct Curl_easy *data, return CURLE_HTTP2; } + result = http2_set_stream_priorities(data); + if(result) + return result; + + /* Best effort to set the request's stream id to 15, like Firefox does. */ + nghttp2_session_set_next_stream_id(httpc->h2, FIREFOX_DEFAULT_STREAM_ID); + /* we are going to copy mem to httpc->inbuf. This is required since mem is part of buffer pointed by stream->mem, and callbacks called by nghttp2_session_mem_recv() will write stream specific diff --git a/lib/impersonate.c b/lib/impersonate.c new file mode 100644 index 000000000..086fb383a --- /dev/null +++ b/lib/impersonate.c @@ -0,0 +1,186 @@ +#include "curl_setup.h" + +#include "impersonate.h" + +const struct impersonate_opts impersonations[] = { + { + .target = "ff91esr", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "aes_128_gcm_sha_256," + "chacha20_poly1305_sha_256," + "aes_256_gcm_sha_384," + "ecdhe_ecdsa_aes_128_gcm_sha_256," + "ecdhe_rsa_aes_128_gcm_sha_256," + "ecdhe_ecdsa_chacha20_poly1305_sha_256," + "ecdhe_rsa_chacha20_poly1305_sha_256," + "ecdhe_ecdsa_aes_256_gcm_sha_384," + "ecdhe_rsa_aes_256_gcm_sha_384," + "ecdhe_ecdsa_aes_256_sha," + "ecdhe_ecdsa_aes_128_sha," + "ecdhe_rsa_aes_128_sha," + "ecdhe_rsa_aes_256_sha," + "rsa_aes_128_gcm_sha_256," + "rsa_aes_256_gcm_sha_384," + "rsa_aes_128_sha," + "rsa_aes_256_sha," + "rsa_3des_ede_cbc_sha", + .http_headers = { + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1", + "TE: Trailers" + } + }, + { + .target = "ff95", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "aes_128_gcm_sha_256," + "chacha20_poly1305_sha_256," + "aes_256_gcm_sha_384," + "ecdhe_ecdsa_aes_128_gcm_sha_256," + "ecdhe_rsa_aes_128_gcm_sha_256," + "ecdhe_ecdsa_chacha20_poly1305_sha_256," + "ecdhe_rsa_chacha20_poly1305_sha_256," + "ecdhe_ecdsa_aes_256_gcm_sha_384," + "ecdhe_rsa_aes_256_gcm_sha_384," + "ecdhe_ecdsa_aes_256_sha," + "ecdhe_ecdsa_aes_128_sha," + "ecdhe_rsa_aes_128_sha," + "ecdhe_rsa_aes_256_sha," + "rsa_aes_128_gcm_sha_256," + "rsa_aes_256_gcm_sha_384," + "rsa_aes_128_sha," + "rsa_aes_256_sha", + .http_headers = { + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1", + "TE: Trailers" + } + }, + { + .target = "ff98", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "aes_128_gcm_sha_256," + "chacha20_poly1305_sha_256," + "aes_256_gcm_sha_384," + "ecdhe_ecdsa_aes_128_gcm_sha_256," + "ecdhe_rsa_aes_128_gcm_sha_256," + "ecdhe_ecdsa_chacha20_poly1305_sha_256," + "ecdhe_rsa_chacha20_poly1305_sha_256," + "ecdhe_ecdsa_aes_256_gcm_sha_384," + "ecdhe_rsa_aes_256_gcm_sha_384," + "ecdhe_ecdsa_aes_256_sha," + "ecdhe_ecdsa_aes_128_sha," + "ecdhe_rsa_aes_128_sha," + "ecdhe_rsa_aes_256_sha," + "rsa_aes_128_gcm_sha_256," + "rsa_aes_256_gcm_sha_384," + "rsa_aes_128_sha," + "rsa_aes_256_sha", + .http_headers = { + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:98.0) Gecko/20100101 Firefox/98.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1", + "TE: Trailers" + } + }, + { + .target = "ff100", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "aes_128_gcm_sha_256," + "chacha20_poly1305_sha_256," + "aes_256_gcm_sha_384," + "ecdhe_ecdsa_aes_128_gcm_sha_256," + "ecdhe_rsa_aes_128_gcm_sha_256," + "ecdhe_ecdsa_chacha20_poly1305_sha_256," + "ecdhe_rsa_chacha20_poly1305_sha_256," + "ecdhe_ecdsa_aes_256_gcm_sha_384," + "ecdhe_rsa_aes_256_gcm_sha_384," + "ecdhe_ecdsa_aes_256_sha," + "ecdhe_ecdsa_aes_128_sha," + "ecdhe_rsa_aes_128_sha," + "ecdhe_rsa_aes_256_sha," + "rsa_aes_128_gcm_sha_256," + "rsa_aes_256_gcm_sha_384," + "rsa_aes_128_sha," + "rsa_aes_256_sha", + .http_headers = { + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1", + "TE: Trailers" + } + }, + { + .target = "ff102", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "aes_128_gcm_sha_256," + "chacha20_poly1305_sha_256," + "aes_256_gcm_sha_384," + "ecdhe_ecdsa_aes_128_gcm_sha_256," + "ecdhe_rsa_aes_128_gcm_sha_256," + "ecdhe_ecdsa_chacha20_poly1305_sha_256," + "ecdhe_rsa_chacha20_poly1305_sha_256," + "ecdhe_ecdsa_aes_256_gcm_sha_384," + "ecdhe_rsa_aes_256_gcm_sha_384," + "ecdhe_ecdsa_aes_256_sha," + "ecdhe_ecdsa_aes_128_sha," + "ecdhe_rsa_aes_128_sha," + "ecdhe_rsa_aes_256_sha," + "rsa_aes_128_gcm_sha_256," + "rsa_aes_256_gcm_sha_384," + "rsa_aes_128_sha," + "rsa_aes_256_sha", + .http_headers = { + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1", + "TE: Trailers" + } + }, + { + /* Last one must be NULL. */ + .target = NULL + } +}; diff --git a/lib/impersonate.h b/lib/impersonate.h new file mode 100644 index 000000000..964b81f2e --- /dev/null +++ b/lib/impersonate.h @@ -0,0 +1,25 @@ +#ifndef HEADER_CURL_IMPERSONATE_H +#define HEADER_CURL_IMPERSONATE_H + +#define IMPERSONATE_MAX_HEADERS 32 + +/* + * curl-impersonate: Options to be set for each supported target browser. + */ +struct impersonate_opts { + const char *target; + int httpversion; + int ssl_version; + const char *ciphers; + const char *http_headers[IMPERSONATE_MAX_HEADERS]; + /* Other TLS options will come here in the future once they are + * configurable through curl_easy_setopt() */ +}; + +/* + * curl-impersonate: Global array of supported browsers and their + * impersonation options. + */ +extern const struct impersonate_opts impersonations[]; + +#endif /* HEADER_CURL_IMPERSONATE_H */ diff --git a/lib/setopt.c b/lib/setopt.c index 6b16e1c7c..a83d69917 100644 --- a/lib/setopt.c +++ b/lib/setopt.c @@ -50,6 +50,7 @@ #include "multiif.h" #include "altsvc.h" #include "hsts.h" +#include "slist.h" /* The last 3 #include files should be in this order */ #include "curl_printf.h" @@ -674,6 +675,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) va_arg(param, char *)); break; + case CURLOPT_HTTPBASEHEADER: + /* + * curl-impersonate: + * Set a list of "base" headers. These will be merged with any headers + * set by CURLOPT_HTTPHEADER. curl-impersonate uses this option in order + * to set a list of default browser headers. + * + * Unlike CURLOPT_HTTPHEADER, + * the list is copied and can be immediately freed by the user. + */ + curl_slist_free_all(data->state.base_headers); + data->state.base_headers = \ + Curl_slist_duplicate(va_arg(param, struct curl_slist *)); + if (!data->state.base_headers) + result = CURLE_OUT_OF_MEMORY; + break; + case CURLOPT_HTTPHEADER: /* * Set a list with HTTP headers to use (or replace internals with) diff --git a/lib/transfer.c b/lib/transfer.c index 1720b24b1..dcae3c143 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -104,7 +104,15 @@ char *Curl_checkheaders(const struct Curl_easy *data, DEBUGASSERT(thislen); DEBUGASSERT(thisheader[thislen-1] != ':'); - for(head = data->set.headers; head; head = head->next) { + /* + * curl-impersonate: + * Check if we have overriden the user-supplied list of headers. + */ + head = data->set.headers; + if (data->state.merged_headers) + head = data->state.merged_headers; + + for(; head; head = head->next) { if(strncasecompare(head->data, thisheader, thislen) && Curl_headersep(head->data[thislen]) ) return head->data; diff --git a/lib/url.c b/lib/url.c index 1114c6c12..89e29be62 100644 --- a/lib/url.c +++ b/lib/url.c @@ -465,6 +465,11 @@ CURLcode Curl_close(struct Curl_easy **datap) Curl_safefree(data->state.aptr.proxyuser); Curl_safefree(data->state.aptr.proxypasswd); + /* curl-impersonate: Free the list set by CURLOPT_HTTPBASEHEADER. */ + curl_slist_free_all(data->state.base_headers); + /* curl-impersonate: Free the dynamic list of headers. */ + curl_slist_free_all(data->state.merged_headers); + #ifndef CURL_DISABLE_DOH if(data->req.doh) { Curl_dyn_free(&data->req.doh->probe[0].serverdoh); diff --git a/lib/urldata.h b/lib/urldata.h index bcb4d460c..148cd1927 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -1453,6 +1453,19 @@ struct UrlState { CURLcode hresult; /* used to pass return codes back from hyper callbacks */ #endif + /* + * curl-impersonate: + * List of "base" headers set by CURLOPT_HTTPBASEHEADER. + */ + struct curl_slist *base_headers; + /* + * curl-impersonate: + * Dynamically-constructed list of HTTP headers. + * This list is a merge of the default HTTP headers needed to impersonate a + * browser, together with any user-supplied headers. + */ + struct curl_slist *merged_headers; + /* Dynamically allocated strings, MUST be freed before this struct is killed. */ struct dynamically_allocated_data { diff --git a/lib/vtls/nss.c b/lib/vtls/nss.c index 9d3a8584c..8ffd68171 100644 --- a/lib/vtls/nss.c +++ b/lib/vtls/nss.c @@ -145,6 +145,7 @@ static const struct cipher_s cipherlist[] = { {"dhe_dss_3des_sha", SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA}, {"dhe_rsa_des_sha", SSL_DHE_RSA_WITH_DES_CBC_SHA}, {"dhe_dss_des_sha", SSL_DHE_DSS_WITH_DES_CBC_SHA}, + {"rsa_3des_ede_cbc_sha", TLS_RSA_WITH_3DES_EDE_CBC_SHA}, /* TLS 1.0: Exportable 56-bit Cipher Suites. */ {"rsa_des_56_sha", TLS_RSA_EXPORT1024_WITH_DES_CBC_SHA}, {"rsa_rc4_56_sha", TLS_RSA_EXPORT1024_WITH_RC4_56_SHA}, @@ -380,6 +381,95 @@ static SECStatus set_ciphers(struct Curl_easy *data, PRFileDesc *model, return SECSuccess; } +/* See nsSSLIOLayerSetOptions@nsNSSIOLayer.cpp, Firefox source code */ +const SSLNamedGroup named_groups[] = { + ssl_grp_ec_curve25519, ssl_grp_ec_secp256r1, ssl_grp_ec_secp384r1, + ssl_grp_ec_secp521r1, ssl_grp_ffdhe_2048, ssl_grp_ffdhe_3072}; + +#define NUM_OF_NAMED_GROUPS sizeof(named_groups)/sizeof(named_groups[0]) + +static SECStatus set_named_groups(PRFileDesc *model) +{ + /* This aligns TLS extension 10 (supported_groups) to what Firefox does. */ + return SSL_NamedGroupConfig(model, named_groups, NUM_OF_NAMED_GROUPS); +} + +static const SSLSignatureScheme signatures[] = { + ssl_sig_ecdsa_secp256r1_sha256, ssl_sig_ecdsa_secp384r1_sha384, + ssl_sig_ecdsa_secp521r1_sha512, ssl_sig_rsa_pss_sha256, + ssl_sig_rsa_pss_sha384, ssl_sig_rsa_pss_sha512, + ssl_sig_rsa_pkcs1_sha256, ssl_sig_rsa_pkcs1_sha384, + ssl_sig_rsa_pkcs1_sha512, ssl_sig_ecdsa_sha1, + ssl_sig_rsa_pkcs1_sha1 +}; + +#define NUM_OF_SIGNATURES sizeof(signatures)/sizeof(signatures[0]) + +static SECStatus set_additional_key_shares(PRFileDesc *model) +{ + /* This aligns TLS extension 51 (key_share) to what Firefox does. */ + return SSL_SendAdditionalKeyShares(model, 1); +} + +static SECStatus set_signatures(PRFileDesc *model) +{ + /* Align TLS extension 13 (signature_algorithms) to what Firefox does. */ + return SSL_SignatureSchemePrefSet(model, signatures, NUM_OF_SIGNATURES); +} + +static SECStatus set_ssl_options(PRFileDesc *model) +{ + SECStatus s; + + /* Enable TLS 1.3 compat mode. Firefox does this, as can be seen at + * nsSSLIOLayerSetOptions()@nsNSSIOLayer.cpp. + * This has the side effect of NSS faking a TLS session ID. + * See ssl3_CreateClientHelloPreamble()@ssl3con.c + */ + s = SSL_OptionSet(model, SSL_ENABLE_TLS13_COMPAT_MODE, PR_TRUE); + if (s != SECSuccess) { + return s; + } + + /* Firefox sets the following options. I don't know what they do. */ + s = SSL_OptionSet(model, SSL_REQUIRE_SAFE_NEGOTIATION, false); + if (s != SECSuccess) { + return s; + } + s = SSL_OptionSet(model, SSL_ENABLE_EXTENDED_MASTER_SECRET, true); + if (s != SECSuccess) { + return s; + } + s = SSL_OptionSet(model, SSL_ENABLE_HELLO_DOWNGRADE_CHECK, true); + if (s != SECSuccess) { + return s; + } + s = SSL_OptionSet(model, SSL_ENABLE_0RTT_DATA, true); + if (s != SECSuccess) { + return s; + } + + /* This adds TLS extension 34 to the Client Hello. */ + s = SSL_OptionSet(model, SSL_ENABLE_DELEGATED_CREDENTIALS, true); + if (s != SECSuccess) { + return s; + } + + /* This adds TLS extension 5 (status_request) to the Client Hello. */ + s = SSL_OptionSet(model, SSL_ENABLE_OCSP_STAPLING, true); + if (s != SECSuccess) { + return s; + } + + /* Remove TLS extension 18 (signed_certificate_timestamp) */ + s = SSL_OptionSet(model, SSL_ENABLE_SIGNED_CERT_TIMESTAMPS, false); + if (s != SECSuccess) { + return s; + } + + return SSL_OptionSet(model, SSL_HANDSHAKE_AS_CLIENT, true); +} + /* * Return true if at least one cipher-suite is enabled. Used to determine * if we need to call NSS_SetDomesticPolicy() to enable the default ciphers. @@ -1347,6 +1437,7 @@ static CURLcode nss_load_module(SECMODModule **pmod, const char *library, if(module) SECMOD_DestroyModule(module); + return CURLE_FAILED_INIT; } @@ -1970,6 +2061,12 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, if(SSL_OptionSet(model, SSL_NO_CACHE, ssl_no_cache) != SECSuccess) goto error; + if(SSL_SET_OPTION(primary.sessionid)) { + if(SSL_OptionSet(model, SSL_ENABLE_SESSION_TICKETS, + PR_TRUE) != SECSuccess) + goto error; + } + /* enable/disable the requested SSL version(s) */ if(nss_init_sslver(&sslver, data, conn) != CURLE_OK) goto error; @@ -2009,6 +2106,14 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, } } + if (set_named_groups(model) != SECSuccess || + set_additional_key_shares(model) != SECSuccess || + set_signatures(model) != SECSuccess || + set_ssl_options(model) != SECSuccess) { + result = CURLE_SSL_CIPHER; + goto error; + } + if(!SSL_CONN_CONFIG(verifypeer) && SSL_CONN_CONFIG(verifyhost)) infof(data, "WARNING: ignoring value of ssl.verifyhost"); @@ -2165,6 +2270,10 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, int cur = 0; unsigned char protocols[128]; + protocols[cur++] = ALPN_HTTP_1_1_LENGTH; + memcpy(&protocols[cur], ALPN_HTTP_1_1, ALPN_HTTP_1_1_LENGTH); + cur += ALPN_HTTP_1_1_LENGTH; + #ifdef USE_HTTP2 if(data->state.httpwant >= CURL_HTTP_VERSION_2 #ifndef CURL_DISABLE_PROXY @@ -2176,9 +2285,6 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, cur += ALPN_H2_LENGTH; } #endif - protocols[cur++] = ALPN_HTTP_1_1_LENGTH; - memcpy(&protocols[cur], ALPN_HTTP_1_1, ALPN_HTTP_1_1_LENGTH); - cur += ALPN_HTTP_1_1_LENGTH; if(SSL_SetNextProtoNego(backend->handle, protocols, cur) != SECSuccess) goto error; diff --git a/libcurl.pc.in b/libcurl.pc.in index 49485f192..7c2ea437c 100644 --- a/libcurl.pc.in +++ b/libcurl.pc.in @@ -36,6 +36,6 @@ Name: libcurl URL: https://curl.se/ Description: Library to transfer files with ftp, http, etc. Version: @CURLVERSION@ -Libs: -L${libdir} -lcurl @LIBCURL_NO_SHARED@ +Libs: -L${libdir} -lcurl-impersonate-ff @LIBCURL_NO_SHARED@ Libs.private: @LIBCURL_LIBS@ Cflags: -I${includedir} @CPPFLAG_CURL_STATICLIB@ diff --git a/m4/curl-nss.m4 b/m4/curl-nss.m4 index cb162755d..13ee571aa 100644 --- a/m4/curl-nss.m4 +++ b/m4/curl-nss.m4 @@ -76,7 +76,123 @@ if test "x$OPT_NSS" != xno; then # Without pkg-config, we'll kludge in some defaults AC_MSG_WARN([Using hard-wired libraries and compilation flags for NSS.]) addld="-L$OPT_NSS/lib" - addlib="-lssl3 -lsmime3 -lnss3 -lplds4 -lplc4 -lnspr4" + + # curl-impersonate: Link NSS statically. + # NSS is poorly documented in this regard and a lot of trial and error + # was made to come up with the correct list of linking flags. The + # libraries have circular dependencies which makes their order extremely + # difficult to find out. + + # Some references: + # https://github.com/mozilla/application-services/blob/b2690fd2e4cc3e8e10b6868ab0de8b79c89d3a93/components/support/rc_crypto/nss/nss_build_common/src/lib.rs#L94 + # and + # https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/freebl/freebl.gyp + + # On Linux we can use special linker flags to force static linking + # (-l:libplc4.a etc.), otherwise the linker will prefer to use + # libplc4.so. On other systems the dynamic libraries would have to be + # removed manually from the NSS directory before building curl. + case $host_os in + linux*) + addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -l:libplc4.a -l:libplds4.a -l:libnspr4.a -lsqlite" + ;; + darwin*) + addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4" + ;; + *) + addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4 -lsqlite" + ;; + esac + + case $host_cpu in + arm) + addlib="$addlib -larmv8_c_lib" + ;; + aarch64) + addlib="$addlib -larmv8_c_lib -lgcm-aes-aarch64_c_lib" + ;; + x86) + addlib="$addlib -lgcm-aes-x86_c_lib" + ;; + x86_64) + addlib="$addlib -lgcm-aes-x86_c_lib -lhw-acc-crypto-avx -lhw-acc-crypto-avx2 -lsha-x86_c_lib" + case $host_os in + linux*) + addlib="$addlib -lintel-gcm-wrap_c_lib -lintel-gcm-s_lib" + ;; + esac + ;; + esac + + # curl-impersonate: + # On Linux these linker flags are necessary to resolve + # the symbol mess and circular dependencies of NSS .a libraries + # to make the AC_CHECK_LIB test below pass. + case $host_os in + linux*) + addlib="-Wl,--start-group $addlib -Wl,--end-group" + ;; + esac + + # External dependencies for nss + case $host_os in + linux*) + addlib="$addlib -pthread -ldl" + ;; + darwin*) + addlib="$addlib -lsqlite3" + ;; + esac + + # Attempt to locate libnssckbi. + # This library file contains the trusted certificates and nss loads it + # at runtime using dlopen. If it's not in a path findable by dlopen + # we have to add that path explicitly using -rpath so it may find it. + # On Ubuntu and Mac M1 it is in a non-standard location. + AC_ARG_WITH(libnssckbi, + [AS_HELP_STRING([--with-libnssckbi=DIRECTORY], + [Path where libnssckbi can be found when using NSS])], + [AS_IF( + [test x"$withval" = xyes], + [nssckbi_path="check"], + [nssckbi_path="$withval"])], + [nssckbi_path="check"]) + + AS_IF( + [test "x$nssckbi_path" = xno], + [], + [test "x$nssckbi_path" != xcheck], + [addld="$addld -Wl,-rpath,$nssckbi_path"], + [ + AC_MSG_CHECKING([if libnssckbi is in a non-standard location]) + case $host_os in + linux*) + search_paths="/usr/lib/$host /usr/lib/$host/nss" + search_paths="$search_paths /usr/lib/$host_cpu-$host_os" + search_paths="$search_paths /usr/lib/$host_cpu-$host_os/nss" + search_ext="so" + ;; + darwin*) + search_paths="/opt/homebrew/lib" + search_ext="dylib" + ;; + esac + + found="no" + for path in $search_paths; do + if test -f "$path/libnssckbi.$search_ext"; then + AC_MSG_RESULT([$path]) + addld="$addld -Wl,-rpath,$path" + found="yes" + break + fi + done + + if test "$found" = "no"; then + AC_MSG_RESULT([no]) + fi + ]) + addcflags="-I$OPT_NSS/include" version="unknown" nssprefix=$OPT_NSS @@ -93,7 +209,7 @@ if test "x$OPT_NSS" != xno; then fi dnl The function SSL_VersionRangeSet() is needed to enable TLS > 1.0 - AC_CHECK_LIB(nss3, SSL_VersionRangeSet, + AC_CHECK_LIB(nss_static, SSL_VersionRangeSet, [ AC_DEFINE(USE_NSS, 1, [if NSS is enabled]) AC_SUBST(USE_NSS, [1]) @@ -103,9 +219,7 @@ if test "x$OPT_NSS" != xno; then test nss != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes ], [ - LDFLAGS="$CLEANLDFLAGS" - LIBS="$CLEANLIBS" - CPPFLAGS="$CLEANCPPFLAGS" + AC_MSG_ERROR([Failed linking NSS statically]) ]) if test "x$USE_NSS" = "xyes"; then diff --git a/src/Makefile.am b/src/Makefile.am index 706f0aac3..0ad94622e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -43,7 +43,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/include \ -I$(top_srcdir)/lib \ -I$(top_srcdir)/src -bin_PROGRAMS = curl +bin_PROGRAMS = curl-impersonate-ff SUBDIRS = ../docs @@ -54,7 +54,7 @@ endif include Makefile.inc # CURL_FILES comes from Makefile.inc -curl_SOURCES = $(CURL_FILES) +curl_impersonate_ff_SOURCES = $(CURL_FILES) # This might hold -Werror CFLAGS += @CURL_CFLAG_EXTRAS@ @@ -63,9 +63,9 @@ CFLAGS += @CURL_CFLAG_EXTRAS@ LIBS = $(BLANK_AT_MAKETIME) if USE_EXPLICIT_LIB_DEPS -curl_LDADD = $(top_builddir)/lib/libcurl.la @LIBCURL_LIBS@ +curl_impersonate_ff_LDADD = $(top_builddir)/lib/libcurl-impersonate-ff.la @LIBCURL_LIBS@ else -curl_LDADD = $(top_builddir)/lib/libcurl.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@ +curl_impersonate_ff_LDADD = $(top_builddir)/lib/libcurl-impersonate-ff.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@ endif # if unit tests are enabled, build a static library to link them with