diff --git a/Makefile.am b/Makefile.am index f25e4e2f0..ff0c5630b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -156,13 +156,13 @@ CLEANFILES = $(VC10_LIBVCXPROJ) $(VC10_SRCVCXPROJ) $(VC11_LIBVCXPROJ) \ $(VC14_SRCVCXPROJ) $(VC14_10_LIBVCXPROJ) $(VC14_10_SRCVCXPROJ) \ $(VC14_30_LIBVCXPROJ) $(VC14_30_SRCVCXPROJ) -bin_SCRIPTS = curl-config +bin_SCRIPTS = curl-impersonate-chrome-config SUBDIRS = lib src DIST_SUBDIRS = $(SUBDIRS) tests packages scripts include docs pkgconfigdir = $(libdir)/pkgconfig -pkgconfig_DATA = libcurl.pc +pkgconfig_DATA = libcurl-impersonate-chrome.pc # List of files required to generate VC IDE .dsp, .vcproj and .vcxproj files include lib/Makefile.inc diff --git a/configure.ac b/configure.ac index 75a882b12..71fd3290a 100644 --- a/configure.ac +++ b/configure.ac @@ -1493,7 +1493,8 @@ if test X"$OPT_BROTLI" != Xno; then dnl if given with a prefix, we set -L and -I based on that if test -n "$PREFIX_BROTLI"; then - LIB_BROTLI="-lbrotlidec" + # curl-impersonate: Use static libbrotli + LIB_BROTLI="-lbrotlidec-static -lbrotlicommon-static" LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff CPP_BROTLI=-I${PREFIX_BROTLI}/include DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff @@ -1503,7 +1504,11 @@ if test X"$OPT_BROTLI" != Xno; then CPPFLAGS="$CPPFLAGS $CPP_BROTLI" LIBS="$LIB_BROTLI $LIBS" - AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress) + AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress, + # curl-impersonate: Define 'action-if-found' explicitly to prevent + # -lbrotlidec from being added to LIBS (already added before) + AC_DEFINE(HAVE_LIBBROTLI, 1, [Define to 1 if libbrotli exists]) + ) AC_CHECK_HEADERS(brotli/decode.h, curl_brotli_msg="enabled (libbrotlidec)" @@ -4706,8 +4711,8 @@ AC_CONFIG_FILES([Makefile \ tests/http/clients/Makefile \ packages/Makefile \ packages/vms/Makefile \ - curl-config \ - libcurl.pc + curl-impersonate-chrome-config:curl-config.in \ + libcurl-impersonate-chrome.pc:libcurl.pc.in ]) AC_OUTPUT diff --git a/curl-config.in b/curl-config.in index 54f92d931..ea5895e9b 100644 --- a/curl-config.in +++ b/curl-config.in @@ -163,9 +163,9 @@ while test $# -gt 0; do CURLLIBDIR="" fi if test "X@ENABLE_SHARED@" = "Xno"; then - echo ${CURLLIBDIR}-lcurl @LIBCURL_LIBS@ + echo ${CURLLIBDIR}-lcurl-impersonate-chrome @LIBCURL_LIBS@ else - echo ${CURLLIBDIR}-lcurl + echo ${CURLLIBDIR}-lcurl-impersonate-chrome fi ;; --ssl-backends) @@ -174,7 +174,7 @@ while test $# -gt 0; do --static-libs) if test "X@ENABLE_STATIC@" != "Xno" ; then - echo "@libdir@/libcurl.@libext@" @LDFLAGS@ @LIBCURL_LIBS@ + echo "@libdir@/libcurl-impersonate-chrome.@libext@" @LDFLAGS@ @LIBCURL_LIBS@ else echo "curl was built with static libraries disabled" >&2 exit 1 diff --git a/include/curl/curl.h b/include/curl/curl.h index 944352421..532fe3acd 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -2207,6 +2207,50 @@ typedef enum { /* Can leak things, gonna exit() soon */ CURLOPT(CURLOPT_QUICK_EXIT, CURLOPTTYPE_LONG, 322), + /* curl-impersonate: A list of headers used by the impersonated browser. + * If given, merged with CURLOPT_HTTPHEADER. */ + CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 323), + + /* curl-impersonate: A list of TLS signature hash algorithms. + * See https://datatracker.ietf.org/doc/html/rfc5246#section-7.4.1.4.1 */ + CURLOPT(CURLOPT_SSL_SIG_HASH_ALGS, CURLOPTTYPE_STRINGPOINT, 324), + + /* curl-impersonate: Whether to enable ALPS in TLS or not. + * See https://datatracker.ietf.org/doc/html/draft-vvv-tls-alps. + * Support for ALPS is minimal and is intended only for the TLS client + * hello to match. */ + CURLOPT(CURLOPT_SSL_ENABLE_ALPS, CURLOPTTYPE_LONG, 325), + + /* curl-impersonate: Comma-separated list of certificate compression + * algorithms to use. These are published in the client hello. + * Supported algorithms are "zlib" and "brotli". + * See https://datatracker.ietf.org/doc/html/rfc8879 */ + CURLOPT(CURLOPT_SSL_CERT_COMPRESSION, CURLOPTTYPE_STRINGPOINT, 326), + + /* Enable/disable TLS session ticket extension (RFC5077) */ + CURLOPT(CURLOPT_SSL_ENABLE_TICKET, CURLOPTTYPE_LONG, 327), + + /* + * curl-impersonate: + * Set the order of the HTTP/2 pseudo headers. The value must contain + * the letters 'm', 'a', 's', 'p' representing the pseudo-headers + * ":method", ":authority", ":scheme", ":path" in the desired order of + * appearance in the HTTP/2 HEADERS frame. + */ + CURLOPT(CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER, CURLOPTTYPE_STRINGPOINT, 328), + + /* + * curl-impersonate: + * Disable HTTP2 server push in the HTTP2 SETTINGS. + */ + CURLOPT(CURLOPT_HTTP2_NO_SERVER_PUSH, CURLOPTTYPE_LONG, 329), + + /* + * curl-impersonate: Whether to enable Boringssl permute extensions + * See https://commondatastorage.googleapis.com/chromium-boringssl-docs/ssl.h.html#SSL_set_permute_extensions. + */ + CURLOPT(CURLOPT_SSL_PERMUTE_EXTENSIONS, CURLOPTTYPE_LONG, 330), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; diff --git a/include/curl/easy.h b/include/curl/easy.h index 1285101c5..c620065dc 100644 --- a/include/curl/easy.h +++ b/include/curl/easy.h @@ -43,6 +43,16 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); CURL_EXTERN void curl_easy_cleanup(CURL *curl); +/* + * curl-impersonate: Tell libcurl to impersonate a browser. + * This is a wrapper function that calls curl_easy_setopt() + * multiple times with all the parameters required. That's also why it was + * created as a separate API function and not just as another option to + * curl_easy_setopt(). + */ +CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target, + int default_headers); + /* * NAME curl_easy_getinfo() * diff --git a/lib/Makefile.am b/lib/Makefile.am index 3c0a70912..61a9eb90b 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -31,7 +31,7 @@ EXTRA_DIST = Makefile.mk config-win32.h config-win32ce.h config-plan9.h \ config-os400.h setup-os400.h $(CMAKE_DIST) setup-win32.h .checksrc \ Makefile.soname -lib_LTLIBRARIES = libcurl.la +lib_LTLIBRARIES = libcurl-impersonate-chrome.la if BUILD_UNITTESTS noinst_LTLIBRARIES = libcurlu.la @@ -67,51 +67,51 @@ AM_CFLAGS = # Makefile.inc provides the CSOURCES and HHEADERS defines include Makefile.inc -libcurl_la_SOURCES = $(CSOURCES) $(HHEADERS) +libcurl_impersonate_chrome_la_SOURCES = $(CSOURCES) $(HHEADERS) libcurlu_la_SOURCES = $(CSOURCES) $(HHEADERS) -libcurl_la_CPPFLAGS_EXTRA = -libcurl_la_LDFLAGS_EXTRA = -libcurl_la_CFLAGS_EXTRA = +libcurl_impersonate_chrome_la_CPPFLAGS_EXTRA = +libcurl_impersonate_chrome_la_LDFLAGS_EXTRA = +libcurl_impersonate_chrome_la_CFLAGS_EXTRA = if CURL_LT_SHLIB_USE_VERSION_INFO -libcurl_la_LDFLAGS_EXTRA += $(VERSIONINFO) +libcurl_impersonate_chrome_la_LDFLAGS_EXTRA += $(VERSIONINFO) endif if CURL_LT_SHLIB_USE_NO_UNDEFINED -libcurl_la_LDFLAGS_EXTRA += -no-undefined +libcurl_impersonate_chrome_la_LDFLAGS_EXTRA += -no-undefined endif if CURL_LT_SHLIB_USE_MIMPURE_TEXT -libcurl_la_LDFLAGS_EXTRA += -mimpure-text +libcurl_impersonate_chrome_la_LDFLAGS_EXTRA += -mimpure-text endif if CURL_LT_SHLIB_USE_VERSIONED_SYMBOLS -libcurl_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers +libcurl_impersonate_chrome_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers else # if symbol-hiding is enabled, hide them! if DOING_CURL_SYMBOL_HIDING -libcurl_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*' +libcurl_impersonate_chrome_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*' endif endif if USE_CPPFLAG_CURL_STATICLIB -libcurl_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB +libcurl_impersonate_chrome_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB else if HAVE_WINDRES -libcurl_la_SOURCES += $(LIB_RCFILES) +libcurl_impersonate_chrome_la_SOURCES += $(LIB_RCFILES) $(LIB_RCFILES): $(top_srcdir)/include/curl/curlver.h endif endif if DOING_CURL_SYMBOL_HIDING -libcurl_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS -libcurl_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING) +libcurl_impersonate_chrome_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS +libcurl_impersonate_chrome_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING) endif -libcurl_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_la_CPPFLAGS_EXTRA) -libcurl_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS) -libcurl_la_CFLAGS = $(AM_CFLAGS) $(libcurl_la_CFLAGS_EXTRA) +libcurl_impersonate_chrome_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_impersonate_chrome_la_CPPFLAGS_EXTRA) +libcurl_impersonate_chrome_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_impersonate_chrome_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS) +libcurl_impersonate_chrome_la_CFLAGS = $(AM_CFLAGS) $(libcurl_impersonate_chrome_la_CFLAGS_EXTRA) libcurlu_la_CPPFLAGS = $(AM_CPPFLAGS) -DCURL_STATICLIB -DUNITTESTS libcurlu_la_LDFLAGS = $(AM_LDFLAGS) -static $(LIBCURL_LIBS) diff --git a/lib/Makefile.inc b/lib/Makefile.inc index f815170a7..9d9417edc 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -174,6 +174,7 @@ LIB_CFILES = \ idn.c \ if2ip.c \ imap.c \ + impersonate.c \ inet_ntop.c \ inet_pton.c \ krb5.c \ diff --git a/lib/dynhds.c b/lib/dynhds.c index b325e0060..4c8a73bab 100644 --- a/lib/dynhds.c +++ b/lib/dynhds.c @@ -52,6 +52,8 @@ entry_new(const char *name, size_t namelen, e->valuelen = valuelen; if(opts & DYNHDS_OPT_LOWERCASE) Curl_strntolower(e->name, e->name, e->namelen); + if(opts & DYNHDS_OPT_LOWERCASE_VAL) + Curl_strntolower(e->value, e->value, e->valuelen); return e; } @@ -134,6 +136,16 @@ void Curl_dynhds_set_opts(struct dynhds *dynhds, int opts) dynhds->opts = opts; } +void Curl_dynhds_set_opt(struct dynhds *dynhds, int opt) +{ + dynhds->opts |= opt; +} + +void Curl_dynhds_del_opt(struct dynhds *dynhds, int opt) +{ + dynhds->opts &= ~opt; +} + struct dynhds_entry *Curl_dynhds_getn(struct dynhds *dynhds, size_t n) { DEBUGASSERT(dynhds); diff --git a/lib/dynhds.h b/lib/dynhds.h index 777baa58a..2d542dfd6 100644 --- a/lib/dynhds.h +++ b/lib/dynhds.h @@ -53,6 +53,7 @@ struct dynhds { #define DYNHDS_OPT_NONE (0) #define DYNHDS_OPT_LOWERCASE (1 << 0) +#define DYNHDS_OPT_LOWERCASE_VAL (1 << 1) /** * Init for use on first time or after a reset. @@ -82,6 +83,8 @@ size_t Curl_dynhds_count(struct dynhds *dynhds); * This will not have an effect on already existing headers. */ void Curl_dynhds_set_opts(struct dynhds *dynhds, int opts); +void Curl_dynhds_set_opt(struct dynhds *dynhds, int opt); +void Curl_dynhds_del_opt(struct dynhds *dynhds, int opt); /** * Return the n-th header entry or NULL if it does not exist. diff --git a/lib/easy.c b/lib/easy.c index d36cc03d1..65e94f1ad 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -73,6 +73,8 @@ #include "dynbuf.h" #include "altsvc.h" #include "hsts.h" +#include "strcase.h" +#include "impersonate.h" #include "easy_lock.h" @@ -330,6 +332,134 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, return rc; } +/* + * curl-impersonate: + * Call curl_easy_setopt() with all the needed options as defined in the + * 'impersonations' array. + * */ +CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target, + int default_headers) +{ + int i; + int ret; + const struct impersonate_opts *opts = NULL; + struct curl_slist *headers = NULL; + + for(opts = impersonations; opts->target != NULL; opts++) { + if (strcasecompare(target, opts->target)) { + break; + } + } + + if(opts->target == NULL) { + DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n", + target)); + return CURLE_BAD_FUNCTION_ARGUMENT; + } + + if(opts->httpversion != CURL_HTTP_VERSION_NONE) { + ret = curl_easy_setopt(data, CURLOPT_HTTP_VERSION, opts->httpversion); + if(ret) + return ret; + } + + if (opts->ssl_version != CURL_SSLVERSION_DEFAULT) { + ret = curl_easy_setopt(data, CURLOPT_SSLVERSION, opts->ssl_version); + if(ret) + return ret; + } + + if(opts->ciphers) { + ret = curl_easy_setopt(data, CURLOPT_SSL_CIPHER_LIST, opts->ciphers); + if (ret) + return ret; + } + + if(opts->curves) { + ret = curl_easy_setopt(data, CURLOPT_SSL_EC_CURVES, opts->curves); + if(ret) + return ret; + } + + if(opts->sig_hash_algs) { + ret = curl_easy_setopt(data, CURLOPT_SSL_SIG_HASH_ALGS, + opts->sig_hash_algs); + if(ret) + return ret; + } + + ret = curl_easy_setopt(data, CURLOPT_SSL_ENABLE_NPN, opts->npn ? 1 : 0); + if(ret) + return ret; + + ret = curl_easy_setopt(data, CURLOPT_SSL_ENABLE_ALPN, opts->alpn ? 1 : 0); + if(ret) + return ret; + + ret = curl_easy_setopt(data, CURLOPT_SSL_ENABLE_ALPS, opts->alps ? 1 : 0); + if(ret) + return ret; + + ret = curl_easy_setopt(data, CURLOPT_SSL_ENABLE_TICKET, + opts->tls_session_ticket ? 1 : 0); + if(ret) + return ret; + + if(opts->tls_permute_extensions) { + ret = curl_easy_setopt(data, CURLOPT_SSL_PERMUTE_EXTENSIONS, 1); + if(ret) + return ret; + } + + if(opts->cert_compression) { + ret = curl_easy_setopt(data, + CURLOPT_SSL_CERT_COMPRESSION, + opts->cert_compression); + if(ret) + return ret; + } + + if(default_headers) { + /* Build a linked list out of the static array of headers. */ + for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) { + if(opts->http_headers[i]) { + headers = curl_slist_append(headers, opts->http_headers[i]); + if(!headers) { + return CURLE_OUT_OF_MEMORY; + } + } + } + + if(headers) { + ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers); + curl_slist_free_all(headers); + if(ret) + return ret; + } + } + + if(opts->http2_pseudo_headers_order) { + ret = curl_easy_setopt(data, + CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER, + opts->http2_pseudo_headers_order); + if(ret) + return ret; + } + + if(opts->http2_no_server_push) { + ret = curl_easy_setopt(data, CURLOPT_HTTP2_NO_SERVER_PUSH, 1L); + if(ret) + return ret; + } + + /* Always enable all supported compressions. */ + ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, ""); + if(ret) + return ret; + + return CURLE_OK; +} + /* * curl_easy_init() is the external interface to alloc, setup and init an * easy handle that is returned. If anything goes wrong, NULL is returned. @@ -338,6 +468,8 @@ struct Curl_easy *curl_easy_init(void) { CURLcode result; struct Curl_easy *data; + char *env_target; + char *env_headers; /* Make sure we inited the global SSL stuff */ global_init_lock(); @@ -360,6 +492,29 @@ struct Curl_easy *curl_easy_init(void) return NULL; } + /* + * curl-impersonate: Hook into curl_easy_init() to set the required options + * from an environment variable. + * This is a bit hacky but allows seamless integration of libcurl-impersonate + * without code modifications to the app. + */ + env_target = curl_getenv("CURL_IMPERSONATE"); + if(env_target) { + env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS"); + if(env_headers) { + result = curl_easy_impersonate(data, env_target, + !strcasecompare(env_headers, "no")); + free(env_headers); + } else { + result = curl_easy_impersonate(data, env_target, true); + } + free(env_target); + if(result) { + Curl_close(&data); + return NULL; + } + } + return data; } @@ -930,6 +1085,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) outcurl->state.referer_alloc = TRUE; } + if(data->state.base_headers) { + outcurl->state.base_headers = + Curl_slist_duplicate(data->state.base_headers); + if(!outcurl->state.base_headers) + goto fail; + } + /* Reinitialize an SSL engine for the new handle * note: the engine name has already been copied by dupset */ if(outcurl->set.str[STRING_SSL_ENGINE]) { @@ -1019,6 +1181,9 @@ fail: */ void curl_easy_reset(struct Curl_easy *data) { + char *env_target; + char *env_headers; + Curl_free_request_state(data); /* zero out UserDefined data: */ @@ -1043,6 +1208,23 @@ void curl_easy_reset(struct Curl_easy *data) #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH) Curl_http_auth_cleanup_digest(data); #endif + + /* + * curl-impersonate: Hook into curl_easy_reset() to set the required options + * from an environment variable, just like in curl_easy_init(). + */ + env_target = curl_getenv("CURL_IMPERSONATE"); + if(env_target) { + env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS"); + if(env_headers) { + curl_easy_impersonate(data, env_target, + !strcasecompare(env_headers, "no")); + free(env_headers); + } else { + curl_easy_impersonate(data, env_target, true); + } + free(env_target); + } } /* diff --git a/lib/easyoptions.c b/lib/easyoptions.c index a9c1efd00..136a43e22 100644 --- a/lib/easyoptions.c +++ b/lib/easyoptions.c @@ -132,8 +132,12 @@ struct curl_easyoption Curl_easyopts[] = { {"HSTS_CTRL", CURLOPT_HSTS_CTRL, CURLOT_LONG, 0}, {"HTTP09_ALLOWED", CURLOPT_HTTP09_ALLOWED, CURLOT_LONG, 0}, {"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0}, + {"HTTP2_PSEUDO_HEADERS_ORDER", CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER, + CURLOT_STRING, 0}, + {"HTTP2_NO_SERVER_PUSH", CURLOPT_HTTP2_NO_SERVER_PUSH, CURLOT_LONG, 0}, {"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0}, {"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0}, + {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0}, {"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0}, {"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0}, {"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0}, @@ -302,18 +306,23 @@ struct curl_easyoption Curl_easyopts[] = { {"SSLKEYTYPE", CURLOPT_SSLKEYTYPE, CURLOT_STRING, 0}, {"SSLKEY_BLOB", CURLOPT_SSLKEY_BLOB, CURLOT_BLOB, 0}, {"SSLVERSION", CURLOPT_SSLVERSION, CURLOT_VALUES, 0}, + {"SSL_CERT_COMPRESSION", CURLOPT_SSL_CERT_COMPRESSION, CURLOT_STRING, 0}, {"SSL_CIPHER_LIST", CURLOPT_SSL_CIPHER_LIST, CURLOT_STRING, 0}, {"SSL_CTX_DATA", CURLOPT_SSL_CTX_DATA, CURLOT_CBPTR, 0}, {"SSL_CTX_FUNCTION", CURLOPT_SSL_CTX_FUNCTION, CURLOT_FUNCTION, 0}, {"SSL_EC_CURVES", CURLOPT_SSL_EC_CURVES, CURLOT_STRING, 0}, {"SSL_ENABLE_ALPN", CURLOPT_SSL_ENABLE_ALPN, CURLOT_LONG, 0}, + {"SSL_ENABLE_ALPS", CURLOPT_SSL_ENABLE_ALPS, CURLOT_LONG, 0}, {"SSL_ENABLE_NPN", CURLOPT_SSL_ENABLE_NPN, CURLOT_LONG, 0}, + {"SSL_ENABLE_TICKET", CURLOPT_SSL_ENABLE_TICKET, CURLOT_LONG, 0}, {"SSL_FALSESTART", CURLOPT_SSL_FALSESTART, CURLOT_LONG, 0}, {"SSL_OPTIONS", CURLOPT_SSL_OPTIONS, CURLOT_VALUES, 0}, {"SSL_SESSIONID_CACHE", CURLOPT_SSL_SESSIONID_CACHE, CURLOT_LONG, 0}, + {"SSL_SIG_HASH_ALGS", CURLOPT_SSL_SIG_HASH_ALGS, CURLOT_STRING, 0}, {"SSL_VERIFYHOST", CURLOPT_SSL_VERIFYHOST, CURLOT_LONG, 0}, {"SSL_VERIFYPEER", CURLOPT_SSL_VERIFYPEER, CURLOT_LONG, 0}, {"SSL_VERIFYSTATUS", CURLOPT_SSL_VERIFYSTATUS, CURLOT_LONG, 0}, + {"SSL_PERMUTE_EXTENSIONS", CURLOPT_SSL_PERMUTE_EXTENSIONS, CURLOT_LONG, 0}, {"STDERR", CURLOPT_STDERR, CURLOT_OBJECT, 0}, {"STREAM_DEPENDS", CURLOPT_STREAM_DEPENDS, CURLOT_OBJECT, 0}, {"STREAM_DEPENDS_E", CURLOPT_STREAM_DEPENDS_E, CURLOT_OBJECT, 0}, @@ -370,6 +379,6 @@ struct curl_easyoption Curl_easyopts[] = { */ int Curl_easyopts_check(void) { - return ((CURLOPT_LASTENTRY%10000) != (322 + 1)); + return ((CURLOPT_LASTENTRY%10000) != (330 + 1)); } #endif diff --git a/lib/http.c b/lib/http.c index 219dcc2c0..7b04c6c36 100644 --- a/lib/http.c +++ b/lib/http.c @@ -90,6 +90,7 @@ #include "ws.h" #include "c-hyper.h" #include "curl_ctype.h" +#include "slist.h" /* The last 3 #include files should be in this order */ #include "curl_printf.h" @@ -1881,6 +1882,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, int numlists = 1; /* by default */ int i; + /* + * curl-impersonate: Use the merged list of headers if it exists (i.e. when + * the CURLOPT_HTTPBASEHEADER option was set. + */ + struct curl_slist *noproxyheaders = + (data->state.merged_headers ? + data->state.merged_headers : + data->set.headers); + #ifndef CURL_DISABLE_PROXY enum proxy_use proxy; @@ -1892,10 +1902,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, switch(proxy) { case HEADER_SERVER: - h[0] = data->set.headers; + h[0] = noproxyheaders; break; case HEADER_PROXY: - h[0] = data->set.headers; + h[0] = noproxyheaders; if(data->set.sep_headers) { h[1] = data->set.proxyheaders; numlists++; @@ -1905,12 +1915,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, if(data->set.sep_headers) h[0] = data->set.proxyheaders; else - h[0] = data->set.headers; + h[0] = noproxyheaders; break; } #else (void)is_connect; - h[0] = data->set.headers; + h[0] = noproxyheaders; #endif /* loop through one or two lists */ @@ -2146,6 +2156,108 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn, *reqp = httpreq; } +/* + * curl-impersonate: + * Create a new linked list of headers. + * The new list is a merge between the "base" headers and the application given + * headers. The "base" headers contain curl-impersonate's list of headers + * used by default by the impersonated browser. + * + * The application given headers will override the "base" headers if supplied. + */ +CURLcode Curl_http_merge_headers(struct Curl_easy *data) +{ + int i; + int ret; + struct curl_slist *head; + struct curl_slist *dup = NULL; + struct curl_slist *new_list = NULL; + char *uagent; + + if (!data->state.base_headers) + return CURLE_OK; + + /* Duplicate the list for temporary use. */ + if (data->set.headers) { + dup = Curl_slist_duplicate(data->set.headers); + if(!dup) + return CURLE_OUT_OF_MEMORY; + } + + for(head = data->state.base_headers; head; head = head->next) { + char *sep; + size_t prefix_len; + bool found = FALSE; + struct curl_slist *head2; + + sep = strchr(head->data, ':'); + if(!sep) + continue; + + prefix_len = sep - head->data; + + /* Check if this header was added by the application. */ + for(head2 = dup; head2; head2 = head2->next) { + if(head2->data && + strncasecompare(head2->data, head->data, prefix_len) && + Curl_headersep(head2->data[prefix_len]) ) { + new_list = curl_slist_append(new_list, head2->data); + /* Free and set to NULL to mark that it's been added. */ + Curl_safefree(head2->data); + found = TRUE; + break; + } + } + + /* If the user agent was set with CURLOPT_USERAGENT, but not with + * CURLOPT_HTTPHEADER, take it from there instead. */ + if(!found && + strncasecompare(head->data, "User-Agent", prefix_len) && + data->set.str[STRING_USERAGENT] && + *data->set.str[STRING_USERAGENT]) { + uagent = aprintf("User-Agent: %s", data->set.str[STRING_USERAGENT]); + if(!uagent) { + ret = CURLE_OUT_OF_MEMORY; + goto fail; + } + new_list = Curl_slist_append_nodup(new_list, uagent); + found = TRUE; + } + + if (!found) { + new_list = curl_slist_append(new_list, head->data); + } + + if (!new_list) { + ret = CURLE_OUT_OF_MEMORY; + goto fail; + } + } + + /* Now go over any additional application-supplied headers. */ + for(head = dup; head; head = head->next) { + if(head->data) { + new_list = curl_slist_append(new_list, head->data); + if(!new_list) { + ret = CURLE_OUT_OF_MEMORY; + goto fail; + } + } + } + + curl_slist_free_all(dup); + /* Save the new, merged list separately, so it can be freed later. */ + curl_slist_free_all(data->state.merged_headers); + data->state.merged_headers = new_list; + + return CURLE_OK; + +fail: + Curl_safefree(dup); + curl_slist_free_all(new_list); + return ret; +} + CURLcode Curl_http_useragent(struct Curl_easy *data) { /* The User-Agent string might have been allocated in url.c already, because @@ -3165,6 +3277,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done) http = data->req.p.http; DEBUGASSERT(http); + /* curl-impersonate: Add HTTP headers to impersonate real browsers. */ + result = Curl_http_merge_headers(data); + if (result) + return result; + result = Curl_http_host(data, conn); if(result) return result; @@ -4777,12 +4894,41 @@ static bool h2_non_field(const char *name, size_t namelen) return FALSE; } +/* + * curl-impersonate: + * Determine the position of HTTP/2 pseudo headers. + * The pseudo headers ":method", ":path", ":scheme", ":authority" + * are sent in different order by different browsers. An important part of the + * impersonation is ordering them like the browser does. + */ +static CURLcode h2_check_pseudo_header_order(const char *order) +{ + if(strlen(order) != 4) + return CURLE_BAD_FUNCTION_ARGUMENT; + + // :method should always be first + if(order[0] != 'm') + return CURLE_BAD_FUNCTION_ARGUMENT; + + // All pseudo-headers must be present + if(!strchr(order, 'm') || + !strchr(order, 'a') || + !strchr(order, 's') || + !strchr(order, 'p')) + return CURLE_BAD_FUNCTION_ARGUMENT; + + return CURLE_OK; +} + CURLcode Curl_http_req_to_h2(struct dynhds *h2_headers, struct httpreq *req, struct Curl_easy *data) { const char *scheme = NULL, *authority = NULL; struct dynhds_entry *e; size_t i; + // Use the Chrome ordering by default: + // :method, :authority, :scheme, :path + char *order = "masp"; CURLcode result; DEBUGASSERT(req); @@ -4816,25 +4962,56 @@ CURLcode Curl_http_req_to_h2(struct dynhds *h2_headers, Curl_dynhds_reset(h2_headers); Curl_dynhds_set_opts(h2_headers, DYNHDS_OPT_LOWERCASE); - result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_METHOD), - req->method, strlen(req->method)); - if(!result && scheme) { - result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_SCHEME), - scheme, strlen(scheme)); - } - if(!result && authority) { - result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_AUTHORITY), - authority, strlen(authority)); + + /* curl-impersonate: order of pseudo headers is different from the default */ + if(data->set.str[STRING_HTTP2_PSEUDO_HEADERS_ORDER]) { + order = data->set.str[STRING_HTTP2_PSEUDO_HEADERS_ORDER]; } - if(!result && req->path) { - result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_PATH), - req->path, strlen(req->path)); + + result = h2_check_pseudo_header_order(order); + + /* curl-impersonate: add http2 pseudo headers according to the specified order. */ + for(i = 0; !result && i < strlen(order); ++i) { + switch(order[i]) { + case 'm': + result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_METHOD), + req->method, strlen(req->method)); + break; + case 'a': + if(authority) { + result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_AUTHORITY), + authority, strlen(authority)); + } + break; + case 's': + if(scheme) { + result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_SCHEME), + scheme, strlen(scheme)); + } + break; + case 'p': + if(req->path) { + result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_PATH), + req->path, strlen(req->path)); + } + break; + } } + for(i = 0; !result && i < Curl_dynhds_count(&req->headers); ++i) { e = Curl_dynhds_getn(&req->headers, i); if(!h2_non_field(e->name, e->namelen)) { + /* curl-impersonate: + * Some HTTP/2 servers reject 'te' header value that is not lowercase (e.g. 'Trailers). + * Convert to lowercase explicitly. + */ + if(e->namelen == 2 && strcasecompare(e->name, "te")) + Curl_dynhds_set_opt(h2_headers, DYNHDS_OPT_LOWERCASE_VAL); + result = Curl_dynhds_add(h2_headers, e->name, e->namelen, e->value, e->valuelen); + + Curl_dynhds_del_opt(h2_headers, DYNHDS_OPT_LOWERCASE_VAL); } } diff --git a/lib/http2.c b/lib/http2.c index c666192fc..5ea7d04c8 100644 --- a/lib/http2.c +++ b/lib/http2.c @@ -50,6 +50,7 @@ #include "curl_printf.h" #include "curl_memory.h" #include "memdebug.h" +#include "rand.h" #if (NGHTTP2_VERSION_NUM < 0x010c00) #error too old nghttp2 version, upgrade! @@ -68,7 +69,7 @@ * use 16K as chunk size, as that fits H2 DATA frames well */ #define H2_CHUNK_SIZE (16 * 1024) /* this is how much we want "in flight" for a stream */ -#define H2_STREAM_WINDOW_SIZE (10 * 1024 * 1024) +#define H2_STREAM_WINDOW_SIZE (1024 * 1024) /* on receving from TLS, we prep for holding a full stream window */ #define H2_NW_RECV_CHUNKS (H2_STREAM_WINDOW_SIZE / H2_CHUNK_SIZE) /* on send into TLS, we just want to accumulate small frames */ @@ -86,24 +87,48 @@ * will block their received QUOTA in the connection window. And if we * run out of space, the server is blocked from sending us any data. * See #10988 for an issue with this. */ -#define HTTP2_HUGE_WINDOW_SIZE (100 * H2_STREAM_WINDOW_SIZE) +/* curl-impersonate: match Chrome window size. */ +#define HTTP2_HUGE_WINDOW_SIZE (15 * H2_STREAM_WINDOW_SIZE) -#define H2_SETTINGS_IV_LEN 3 +#define H2_SETTINGS_IV_LEN 8 #define H2_BINSETTINGS_LEN 80 static int populate_settings(nghttp2_settings_entry *iv, struct Curl_easy *data) { - iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; - iv[0].value = Curl_multi_max_concurrent_streams(data->multi); + int i = 0; - iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; - iv[1].value = H2_STREAM_WINDOW_SIZE; + /* curl-impersonate: Align HTTP/2 settings to Chrome's */ + iv[i].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE; + iv[i].value = 0x10000; + i++; - iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; - iv[2].value = data->multi->push_cb != NULL; + if(data->set.http2_no_server_push) { + iv[i].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; + iv[i].value = 0; + i++; + } + + iv[i].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; + iv[i].value = Curl_multi_max_concurrent_streams(data->multi); + i++; + + iv[i].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; + iv[i].value = 0x600000; + i++; - return 3; + iv[i].settings_id = NGHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE; + iv[i].value = 0x40000; + i++; + + // curl-impersonate: + // Up until Chrome 98, there was a randomly chosen setting number in the + // HTTP2 SETTINGS frame. This might be something similar to TLS GREASE. + // However, it seems to have been removed since. + // Curl_rand(data, (unsigned char *)&iv[4].settings_id, sizeof(iv[4].settings_id)); + // Curl_rand(data, (unsigned char *)&iv[4].value, sizeof(iv[4].value)); + + return i; } static size_t populate_binsettings(uint8_t *binsettings, @@ -1616,11 +1641,17 @@ out: return rv; } +/* + * curl-impersonate: Use Chrome's default HTTP/2 stream weight + * instead of NGINX default stream weight. + */ +#define CHROME_DEFAULT_STREAM_WEIGHT (256) + static int sweight_wanted(const struct Curl_easy *data) { /* 0 weight is not set by user and we take the nghttp2 default one */ return data->set.priority.weight? - data->set.priority.weight : NGHTTP2_DEFAULT_WEIGHT; + data->set.priority.weight : CHROME_DEFAULT_STREAM_WEIGHT; } static int sweight_in_effect(const struct Curl_easy *data) @@ -1642,9 +1673,11 @@ static void h2_pri_spec(struct Curl_easy *data, struct Curl_data_priority *prio = &data->set.priority; struct stream_ctx *depstream = H2_STREAM_CTX(prio->parent); int32_t depstream_id = depstream? depstream->id:0; + /* curl-impersonate: Set stream exclusive flag to true. */ + int exclusive = 1; nghttp2_priority_spec_init(pri_spec, depstream_id, sweight_wanted(data), - data->set.priority.exclusive); + exclusive); data->state.priority = *prio; } @@ -1661,20 +1694,25 @@ static CURLcode h2_progress_egress(struct Curl_cfilter *cf, struct stream_ctx *stream = H2_STREAM_CTX(data); int rv = 0; - if((sweight_wanted(data) != sweight_in_effect(data)) || - (data->set.priority.exclusive != data->state.priority.exclusive) || - (data->set.priority.parent != data->state.priority.parent) ) { + /* curl-impersonate: Check if stream exclusive flag is true. */ + if(stream && stream->id > 0 && + ((sweight_wanted(data) != sweight_in_effect(data)) || + (data->set.priority.exclusive != 1) || + (data->set.priority.parent != data->state.priority.parent))) { /* send new weight and/or dependency */ nghttp2_priority_spec pri_spec; h2_pri_spec(data, &pri_spec); - DEBUGF(LOG_CF(data, cf, "[h2sid=%d] Queuing PRIORITY", - stream->id)); - DEBUGASSERT(stream->id != -1); - rv = nghttp2_submit_priority(ctx->h2, NGHTTP2_FLAG_NONE, - stream->id, &pri_spec); - if(rv) - goto out; + /* curl-impersonate: Don't send PRIORITY frames for main stream. */ + if(stream->id != 1) { + DEBUGF(LOG_CF(data, cf, "[h2sid=%d] Queuing PRIORITY", + stream->id)); + DEBUGASSERT(stream->id != -1); + rv = nghttp2_submit_priority(ctx->h2, NGHTTP2_FLAG_NONE, + stream->id, &pri_spec); + if(rv) + goto out; + } } while(!rv && nghttp2_session_want_write(ctx->h2)) diff --git a/lib/http2.h b/lib/http2.h index 562c05c99..b99c085d5 100644 --- a/lib/http2.h +++ b/lib/http2.h @@ -31,7 +31,8 @@ /* value for MAX_CONCURRENT_STREAMS we use until we get an updated setting from the peer */ -#define DEFAULT_MAX_CONCURRENT_STREAMS 100 +/* curl-impersonate: Use 1000 concurrent streams like Chrome. */ +#define DEFAULT_MAX_CONCURRENT_STREAMS 1000 /* * Store nghttp2 version info in this buffer. diff --git a/lib/impersonate.c b/lib/impersonate.c new file mode 100644 index 000000000..a8ea0afa5 --- /dev/null +++ b/lib/impersonate.c @@ -0,0 +1,524 @@ +#include "curl_setup.h" + +#include + +#include "impersonate.h" + +const struct impersonate_opts impersonations[] = { + { + .target = "chrome99", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "ECDHE-ECDSA-AES128-GCM-SHA256," + "ECDHE-RSA-AES128-GCM-SHA256," + "ECDHE-ECDSA-AES256-GCM-SHA384," + "ECDHE-RSA-AES256-GCM-SHA384," + "ECDHE-ECDSA-CHACHA20-POLY1305," + "ECDHE-RSA-CHACHA20-POLY1305," + "ECDHE-RSA-AES128-SHA," + "ECDHE-RSA-AES256-SHA," + "AES128-GCM-SHA256," + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", + .npn = false, + .alpn = true, + .alps = true, + .tls_session_ticket = true, + .cert_compression = "brotli", + .http_headers = { + "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"99\", \"Google Chrome\";v=\"99\"", + "sec-ch-ua-mobile: ?0", + "sec-ch-ua-platform: \"Windows\"", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Sec-Fetch-Site: none", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-User: ?1", + "Sec-Fetch-Dest: document", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + } + }, + { + .target = "chrome100", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "ECDHE-ECDSA-AES128-GCM-SHA256," + "ECDHE-RSA-AES128-GCM-SHA256," + "ECDHE-ECDSA-AES256-GCM-SHA384," + "ECDHE-RSA-AES256-GCM-SHA384," + "ECDHE-ECDSA-CHACHA20-POLY1305," + "ECDHE-RSA-CHACHA20-POLY1305," + "ECDHE-RSA-AES128-SHA," + "ECDHE-RSA-AES256-SHA," + "AES128-GCM-SHA256," + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", + .npn = false, + .alpn = true, + .alps = true, + .tls_session_ticket = true, + .cert_compression = "brotli", + .http_headers = { + "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", + "sec-ch-ua-mobile: ?0", + "sec-ch-ua-platform: \"Windows\"", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Sec-Fetch-Site: none", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-User: ?1", + "Sec-Fetch-Dest: document", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + } + }, + { + .target = "chrome101", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "ECDHE-ECDSA-AES128-GCM-SHA256," + "ECDHE-RSA-AES128-GCM-SHA256," + "ECDHE-ECDSA-AES256-GCM-SHA384," + "ECDHE-RSA-AES256-GCM-SHA384," + "ECDHE-ECDSA-CHACHA20-POLY1305," + "ECDHE-RSA-CHACHA20-POLY1305," + "ECDHE-RSA-AES128-SHA," + "ECDHE-RSA-AES256-SHA," + "AES128-GCM-SHA256," + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", + .npn = false, + .alpn = true, + .alps = true, + .tls_session_ticket = true, + .cert_compression = "brotli", + .http_headers = { + "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"101\", \"Google Chrome\";v=\"101\"", + "sec-ch-ua-mobile: ?0", + "sec-ch-ua-platform: \"Windows\"", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Sec-Fetch-Site: none", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-User: ?1", + "Sec-Fetch-Dest: document", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + } + }, + { + .target = "chrome104", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "ECDHE-ECDSA-AES128-GCM-SHA256," + "ECDHE-RSA-AES128-GCM-SHA256," + "ECDHE-ECDSA-AES256-GCM-SHA384," + "ECDHE-RSA-AES256-GCM-SHA384," + "ECDHE-ECDSA-CHACHA20-POLY1305," + "ECDHE-RSA-CHACHA20-POLY1305," + "ECDHE-RSA-AES128-SHA," + "ECDHE-RSA-AES256-SHA," + "AES128-GCM-SHA256," + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", + .npn = false, + .alpn = true, + .alps = true, + .tls_session_ticket = true, + .cert_compression = "brotli", + .http_headers = { + "sec-ch-ua: \"Chromium\";v=\"104\", \" Not A;Brand\";v=\"99\", \"Google Chrome\";v=\"104\"", + "sec-ch-ua-mobile: ?0", + "sec-ch-ua-platform: \"Windows\"", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Sec-Fetch-Site: none", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-User: ?1", + "Sec-Fetch-Dest: document", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + } + }, + { + .target = "chrome107", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "ECDHE-ECDSA-AES128-GCM-SHA256," + "ECDHE-RSA-AES128-GCM-SHA256," + "ECDHE-ECDSA-AES256-GCM-SHA384," + "ECDHE-RSA-AES256-GCM-SHA384," + "ECDHE-ECDSA-CHACHA20-POLY1305," + "ECDHE-RSA-CHACHA20-POLY1305," + "ECDHE-RSA-AES128-SHA," + "ECDHE-RSA-AES256-SHA," + "AES128-GCM-SHA256," + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", + .npn = false, + .alpn = true, + .alps = true, + .tls_session_ticket = true, + .cert_compression = "brotli", + .http_headers = { + "sec-ch-ua: \"Google Chrome\";v=\"107\", \"Chromium\";v=\"107\", \"Not=A?Brand\";v=\"24\"", + "sec-ch-ua-mobile: ?0", + "sec-ch-ua-platform: \"Windows\"", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Sec-Fetch-Site: none", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-User: ?1", + "Sec-Fetch-Dest: document", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + }, + .http2_no_server_push = true + }, + { + .target = "chrome110", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "ECDHE-ECDSA-AES128-GCM-SHA256," + "ECDHE-RSA-AES128-GCM-SHA256," + "ECDHE-ECDSA-AES256-GCM-SHA384," + "ECDHE-RSA-AES256-GCM-SHA384," + "ECDHE-ECDSA-CHACHA20-POLY1305," + "ECDHE-RSA-CHACHA20-POLY1305," + "ECDHE-RSA-AES128-SHA," + "ECDHE-RSA-AES256-SHA," + "AES128-GCM-SHA256," + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", + .npn = false, + .alpn = true, + .alps = true, + .tls_permute_extensions = true, + .tls_session_ticket = true, + .cert_compression = "brotli", + .http_headers = { + "sec-ch-ua: \"Chromium\";v=\"110\", \"Not A(Brand\";v=\"24\", \"Google Chrome\";v=\"110\"", + "sec-ch-ua-mobile: ?0", + "sec-ch-ua-platform: \"Windows\"", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "Sec-Fetch-Site: none", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-User: ?1", + "Sec-Fetch-Dest: document", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + }, + .http2_no_server_push = true + }, + { + .target = "chrome116", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "ECDHE-ECDSA-AES128-GCM-SHA256," + "ECDHE-RSA-AES128-GCM-SHA256," + "ECDHE-ECDSA-AES256-GCM-SHA384," + "ECDHE-RSA-AES256-GCM-SHA384," + "ECDHE-ECDSA-CHACHA20-POLY1305," + "ECDHE-RSA-CHACHA20-POLY1305," + "ECDHE-RSA-AES128-SHA," + "ECDHE-RSA-AES256-SHA," + "AES128-GCM-SHA256," + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", + .npn = false, + .alpn = true, + .alps = true, + .tls_permute_extensions = true, + .tls_session_ticket = true, + .cert_compression = "brotli", + .http_headers = { + "sec-ch-ua: \"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"", + "sec-ch-ua-mobile: ?0", + "sec-ch-ua-platform: \"Windows\"", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "Sec-Fetch-Site: none", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-User: ?1", + "Sec-Fetch-Dest: document", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + }, + .http2_no_server_push = true + }, + { + .target = "chrome99_android", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "ECDHE-ECDSA-AES128-GCM-SHA256," + "ECDHE-RSA-AES128-GCM-SHA256," + "ECDHE-ECDSA-AES256-GCM-SHA384," + "ECDHE-RSA-AES256-GCM-SHA384," + "ECDHE-ECDSA-CHACHA20-POLY1305," + "ECDHE-RSA-CHACHA20-POLY1305," + "ECDHE-RSA-AES128-SHA," + "ECDHE-RSA-AES256-SHA," + "AES128-GCM-SHA256," + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", + .npn = false, + .alpn = true, + .alps = true, + .tls_session_ticket = true, + .cert_compression = "brotli", + .http_headers = { + "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"99\", \"Google Chrome\";v=\"99\"", + "sec-ch-ua-mobile: ?1", + "sec-ch-ua-platform: \"Android\"", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Sec-Fetch-Site: none", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-User: ?1", + "Sec-Fetch-Dest: document", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + } + }, + { + .target = "edge99", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "ECDHE-ECDSA-AES128-GCM-SHA256," + "ECDHE-RSA-AES128-GCM-SHA256," + "ECDHE-ECDSA-AES256-GCM-SHA384," + "ECDHE-RSA-AES256-GCM-SHA384," + "ECDHE-ECDSA-CHACHA20-POLY1305," + "ECDHE-RSA-CHACHA20-POLY1305," + "ECDHE-RSA-AES128-SHA," + "ECDHE-RSA-AES256-SHA," + "AES128-GCM-SHA256," + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", + .npn = false, + .alpn = true, + .alps = true, + .tls_session_ticket = true, + .cert_compression = "brotli", + .http_headers = { + "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"99\", \"Microsoft Edge\";v=\"99\"", + "sec-ch-ua-mobile: ?0", + "sec-ch-ua-platform: \"Windows\"", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.30", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Sec-Fetch-Site: none", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-User: ?1", + "Sec-Fetch-Dest: document", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + } + }, + { + .target = "edge101", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "ECDHE-ECDSA-AES128-GCM-SHA256," + "ECDHE-RSA-AES128-GCM-SHA256," + "ECDHE-ECDSA-AES256-GCM-SHA384," + "ECDHE-RSA-AES256-GCM-SHA384," + "ECDHE-ECDSA-CHACHA20-POLY1305," + "ECDHE-RSA-CHACHA20-POLY1305," + "ECDHE-RSA-AES128-SHA," + "ECDHE-RSA-AES256-SHA," + "AES128-GCM-SHA256," + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", + .npn = false, + .alpn = true, + .alps = true, + .tls_session_ticket = true, + .cert_compression = "brotli", + .http_headers = { + "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"101\", \"Microsoft Edge\";v=\"101\"", + "sec-ch-ua-mobile: ?0", + "sec-ch-ua-platform: \"Windows\"", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.47", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Sec-Fetch-Site: none", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-User: ?1", + "Sec-Fetch-Dest: document", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9" + } + }, + { + .target = "safari15_3", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_0 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384," + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256," + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384," + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256," + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384," + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256," + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA," + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA," + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384," + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256," + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA," + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA," + "TLS_RSA_WITH_AES_256_GCM_SHA384," + "TLS_RSA_WITH_AES_128_GCM_SHA256," + "TLS_RSA_WITH_AES_256_CBC_SHA256," + "TLS_RSA_WITH_AES_128_CBC_SHA256," + "TLS_RSA_WITH_AES_256_CBC_SHA," + "TLS_RSA_WITH_AES_128_CBC_SHA," + "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA," + "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA," + "TLS_RSA_WITH_3DES_EDE_CBC_SHA,", + .curves = "X25519:P-256:P-384:P-521", + .sig_hash_algs = + "ecdsa_secp256r1_sha256," + "rsa_pss_rsae_sha256," + "rsa_pkcs1_sha256," + "ecdsa_secp384r1_sha384," + "ecdsa_sha1," + "rsa_pss_rsae_sha384," + "rsa_pss_rsae_sha384," + "rsa_pkcs1_sha384," + "rsa_pss_rsae_sha512," + "rsa_pkcs1_sha512," + "rsa_pkcs1_sha1", + .npn = false, + .alpn = true, + .alps = false, + .tls_session_ticket = false, + .http_headers = { + "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language: en-us", + "Accept-Encoding: gzip, deflate, br" + }, + .http2_pseudo_headers_order = "mspa" + }, + { + .target = "safari15_5", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_0 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384," + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256," + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384," + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256," + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA," + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA," + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA," + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA," + "TLS_RSA_WITH_AES_256_GCM_SHA384," + "TLS_RSA_WITH_AES_128_GCM_SHA256," + "TLS_RSA_WITH_AES_256_CBC_SHA," + "TLS_RSA_WITH_AES_128_CBC_SHA," + "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA," + "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA," + "TLS_RSA_WITH_3DES_EDE_CBC_SHA", + .curves = "X25519:P-256:P-384:P-521", + .sig_hash_algs = + "ecdsa_secp256r1_sha256," + "rsa_pss_rsae_sha256," + "rsa_pkcs1_sha256," + "ecdsa_secp384r1_sha384," + "ecdsa_sha1," + "rsa_pss_rsae_sha384," + "rsa_pss_rsae_sha384," + "rsa_pkcs1_sha384," + "rsa_pss_rsae_sha512," + "rsa_pkcs1_sha512," + "rsa_pkcs1_sha1", + .npn = false, + .alpn = true, + .alps = false, + .tls_session_ticket = false, + .cert_compression = "zlib", + .http_headers = { + "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8", + "Accept-Encoding: gzip, deflate, br" + }, + .http2_pseudo_headers_order = "mspa" + }, + { + /* Last one must be NULL. */ + .target = NULL + } +}; diff --git a/lib/impersonate.h b/lib/impersonate.h new file mode 100644 index 000000000..c62991c5a --- /dev/null +++ b/lib/impersonate.h @@ -0,0 +1,45 @@ +#ifndef HEADER_CURL_IMPERSONATE_H +#define HEADER_CURL_IMPERSONATE_H + +#define IMPERSONATE_MAX_HEADERS 32 + +/* + * curl-impersonate: Options to be set for each supported target browser. + */ +struct impersonate_opts { + const char *target; + int httpversion; + int ssl_version; + const char *ciphers; + /* Elliptic curves (TLS extension 10). + * Passed to CURLOPT_SSL_EC_CURVES */ + const char *curves; + /* Signature hash algorithms (TLS extension 13). + * Passed to CURLOPT_SSL_SIG_HASH_ALGS */ + const char *sig_hash_algs; + /* Enable TLS NPN extension. */ + bool npn; + /* Enable TLS ALPN extension. */ + bool alpn; + /* Enable TLS ALPS extension. */ + bool alps; + /* Enable TLS session ticket extension. */ + bool tls_session_ticket; + /* TLS certificate compression algorithms. + * (TLS extension 27) */ + const char *cert_compression; + const char *http_headers[IMPERSONATE_MAX_HEADERS]; + const char *http2_pseudo_headers_order; + bool http2_no_server_push; + bool tls_permute_extensions; + /* Other TLS options will come here in the future once they are + * configurable through curl_easy_setopt() */ +}; + +/* + * curl-impersonate: Global array of supported browsers and their + * impersonation options. + */ +extern const struct impersonate_opts impersonations[]; + +#endif /* HEADER_CURL_IMPERSONATE_H */ diff --git a/lib/multi.c b/lib/multi.c index d1d32b793..3b49a1b4c 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -424,7 +424,8 @@ struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */ /* -1 means it not set by user, use the default value */ multi->maxconnects = -1; - multi->max_concurrent_streams = 100; + /* curl-impersonate: Use 1000 concurrent streams like Chrome. */ + multi->max_concurrent_streams = 1000; #ifdef USE_WINSOCK multi->wsa_event = WSACreateEvent(); diff --git a/lib/setopt.c b/lib/setopt.c index 0c3b9634d..c22591f72 100644 --- a/lib/setopt.c +++ b/lib/setopt.c @@ -50,6 +50,7 @@ #include "multiif.h" #include "altsvc.h" #include "hsts.h" +#include "slist.h" /* The last 3 #include files should be in this order */ #include "curl_printf.h" @@ -712,6 +713,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) va_arg(param, char *)); break; + case CURLOPT_HTTPBASEHEADER: + /* + * curl-impersonate: + * Set a list of "base" headers. These will be merged with any headers + * set by CURLOPT_HTTPHEADER. curl-impersonate uses this option in order + * to set a list of default browser headers. + * + * Unlike CURLOPT_HTTPHEADER, + * the list is copied and can be immediately freed by the user. + */ + curl_slist_free_all(data->state.base_headers); + data->state.base_headers = \ + Curl_slist_duplicate(va_arg(param, struct curl_slist *)); + if (!data->state.base_headers) + result = CURLE_OUT_OF_MEMORY; + break; + #ifndef CURL_DISABLE_PROXY case CURLOPT_PROXYHEADER: /* @@ -2410,6 +2428,27 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) result = Curl_setstropt(&data->set.str[STRING_SSL_EC_CURVES], va_arg(param, char *)); break; + + case CURLOPT_SSL_SIG_HASH_ALGS: + /* + * Set the list of hash algorithms we want to use in the SSL connection. + * Specify comma-delimited list of algorithms to use. + */ + result = Curl_setstropt(&data->set.str[STRING_SSL_SIG_HASH_ALGS], + va_arg(param, char *)); + break; + + case CURLOPT_SSL_CERT_COMPRESSION: + /* + * Set the list of ceritifcate compression algorithms we support in the TLS + * connection. + * Specify comma-delimited list of algorithms to use. Options are "zlib" + * and "brotli". + */ + result = Curl_setstropt(&data->set.str[STRING_SSL_CERT_COMPRESSION], + va_arg(param, char *)); + break; + #endif case CURLOPT_IPRESOLVE: arg = va_arg(param, long); @@ -2953,6 +2992,22 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) case CURLOPT_SSL_ENABLE_ALPN: data->set.ssl_enable_alpn = (0 != va_arg(param, long)) ? TRUE : FALSE; break; + case CURLOPT_SSL_ENABLE_ALPS: + data->set.ssl_enable_alps = (0 != va_arg(param, long)) ? TRUE : FALSE; + break; + case CURLOPT_SSL_ENABLE_TICKET: + data->set.ssl_enable_ticket = (0 != va_arg(param, long)) ? TRUE : FALSE; + break; + case CURLOPT_SSL_PERMUTE_EXTENSIONS: + data->set.ssl_permute_extensions = (0 != va_arg(param, long)) ? TRUE : FALSE; + break; + case CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER: + result = Curl_setstropt(&data->set.str[STRING_HTTP2_PSEUDO_HEADERS_ORDER], + va_arg(param, char *)); + break; + case CURLOPT_HTTP2_NO_SERVER_PUSH: + data->set.http2_no_server_push = (0 != va_arg(param, long)) ? TRUE : FALSE; + break; #ifdef USE_UNIX_SOCKETS case CURLOPT_UNIX_SOCKET_PATH: data->set.abstract_unix_socket = FALSE; diff --git a/lib/transfer.c b/lib/transfer.c index d2ff0c24c..56e2090b6 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -106,7 +106,15 @@ char *Curl_checkheaders(const struct Curl_easy *data, DEBUGASSERT(thislen); DEBUGASSERT(thisheader[thislen-1] != ':'); - for(head = data->set.headers; head; head = head->next) { + /* + * curl-impersonate: + * Check if we have overriden the user-supplied list of headers. + */ + head = data->set.headers; + if (data->state.merged_headers) + head = data->state.merged_headers; + + for(; head; head = head->next) { if(strncasecompare(head->data, thisheader, thislen) && Curl_headersep(head->data[thislen]) ) return head->data; diff --git a/lib/url.c b/lib/url.c index b37d13f8f..f1b3b5440 100644 --- a/lib/url.c +++ b/lib/url.c @@ -444,6 +444,11 @@ CURLcode Curl_close(struct Curl_easy **datap) Curl_safefree(data->state.aptr.proxyuser); Curl_safefree(data->state.aptr.proxypasswd); + /* curl-impersonate: Free the list set by CURLOPT_HTTPBASEHEADER. */ + curl_slist_free_all(data->state.base_headers); + /* curl-impersonate: Free the dynamic list of headers. */ + curl_slist_free_all(data->state.merged_headers); + #ifndef CURL_DISABLE_DOH if(data->req.doh) { Curl_dyn_free(&data->req.doh->probe[0].serverdoh); @@ -595,6 +600,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data) set->tcp_fastopen = FALSE; set->tcp_nodelay = TRUE; set->ssl_enable_alpn = TRUE; + set->ssl_enable_ticket = TRUE; set->expect_100_timeout = 1000L; /* Wait for a second by default. */ set->sep_headers = TRUE; /* separated header lists by default */ set->buffer_size = READBUFFER_SIZE; @@ -3584,6 +3590,9 @@ static CURLcode create_conn(struct Curl_easy *data, data->set.ssl.primary.cert_blob = data->set.blobs[BLOB_CERT]; data->set.ssl.primary.ca_info_blob = data->set.blobs[BLOB_CAINFO]; data->set.ssl.primary.curves = data->set.str[STRING_SSL_EC_CURVES]; + data->set.ssl.primary.sig_hash_algs = data->set.str[STRING_SSL_SIG_HASH_ALGS]; + data->set.ssl.primary.cert_compression = + data->set.str[STRING_SSL_CERT_COMPRESSION]; #ifndef CURL_DISABLE_PROXY data->set.proxy_ssl.primary.CApath = data->set.str[STRING_SSL_CAPATH_PROXY]; @@ -3695,6 +3704,11 @@ static CURLcode create_conn(struct Curl_easy *data, (default) */ if(data->set.ssl_enable_alpn) conn->bits.tls_enable_alpn = TRUE; + + /* curl-impersonate: Turn on ALPS if ALPN is enabled and the bit is + * enabled. */ + if(data->set.ssl_enable_alps) + conn->bits.tls_enable_alps = TRUE; } if(waitpipe) diff --git a/lib/urldata.h b/lib/urldata.h index f02e66541..d628112a0 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -277,6 +277,8 @@ struct ssl_primary_config { char *password; /* TLS password (for, e.g., SRP) */ #endif char *curves; /* list of curves to use */ + char *sig_hash_algs; /* List of signature hash algorithms to use */ + char *cert_compression; /* List of certificate compression algorithms. */ unsigned char ssl_options; /* the CURLOPT_SSL_OPTIONS bitmask */ unsigned int version_max; /* max supported version the client wants to use */ unsigned char version; /* what version the client wants to use */ @@ -526,6 +528,9 @@ struct ConnectBits { BIT(multiplex); /* connection is multiplexed */ BIT(tcp_fastopen); /* use TCP Fast Open */ BIT(tls_enable_alpn); /* TLS ALPN extension? */ + BIT(tls_enable_alps); /* TLS ALPS extension? */ + BIT(tls_enable_ticket); /* TLS session ticket extension? */ + BIT(tls_permute_extensions); /* TLS extension permutations */ #ifndef CURL_DISABLE_DOH BIT(doh); #endif @@ -1395,6 +1400,19 @@ struct UrlState { CURLcode hresult; /* used to pass return codes back from hyper callbacks */ #endif + /* + * curl-impersonate: + * List of "base" headers set by CURLOPT_HTTPBASEHEADER. + */ + struct curl_slist *base_headers; + /* + * curl-impersonate: + * Dynamically-constructed list of HTTP headers. + * This list is a merge of the default HTTP headers needed to impersonate a + * browser, together with any user-supplied headers. + */ + struct curl_slist *merged_headers; + /* Dynamically allocated strings, MUST be freed before this struct is killed. */ struct dynamically_allocated_data { @@ -1563,6 +1581,9 @@ enum dupstring { STRING_DNS_LOCAL_IP6, STRING_SSL_EC_CURVES, STRING_AWS_SIGV4, /* Parameters for V4 signature */ + STRING_SSL_SIG_HASH_ALGS, + STRING_SSL_CERT_COMPRESSION, + STRING_HTTP2_PSEUDO_HEADERS_ORDER, /* -- end of null-terminated strings -- */ @@ -1857,6 +1878,9 @@ struct UserDefined { BIT(tcp_keepalive); /* use TCP keepalives */ BIT(tcp_fastopen); /* use TCP Fast Open */ BIT(ssl_enable_alpn);/* TLS ALPN extension? */ + BIT(ssl_enable_alps);/* TLS ALPS extension? */ + BIT(ssl_enable_ticket); /* TLS session ticket extension */ + BIT(ssl_permute_extensions); /* TLS Permute extensions */ BIT(path_as_is); /* allow dotdots? */ BIT(pipewait); /* wait for multiplex status before starting a new connection */ @@ -1877,6 +1901,9 @@ struct UserDefined { #ifdef USE_WEBSOCKETS BIT(ws_raw_mode); #endif +#ifdef USE_HTTP2 + BIT(http2_no_server_push); /* Disable HTTP2 server push */ +#endif }; struct Names { diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c index 6543fb19a..da4882af8 100644 --- a/lib/vtls/openssl.c +++ b/lib/vtls/openssl.c @@ -79,6 +79,14 @@ #include #include #include +#include + +#ifdef HAVE_LIBZ +#include +#endif +#ifdef HAVE_BROTLI +#include +#endif #if (OPENSSL_VERSION_NUMBER >= 0x0090808fL) && !defined(OPENSSL_NO_OCSP) #include @@ -266,6 +274,113 @@ #define HAVE_OPENSSL_VERSION #endif +#if defined(OPENSSL_IS_BORINGSSL) +#define HAVE_SSL_CTX_SET_VERIFY_ALGORITHM_PREFS + +/* + * kMaxSignatureAlgorithmNameLen and kSignatureAlgorithmNames + * Taken from BoringSSL, see ssl/ssl_privkey.cc + * */ +static const size_t kMaxSignatureAlgorithmNameLen = 23; + +static const struct { + uint16_t signature_algorithm; + const char *name; +} kSignatureAlgorithmNames[] = { + {SSL_SIGN_RSA_PKCS1_MD5_SHA1, "rsa_pkcs1_md5_sha1"}, + {SSL_SIGN_RSA_PKCS1_SHA1, "rsa_pkcs1_sha1"}, + {SSL_SIGN_RSA_PKCS1_SHA256, "rsa_pkcs1_sha256"}, + {SSL_SIGN_RSA_PKCS1_SHA384, "rsa_pkcs1_sha384"}, + {SSL_SIGN_RSA_PKCS1_SHA512, "rsa_pkcs1_sha512"}, + {SSL_SIGN_ECDSA_SHA1, "ecdsa_sha1"}, + {SSL_SIGN_ECDSA_SECP256R1_SHA256, "ecdsa_secp256r1_sha256"}, + {SSL_SIGN_ECDSA_SECP384R1_SHA384, "ecdsa_secp384r1_sha384"}, + {SSL_SIGN_ECDSA_SECP521R1_SHA512, "ecdsa_secp521r1_sha512"}, + {SSL_SIGN_RSA_PSS_RSAE_SHA256, "rsa_pss_rsae_sha256"}, + {SSL_SIGN_RSA_PSS_RSAE_SHA384, "rsa_pss_rsae_sha384"}, + {SSL_SIGN_RSA_PSS_RSAE_SHA512, "rsa_pss_rsae_sha512"}, + {SSL_SIGN_ED25519, "ed25519"}, +}; + +#define MAX_SIG_ALGS \ + sizeof(kSignatureAlgorithmNames) / sizeof(kSignatureAlgorithmNames[0]) + +/* Default signature hash algorithms taken from Chrome/Chromium. + * See kVerifyPeers @ net/socket/ssl_client_socket_impl.cc */ +static const uint16_t default_sig_algs[] = { + SSL_SIGN_ECDSA_SECP256R1_SHA256, SSL_SIGN_RSA_PSS_RSAE_SHA256, + SSL_SIGN_RSA_PKCS1_SHA256, SSL_SIGN_ECDSA_SECP384R1_SHA384, + SSL_SIGN_RSA_PSS_RSAE_SHA384, SSL_SIGN_RSA_PKCS1_SHA384, + SSL_SIGN_RSA_PSS_RSAE_SHA512, SSL_SIGN_RSA_PKCS1_SHA512, +}; + +#define DEFAULT_SIG_ALGS_LENGTH \ + sizeof(default_sig_algs) / sizeof(default_sig_algs[0]) + +static CURLcode parse_sig_algs(struct Curl_easy *data, + const char *sigalgs, + uint16_t *algs, + size_t *nalgs) +{ + *nalgs = 0; + while (sigalgs && sigalgs[0]) { + int i; + bool found = FALSE; + const char *end; + size_t len; + char algname[kMaxSignatureAlgorithmNameLen + 1]; + + end = strpbrk(sigalgs, ":,"); + if (end) + len = end - sigalgs; + else + len = strlen(sigalgs); + + if (len > kMaxSignatureAlgorithmNameLen) { + failf(data, "Bad signature hash algorithm list"); + return CURLE_BAD_FUNCTION_ARGUMENT; + } + + if (!len) { + ++sigalgs; + continue; + } + + if (*nalgs == MAX_SIG_ALGS) { + /* Reached the maximum number of possible algorithms, but more data + * available in the list. */ + failf(data, "Bad signature hash algorithm list"); + return CURLE_BAD_FUNCTION_ARGUMENT; + } + + memcpy(algname, sigalgs, len); + algname[len] = 0; + + for (i = 0; i < MAX_SIG_ALGS; i++) { + if (strcasecompare(algname, kSignatureAlgorithmNames[i].name)) { + algs[*nalgs] = kSignatureAlgorithmNames[i].signature_algorithm; + (*nalgs)++; + found = TRUE; + break; + } + } + + if (!found) { + failf(data, "Unknown signature hash algorithm: '%s'", algname); + return CURLE_BAD_FUNCTION_ARGUMENT; + } + + if (end) + sigalgs = ++end; + else + break; + } + + return CURLE_OK; +} + +#endif + #ifdef OPENSSL_IS_BORINGSSL typedef uint32_t sslerr_t; #else @@ -2583,6 +2698,151 @@ static const char *tls_rt_type(int type) } } +#ifdef HAVE_LIBZ +int DecompressZlibCert(SSL *ssl, + CRYPTO_BUFFER** out, + size_t uncompressed_len, + const uint8_t* in, + size_t in_len) +{ + z_stream strm; + uint8_t* data; + CRYPTO_BUFFER* decompressed = CRYPTO_BUFFER_alloc(&data, uncompressed_len); + if(!decompressed) { + return 0; + } + + strm.zalloc = NULL; + strm.zfree = NULL; + strm.opaque = NULL; + strm.next_in = (Bytef *)in; + strm.avail_in = in_len; + strm.next_out = (Bytef *)data; + strm.avail_out = uncompressed_len; + + if(inflateInit(&strm) != Z_OK) { + CRYPTO_BUFFER_free(decompressed); + return 0; + } + + if(inflate(&strm, Z_FINISH) != Z_STREAM_END || + strm.avail_in != 0 || + strm.avail_out != 0) { + inflateEnd(&strm); + CRYPTO_BUFFER_free(decompressed); + return 0; + } + + inflateEnd(&strm); + *out = decompressed; + return 1; +} +#endif + +#ifdef HAVE_BROTLI + +/* Taken from Chromium and adapted to C, + * see net/ssl/cert_compression.cc + */ +int DecompressBrotliCert(SSL* ssl, + CRYPTO_BUFFER** out, + size_t uncompressed_len, + const uint8_t* in, + size_t in_len) { + uint8_t* data; + CRYPTO_BUFFER* decompressed = CRYPTO_BUFFER_alloc(&data, uncompressed_len); + if (!decompressed) { + return 0; + } + + size_t output_size = uncompressed_len; + if (BrotliDecoderDecompress(in_len, in, &output_size, data) != + BROTLI_DECODER_RESULT_SUCCESS || + output_size != uncompressed_len) { + CRYPTO_BUFFER_free(decompressed); + return 0; + } + + *out = decompressed; + return 1; +} +#endif + +#if defined(HAVE_LIBZ) || defined(HAVE_BROTLI) +static struct { + char *alg_name; + uint16_t alg_id; + ssl_cert_compression_func_t compress; + ssl_cert_decompression_func_t decompress; +} cert_compress_algs[] = { +#ifdef HAVE_LIBZ + {"zlib", TLSEXT_cert_compression_zlib, NULL, DecompressZlibCert}, +#endif +#ifdef HAVE_BROTLI + {"brotli", TLSEXT_cert_compression_brotli, NULL, DecompressBrotliCert}, +#endif +}; + +#define NUM_CERT_COMPRESSION_ALGS \ + sizeof(cert_compress_algs) / sizeof(cert_compress_algs[0]) + +/* + * curl-impersonate: + * Add support for TLS extension 27 - compress_certificate. + * This calls the BoringSSL-specific API SSL_CTX_add_cert_compression_alg + * for each algorithm specified in cert_compression, which is a comma separated list. + */ +static CURLcode add_cert_compression(struct Curl_easy *data, + SSL_CTX *ctx, + const char *algorithms) +{ + int i; + const char *s = algorithms; + char *alg_name; + size_t alg_name_len; + bool found; + + while (s && s[0]) { + found = FALSE; + + for(i = 0; i < NUM_CERT_COMPRESSION_ALGS; i++) { + alg_name = cert_compress_algs[i].alg_name; + alg_name_len = strlen(alg_name); + if(strlen(s) >= alg_name_len && + strncasecompare(s, alg_name, alg_name_len) && + (s[alg_name_len] == ',' || s[alg_name_len] == 0)) { + if(!SSL_CTX_add_cert_compression_alg(ctx, + cert_compress_algs[i].alg_id, + cert_compress_algs[i].compress, + cert_compress_algs[i].decompress)) { + failf(data, "Error adding certificate compression algorithm '%s'", + alg_name); + return CURLE_SSL_CIPHER; + } + s += alg_name_len; + if(*s == ',') + s += 1; + found = TRUE; + break; + } + } + + if(!found) { + failf(data, "Invalid compression algorithm list"); + return CURLE_BAD_FUNCTION_ARGUMENT; + } + } + + return CURLE_OK; +} +#else +static CURLcode add_cert_compression(SSL_CTX *ctx, const char *algorithms) +{ + /* No compression algorithms are available. */ + return CURLE_BAD_FUNCTION_ARGUMENT; +} +#endif + /* * Our callback from the SSL/TLS layers. */ @@ -3536,7 +3796,14 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, ctx_options = SSL_OP_ALL; #ifdef SSL_OP_NO_TICKET - ctx_options |= SSL_OP_NO_TICKET; + if(data->set.ssl_enable_ticket) { + /* curl-impersonate: + * Turn off SSL_OP_NO_TICKET, we want TLS extension 35 (session_ticket) + * to be present in the client hello. */ + ctx_options &= ~SSL_OP_NO_TICKET; + } else { + ctx_options |= SSL_OP_NO_TICKET; + } #endif #ifdef SSL_OP_NO_COMPRESSION @@ -3603,6 +3870,16 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, } #endif + SSL_CTX_set_options(backend->ctx, + SSL_OP_LEGACY_SERVER_CONNECT); + SSL_CTX_set_mode(backend->ctx, + SSL_MODE_CBC_RECORD_SPLITTING | SSL_MODE_ENABLE_FALSE_START); + + /* curl-impersonate: Enable TLS extensions 5 - status_request and + * 18 - signed_certificate_timestamp. */ + SSL_CTX_enable_signed_cert_timestamps(backend->ctx); + SSL_CTX_enable_ocsp_stapling(backend->ctx); + if(ssl_cert || ssl_cert_blob || ssl_cert_type) { if(!result && !cert_stuff(data, backend->ctx, @@ -3656,6 +3933,35 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, } #endif +#ifdef HAVE_SSL_CTX_SET_VERIFY_ALGORITHM_PREFS + { + uint16_t algs[MAX_SIG_ALGS]; + size_t nalgs; + /* curl-impersonate: Set the signature algorithms (TLS extension 13). + * See net/socket/ssl_client_socket_impl.cc in Chromium's source. */ + char *sig_hash_algs = conn_config->sig_hash_algs; + if (sig_hash_algs) { + CURLcode result = parse_sig_algs(data, sig_hash_algs, algs, &nalgs); + if (result) + return result; + if (!SSL_CTX_set_verify_algorithm_prefs(backend->ctx, algs, nalgs)) { + failf(data, "failed setting signature hash algorithms list: '%s'", + sig_hash_algs); + return CURLE_SSL_CIPHER; + } + } else { + /* Use defaults from Chrome. */ + if (!SSL_CTX_set_verify_algorithm_prefs(backend->ctx, + default_sig_algs, + DEFAULT_SIG_ALGS_LENGTH)) { + failf(data, "failed setting signature hash algorithms list: '%s'", + sig_hash_algs); + return CURLE_SSL_CIPHER; + } + } + } +#endif + #ifdef USE_OPENSSL_SRP if(ssl_config->primary.username && Curl_auth_allowed_to_host(data)) { char * const ssl_username = ssl_config->primary.username; @@ -3681,6 +3987,30 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, } #endif + /* curl-impersonate: + * Configure BoringSSL to behave like Chrome. + * See Constructor of SSLContext at net/socket/ssl_client_socket_impl.cc + * and SSLClientSocketImpl::Init() + * in the Chromium's source code. */ + + /* Enable TLS GREASE. */ + SSL_CTX_set_grease_enabled(backend->ctx, 1); + + /* + * curl-impersonate: Enable TLS extension permutation, enabled by default + * since Chrome 110. + */ + if(data->set.ssl_permute_extensions) { + SSL_CTX_set_permute_extensions(backend->ctx, 1); + } + + if(conn_config->cert_compression && + add_cert_compression(data, + backend->ctx, + conn_config->cert_compression)) + return CURLE_SSL_CIPHER; + + /* OpenSSL always tries to verify the peer, this only says whether it should * fail to connect if the verification fails, or if it should continue * anyway. In the latter case the result of the verification is checked with @@ -3727,6 +4057,23 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, SSL_set_app_data(backend->handle, cf); +#ifdef HAS_ALPN + if(connssl->alps) { + size_t i; + struct alpn_proto_buf proto; + + for(i = 0; i < connssl->alps->count; ++i) { + /* curl-impersonate: Add the ALPS extension (17513) like Chrome does. */ + SSL_add_application_settings(backend->handle, connssl->alps->entries[i], + strlen(connssl->alps->entries[i]), NULL, + 0); + } + + Curl_alpn_to_proto_str(&proto, connssl->alps); + infof(data, VTLS_INFOF_ALPS_OFFER_1STR, proto.data); + } +#endif + #if (OPENSSL_VERSION_NUMBER >= 0x0090808fL) && !defined(OPENSSL_NO_TLSEXT) && \ !defined(OPENSSL_NO_OCSP) if(conn_config->verifystatus) diff --git a/lib/vtls/vtls.c b/lib/vtls/vtls.c index 32334016b..1a8a75ade 100644 --- a/lib/vtls/vtls.c +++ b/lib/vtls/vtls.c @@ -141,6 +141,9 @@ static const struct alpn_spec ALPN_SPEC_H11 = { static const struct alpn_spec ALPN_SPEC_H2_H11 = { { ALPN_H2, ALPN_HTTP_1_1 }, 2 }; +static const struct alpn_spec ALPN_SPEC_H2 = { + { ALPN_H2 }, 1 +}; #endif static const struct alpn_spec *alpn_get_spec(int httpwant, bool use_alpn) @@ -155,6 +158,17 @@ static const struct alpn_spec *alpn_get_spec(int httpwant, bool use_alpn) #endif return &ALPN_SPEC_H11; } + +static const struct alpn_spec *alps_get_spec(int httpwant, bool use_alps) +{ + if(!use_alps) + return NULL; +#ifdef USE_HTTP2 + if(httpwant >= CURL_HTTP_VERSION_2) + return &ALPN_SPEC_H2; +#endif + return NULL; +} #endif /* USE_SSL */ @@ -182,6 +196,8 @@ Curl_ssl_config_matches(struct ssl_primary_config *data, strcasecompare(data->cipher_list, needle->cipher_list) && strcasecompare(data->cipher_list13, needle->cipher_list13) && strcasecompare(data->curves, needle->curves) && + strcasecompare(data->sig_hash_algs, needle->sig_hash_algs) && + strcasecompare(data->cert_compression, needle->cert_compression) && strcasecompare(data->CRLfile, needle->CRLfile) && strcasecompare(data->pinned_key, needle->pinned_key)) return TRUE; @@ -212,6 +228,8 @@ Curl_clone_primary_ssl_config(struct ssl_primary_config *source, CLONE_STRING(cipher_list13); CLONE_STRING(pinned_key); CLONE_STRING(curves); + CLONE_STRING(sig_hash_algs); + CLONE_STRING(cert_compression); CLONE_STRING(CRLfile); #ifdef USE_TLS_SRP CLONE_STRING(username); @@ -234,6 +252,8 @@ void Curl_free_primary_ssl_config(struct ssl_primary_config *sslc) Curl_safefree(sslc->ca_info_blob); Curl_safefree(sslc->issuercert_blob); Curl_safefree(sslc->curves); + Curl_safefree(sslc->sig_hash_algs); + Curl_safefree(sslc->cert_compression); Curl_safefree(sslc->CRLfile); #ifdef USE_TLS_SRP Curl_safefree(sslc->username); @@ -318,7 +338,8 @@ static bool ssl_prefs_check(struct Curl_easy *data) } static struct ssl_connect_data *cf_ctx_new(struct Curl_easy *data, - const struct alpn_spec *alpn) + const struct alpn_spec *alpn, + const struct alpn_spec *alps) { struct ssl_connect_data *ctx; @@ -328,6 +349,7 @@ static struct ssl_connect_data *cf_ctx_new(struct Curl_easy *data, return NULL; ctx->alpn = alpn; + ctx->alps = alps; ctx->backend = calloc(1, Curl_ssl->sizeof_ssl_backend_data); if(!ctx->backend) { free(ctx); @@ -1760,8 +1782,11 @@ static CURLcode cf_ssl_create(struct Curl_cfilter **pcf, DEBUGASSERT(data->conn); - ctx = cf_ctx_new(data, alpn_get_spec(data->state.httpwant, - conn->bits.tls_enable_alpn)); + ctx = cf_ctx_new(data, + alpn_get_spec(data->state.httpwant, + conn->bits.tls_enable_alpn), + alps_get_spec(data->state.httpwant, + conn->bits.tls_enable_alps)); if(!ctx) { result = CURLE_OUT_OF_MEMORY; goto out; @@ -1811,6 +1836,7 @@ static CURLcode cf_ssl_proxy_create(struct Curl_cfilter **pcf, struct ssl_connect_data *ctx; CURLcode result; bool use_alpn = conn->bits.tls_enable_alpn; + bool use_alps = conn->bits.tls_enable_alps; int httpwant = CURL_HTTP_VERSION_1_1; #ifdef USE_HTTP2 @@ -1820,7 +1846,8 @@ static CURLcode cf_ssl_proxy_create(struct Curl_cfilter **pcf, } #endif - ctx = cf_ctx_new(data, alpn_get_spec(httpwant, use_alpn)); + ctx = cf_ctx_new(data, alpn_get_spec(httpwant, use_alpn), + alps_get_spec(httpwant, use_alps)); if(!ctx) { result = CURLE_OUT_OF_MEMORY; goto out; diff --git a/lib/vtls/vtls.h b/lib/vtls/vtls.h index f24dca15b..595d437f9 100644 --- a/lib/vtls/vtls.h +++ b/lib/vtls/vtls.h @@ -44,6 +44,8 @@ struct Curl_ssl_session; "ALPN: server did not agree on a protocol. Uses default." #define VTLS_INFOF_ALPN_OFFER_1STR \ "ALPN: offers %s" +#define VTLS_INFOF_ALPS_OFFER_1STR \ + "ALPS: offers %s" #define VTLS_INFOF_ALPN_ACCEPTED_1STR \ ALPN_ACCEPTED "%s" #define VTLS_INFOF_ALPN_ACCEPTED_LEN_1STR \ diff --git a/lib/vtls/vtls_int.h b/lib/vtls/vtls_int.h index ed49339e4..9fddc7494 100644 --- a/lib/vtls/vtls_int.h +++ b/lib/vtls/vtls_int.h @@ -73,6 +73,7 @@ struct ssl_connect_data { char *hostname; /* hostname for verification */ char *dispname; /* display version of hostname */ const struct alpn_spec *alpn; /* ALPN to use or NULL for none */ + const struct alpn_spec *alps; /* ALPS to use or NULL for none */ struct ssl_backend_data *backend; /* vtls backend specific props */ struct cf_call_data call_data; /* data handle used in current call */ struct curltime handshake_done; /* time when handshake finished */ diff --git a/libcurl.pc.in b/libcurl.pc.in index 9db6b0f89..14c2f23e0 100644 --- a/libcurl.pc.in +++ b/libcurl.pc.in @@ -36,6 +36,6 @@ Name: libcurl URL: https://curl.se/ Description: Library to transfer files with ftp, http, etc. Version: @CURLVERSION@ -Libs: -L${libdir} -lcurl @LIBCURL_NO_SHARED@ +Libs: -L${libdir} -lcurl-impersonate-chrome @LIBCURL_NO_SHARED@ Libs.private: @LIBCURL_LIBS@ Cflags: -I${includedir} @CPPFLAG_CURL_STATICLIB@ diff --git a/m4/curl-compilers.m4 b/m4/curl-compilers.m4 index caa2b14cb..0a0af4361 100644 --- a/m4/curl-compilers.m4 +++ b/m4/curl-compilers.m4 @@ -373,42 +373,55 @@ AC_DEFUN([CURL_CONVERT_INCLUDE_TO_ISYSTEM], [ AC_REQUIRE([CURL_SHFUNC_SQUEEZE])dnl AC_REQUIRE([CURL_CHECK_COMPILER])dnl AC_MSG_CHECKING([convert -I options to -isystem]) - if test "$compiler_id" = "GNU_C" || - test "$compiler_id" = "CLANG"; then - AC_MSG_RESULT([yes]) - tmp_has_include="no" - tmp_chg_FLAGS="$CFLAGS" - for word1 in $tmp_chg_FLAGS; do - case "$word1" in - -I*) - tmp_has_include="yes" - ;; - esac - done - if test "$tmp_has_include" = "yes"; then - tmp_chg_FLAGS=`echo "$tmp_chg_FLAGS" | "$SED" 's/^-I/ -isystem /g'` - tmp_chg_FLAGS=`echo "$tmp_chg_FLAGS" | "$SED" 's/ -I/ -isystem /g'` - CFLAGS="$tmp_chg_FLAGS" - squeeze CFLAGS - fi - tmp_has_include="no" - tmp_chg_FLAGS="$CPPFLAGS" - for word1 in $tmp_chg_FLAGS; do - case "$word1" in - -I*) - tmp_has_include="yes" - ;; - esac - done - if test "$tmp_has_include" = "yes"; then - tmp_chg_FLAGS=`echo "$tmp_chg_FLAGS" | "$SED" 's/^-I/ -isystem /g'` - tmp_chg_FLAGS=`echo "$tmp_chg_FLAGS" | "$SED" 's/ -I/ -isystem /g'` - CPPFLAGS="$tmp_chg_FLAGS" - squeeze CPPFLAGS - fi - else + case $host_os in + darwin*) + dnl curl-impersonate: On macos, clang gives priority to /usr/local/include + dnl over locations specified with -isystem for some unknown reason. In turn + dnl this causes clang to use the system's openssl, which conflicts with + dnl curl-impersonate's boringssl headers. + dnl To prevent that, disable curl's automatic conversion of -I flags to + dnl -isystem. AC_MSG_RESULT([no]) - fi + ;; + *) + if test "$compiler_id" = "GNU_C" || + test "$compiler_id" = "CLANG"; then + AC_MSG_RESULT([yes]) + tmp_has_include="no" + tmp_chg_FLAGS="$CFLAGS" + for word1 in $tmp_chg_FLAGS; do + case "$word1" in + -I*) + tmp_has_include="yes" + ;; + esac + done + if test "$tmp_has_include" = "yes"; then + tmp_chg_FLAGS=`echo "$tmp_chg_FLAGS" | "$SED" 's/^-I/ -isystem /g'` + tmp_chg_FLAGS=`echo "$tmp_chg_FLAGS" | "$SED" 's/ -I/ -isystem /g'` + CFLAGS="$tmp_chg_FLAGS" + squeeze CFLAGS + fi + tmp_has_include="no" + tmp_chg_FLAGS="$CPPFLAGS" + for word1 in $tmp_chg_FLAGS; do + case "$word1" in + -I*) + tmp_has_include="yes" + ;; + esac + done + if test "$tmp_has_include" = "yes"; then + tmp_chg_FLAGS=`echo "$tmp_chg_FLAGS" | "$SED" 's/^-I/ -isystem /g'` + tmp_chg_FLAGS=`echo "$tmp_chg_FLAGS" | "$SED" 's/ -I/ -isystem /g'` + CPPFLAGS="$tmp_chg_FLAGS" + squeeze CPPFLAGS + fi + else + AC_MSG_RESULT([no]) + fi + ;; + esac ]) diff --git a/src/Makefile.am b/src/Makefile.am index f24cb6924..30b4fdb0a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -43,7 +43,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/include \ -I$(top_srcdir)/lib \ -I$(top_srcdir)/src -bin_PROGRAMS = curl +bin_PROGRAMS = curl-impersonate-chrome SUBDIRS = ../docs @@ -54,9 +54,9 @@ endif include Makefile.inc # CURL_FILES comes from Makefile.inc -curl_SOURCES = $(CURL_FILES) +curl_impersonate_chrome_SOURCES = $(CURL_FILES) if HAVE_WINDRES -curl_SOURCES += $(CURL_RCFILES) +curl_impersonate_chrome_SOURCES += $(CURL_RCFILES) $(CURL_RCFILES): tool_version.h endif @@ -67,9 +67,9 @@ CFLAGS += @CURL_CFLAG_EXTRAS@ LIBS = $(BLANK_AT_MAKETIME) if USE_EXPLICIT_LIB_DEPS -curl_LDADD = $(top_builddir)/lib/libcurl.la @LIBCURL_LIBS@ +curl_impersonate_chrome_LDADD = $(top_builddir)/lib/libcurl-impersonate-chrome.la @LIBCURL_LIBS@ else -curl_LDADD = $(top_builddir)/lib/libcurl.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@ +curl_impersonate_chrome_LDADD = $(top_builddir)/lib/libcurl-impersonate-chrome.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@ endif # if unit tests are enabled, build a static library to link them with diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h index 9a15659bc..7bf73bd3b 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -160,8 +160,12 @@ struct OperationConfig { bool crlf; char *customrequest; char *ssl_ec_curves; + char *ssl_sig_hash_algs; + char *ssl_cert_compression; char *krblevel; char *request_target; + char *http2_pseudo_headers_order; + bool http2_no_server_push; long httpversion; bool http09_allowed; bool nobuffer; @@ -191,6 +195,7 @@ struct OperationConfig { struct curl_slist *prequote; long ssl_version; long ssl_version_max; + bool ssl_permute_extensions; long proxy_ssl_version; long ip_version; long create_file_mode; /* CURLOPT_NEW_FILE_PERMS */ @@ -266,6 +271,8 @@ struct OperationConfig { bool proxy_ssl_auto_client_cert; /* proxy version of ssl_auto_client_cert */ char *oauth_bearer; /* OAuth 2.0 bearer token */ bool noalpn; /* enable/disable TLS ALPN extension */ + bool alps; /* enable/disable TLS ALPS extension */ + bool noticket; /* enable/disable TLS session ticket */ char *unix_socket_path; /* path to Unix domain socket */ bool abstract_unix_socket; /* path to an abstract Unix domain socket */ bool falsestart; diff --git a/src/tool_getparam.c b/src/tool_getparam.c index c9810e9d4..0be823c2b 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c @@ -287,6 +287,13 @@ static const struct LongShort aliases[]= { {"EC", "etag-save", ARG_FILENAME}, {"ED", "etag-compare", ARG_FILENAME}, {"EE", "curves", ARG_STRING}, + {"EG", "signature-hashes", ARG_STRING}, + {"EH", "alps", ARG_BOOL}, + {"EI", "cert-compression", ARG_STRING}, + {"EJ", "tls-session-ticket", ARG_BOOL}, + {"EK", "http2-pseudo-headers-order", ARG_STRING}, + {"EL", "http2-no-server-push", ARG_BOOL}, + {"EM", "tls-permute-extensions", ARG_BOOL}, {"f", "fail", ARG_BOOL}, {"fa", "fail-early", ARG_BOOL}, {"fb", "styled-output", ARG_BOOL}, @@ -1940,6 +1947,39 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ GetStr(&config->ssl_ec_curves, nextarg); break; + case 'G': + /* --signature-hashes */ + GetStr(&config->ssl_sig_hash_algs, nextarg); + break; + + case 'H': + /* --alps */ + config->alps = toggle; + break; + + case 'I': + /* --cert-compression */ + GetStr(&config->ssl_cert_compression, nextarg); + break; + + case 'J': + /* --tls-session-ticket */ + config->noticket = (!toggle)?TRUE:FALSE; + break; + + case 'K': + /* --http2-pseudo-headers-order */ + GetStr(&config->http2_pseudo_headers_order, nextarg); + break; + + case 'L': + /* --http2-no-server-push */ + config->http2_no_server_push = toggle; + break; + case 'M': + /* --tls-permute-extensions */ + config->ssl_permute_extensions = toggle; + break; default: /* unknown flag */ return PARAM_OPTION_UNKNOWN; } diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c index 61550de72..63ea135b3 100644 --- a/src/tool_listhelp.c +++ b/src/tool_listhelp.c @@ -108,6 +108,24 @@ const struct helptxt helptext[] = { {" --curves ", "(EC) TLS key exchange algorithm(s) to request", CURLHELP_TLS}, + {" --signature-hashes ", + "TLS signature hash algorithm(s) to use", + CURLHELP_TLS}, + {" --cert-compression ", + "TLS cert compressions algorithm(s) to use", + CURLHELP_TLS}, + {" --no-tls-session-ticket", + "Disable the TLS session ticket extension", + CURLHELP_TLS}, + {" --http2-pseudo-headers-order", + "Change the order of the HTTP2 pseudo headers", + CURLHELP_HTTP}, + {" --http2-no-server-push", + "Send HTTP2 setting to disable server push", + CURLHELP_HTTP}, + {" --tls-permute-extensions", + "Enable BoringSSL TLS extensions permutations on client hello", + CURLHELP_TLS}, {"-d, --data ", "HTTP POST data", CURLHELP_IMPORTANT | CURLHELP_HTTP | CURLHELP_POST | CURLHELP_UPLOAD}, @@ -387,6 +405,9 @@ const struct helptxt helptext[] = { {" --no-alpn", "Disable the ALPN TLS extension", CURLHELP_TLS | CURLHELP_HTTP}, + {" --alps", + "Enable the ALPS TLS extension", + CURLHELP_TLS | CURLHELP_HTTP}, {"-N, --no-buffer", "Disable buffering of the output stream", CURLHELP_CURL}, diff --git a/src/tool_operate.c b/src/tool_operate.c index ead7dca63..dc9f435f5 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -1493,6 +1493,15 @@ static CURLcode single_transfer(struct GlobalConfig *global, return result; } + if(config->http2_pseudo_headers_order) + my_setopt_str(curl, + CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER, + config->http2_pseudo_headers_order); + + if(config->http2_no_server_push) + my_setopt(curl, CURLOPT_HTTP2_NO_SERVER_PUSH, + config->http2_no_server_push ? 1L : 0L); + } /* (proto_http) */ if(proto_ftp) @@ -1581,6 +1590,14 @@ static CURLcode single_transfer(struct GlobalConfig *global, if(config->ssl_ec_curves) my_setopt_str(curl, CURLOPT_SSL_EC_CURVES, config->ssl_ec_curves); + if(config->ssl_sig_hash_algs) + my_setopt_str(curl, CURLOPT_SSL_SIG_HASH_ALGS, + config->ssl_sig_hash_algs); + + if(config->ssl_cert_compression) + my_setopt_str(curl, CURLOPT_SSL_CERT_COMPRESSION, + config->ssl_cert_compression); + if(config->writeout) my_setopt_str(curl, CURLOPT_CERTINFO, 1L); @@ -1914,6 +1931,10 @@ static CURLcode single_transfer(struct GlobalConfig *global, my_setopt_str(curl, CURLOPT_PROXY_TLS13_CIPHERS, config->proxy_cipher13_list); + /* curl-impersonate */ + if(config->ssl_permute_extensions) + my_setopt(curl, CURLOPT_SSL_PERMUTE_EXTENSIONS, 1L); + /* new in libcurl 7.9.2: */ if(config->disable_epsv) /* disable it */ @@ -2123,6 +2144,14 @@ static CURLcode single_transfer(struct GlobalConfig *global, my_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, 0L); } + if(config->alps) { + my_setopt(curl, CURLOPT_SSL_ENABLE_ALPS, 1L); + } + + if (config->noticket) { + my_setopt(curl, CURLOPT_SSL_ENABLE_TICKET, 0L); + } + /* new in 7.40.0, abstract support added in 7.53.0 */ if(config->unix_socket_path) { if(config->abstract_unix_socket) { diff --git a/src/tool_setopt.c b/src/tool_setopt.c index 0f3cc83b9..1c14e9854 100644 --- a/src/tool_setopt.c +++ b/src/tool_setopt.c @@ -153,6 +153,8 @@ static const struct NameValue setopt_nv_CURLNONZERODEFAULTS[] = { NV1(CURLOPT_SSL_VERIFYHOST, 1), NV1(CURLOPT_SSL_ENABLE_NPN, 1), NV1(CURLOPT_SSL_ENABLE_ALPN, 1), + NV1(CURLOPT_SSL_ENABLE_TICKET, 1), + NV1(CURLOPT_SSL_PERMUTE_EXTENSIONS, 1), NV1(CURLOPT_TCP_NODELAY, 1), NV1(CURLOPT_PROXY_SSL_VERIFYPEER, 1), NV1(CURLOPT_PROXY_SSL_VERIFYHOST, 1),