hoelee/curl-impersonate
1
0
Fork 0
mirror of https://github.com/lwthiker/curl-impersonate.git synced 2025-08-08 04:42:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
Files
14de5bb0dbe5c1800c51f515736c38036168e2ae
curl-impersonate/chrome/patches/curl-impersonate.patch
lwthiker 14de5bb0db Add impersonation support for Chrome 100 and 101 
.. and for Edge 101 as well. The TLS fingerprint is identical to
previous versions. The HTTP headers have the usual differences in the
user agents. One important change though is in the way the HTTP2
SETTINGS frame is formed. Up until Chrome 98, there was an additional
randomly-generated setting in the frame. This seems to have been removed
since. Therefore it was removed from curl-impersonate as well, and
support for Chrome/Edge 98 was deprecated, since supporting both
signatures requires a lot of work.
2022-05-17 20:53:39 +03:00

2089 lines
72 KiB
Diff
Raw Blame History

diff --git a/Makefile.am b/Makefile.am
index 3e55230ee..e3ea22b96 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -192,13 +192,13 @@ CLEANFILES = $(VC6_LIBDSP) $(VC6_SRCDSP) $(VC7_LIBVCPROJ) $(VC7_SRCVCPROJ) \
$(VC11_LIBVCXPROJ) $(VC11_SRCVCXPROJ) $(VC12_LIBVCXPROJ) $(VC12_SRCVCXPROJ) \
$(VC14_LIBVCXPROJ) $(VC14_SRCVCXPROJ) $(VC15_LIBVCXPROJ) $(VC15_SRCVCXPROJ)
-bin_SCRIPTS = curl-config
+bin_SCRIPTS = curl-impersonate-chrome-config
SUBDIRS = lib src
DIST_SUBDIRS = $(SUBDIRS) tests packages scripts include docs
pkgconfigdir = $(libdir)/pkgconfig
-pkgconfig_DATA = libcurl.pc
+pkgconfig_DATA = libcurl-impersonate-chrome.pc
# List of files required to generate VC IDE .dsp, .vcproj and .vcxproj files
include lib/Makefile.inc
diff --git a/configure.ac b/configure.ac
index 63e320236..86ab654c7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1331,7 +1331,8 @@ if test X"$OPT_BROTLI" != Xno; then
dnl if given with a prefix, we set -L and -I based on that
if test -n "$PREFIX_BROTLI"; then
- LIB_BROTLI="-lbrotlidec"
+ # curl-impersonate: Use static libbrotli
+ LIB_BROTLI="-lbrotlidec-static -lbrotlicommon-static"
LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff
CPP_BROTLI=-I${PREFIX_BROTLI}/include
DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff
@@ -1341,7 +1342,11 @@ if test X"$OPT_BROTLI" != Xno; then
CPPFLAGS="$CPPFLAGS $CPP_BROTLI"
LIBS="$LIB_BROTLI $LIBS"
- AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress)
+ AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress,
+ # curl-impersonate: Define 'action-if-found' explicitly to prevent
+ # -lbrotlidec from being added to LIBS (already added before)
+ AC_DEFINE(HAVE_LIBBROTLI, 1, [Define to 1 if libbrotli exists])
+ )
AC_CHECK_HEADERS(brotli/decode.h,
curl_brotli_msg="enabled (libbrotlidec)"
@@ -2573,15 +2578,15 @@ if test X"$want_nghttp2" != Xno; then
if test "$PKGCONFIG" != "no" ; then
LIB_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path])
- $PKGCONFIG --libs-only-l libnghttp2`
+ $PKGCONFIG --static --libs-only-l libnghttp2`
AC_MSG_NOTICE([-l is $LIB_H2])
CPP_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path]) dnl
- $PKGCONFIG --cflags-only-I libnghttp2`
+ $PKGCONFIG --static --cflags-only-I libnghttp2`
AC_MSG_NOTICE([-I is $CPP_H2])
LD_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path])
- $PKGCONFIG --libs-only-L libnghttp2`
+ $PKGCONFIG --static --libs-only-L libnghttp2`
AC_MSG_NOTICE([-L is $LD_H2])
LDFLAGS="$LDFLAGS $LD_H2"
@@ -4255,8 +4260,8 @@ AC_CONFIG_FILES([Makefile \
tests/unit/Makefile \
packages/Makefile \
packages/vms/Makefile \
- curl-config \
- libcurl.pc
+ curl-impersonate-chrome-config:curl-config.in \
+ libcurl-impersonate-chrome.pc:libcurl.pc.in
])
AC_OUTPUT
diff --git a/curl-config.in b/curl-config.in
index 8b4a29a9a..3d6399251 100644
--- a/curl-config.in
+++ b/curl-config.in
@@ -161,9 +161,9 @@ while test $# -gt 0; do
CURLLIBDIR=""
fi
if test "X@ENABLE_SHARED@" = "Xno"; then
- echo ${CURLLIBDIR}-lcurl @LIBCURL_LIBS@
+ echo ${CURLLIBDIR}-lcurl-impersonate-chrome @LIBCURL_LIBS@
else
- echo ${CURLLIBDIR}-lcurl
+ echo ${CURLLIBDIR}-lcurl-impersonate-chrome
fi
;;
--ssl-backends)
@@ -172,7 +172,7 @@ while test $# -gt 0; do
--static-libs)
if test "X@ENABLE_STATIC@" != "Xno" ; then
- echo @libdir@/libcurl.@libext@ @LDFLAGS@ @LIBCURL_LIBS@
+ echo @libdir@/libcurl-impersonate-chrome.@libext@ @LDFLAGS@ @LIBCURL_LIBS@
else
echo "curl was built with static libraries disabled" >&2
exit 1
diff --git a/include/curl/curl.h b/include/curl/curl.h
index 7b69ce2d6..a62c8a4a9 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -2135,6 +2135,38 @@ typedef enum {
/* Set MIME option flags. */
CURLOPT(CURLOPT_MIME_OPTIONS, CURLOPTTYPE_LONG, 315),
+ /* curl-impersonate: A list of headers used by the impersonated browser.
+ * If given, merged with CURLOPT_HTTPHEADER. */
+ CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 316),
+
+ /* curl-impersonate: A list of TLS signature hash algorithms.
+ * See https://datatracker.ietf.org/doc/html/rfc5246#section-7.4.1.4.1 */
+ CURLOPT(CURLOPT_SSL_SIG_HASH_ALGS, CURLOPTTYPE_STRINGPOINT, 317),
+
+ /* curl-impersonate: Whether to enable ALPS in TLS or not.
+ * See https://datatracker.ietf.org/doc/html/draft-vvv-tls-alps.
+ * Support for ALPS is minimal and is intended only for the TLS client
+ * hello to match. */
+ CURLOPT(CURLOPT_SSL_ENABLE_ALPS, CURLOPTTYPE_LONG, 318),
+
+ /* curl-impersonate: Comma-separated list of certificate compression
+ * algorithms to use. These are published in the client hello.
+ * Supported algorithms are "zlib" and "brotli".
+ * See https://datatracker.ietf.org/doc/html/rfc8879 */
+ CURLOPT(CURLOPT_SSL_CERT_COMPRESSION, CURLOPTTYPE_STRINGPOINT, 319),
+
+ /* Enable/disable TLS session ticket extension (RFC5077) */
+ CURLOPT(CURLOPT_SSL_ENABLE_TICKET, CURLOPTTYPE_LONG, 320),
+
+ /*
+ * curl-impersonate:
+ * Set the order of the HTTP/2 pseudo headers. The value must contain
+ * the letters 'm', 'a', 's', 'p' representing the pseudo-headers
+ * ":method", ":authority", ":scheme", ":path" in the desired order of
+ * appearance in the HTTP/2 HEADERS frame.
+ */
+ CURLOPT(CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER, CURLOPTTYPE_STRINGPOINT, 321),
+
CURLOPT_LASTENTRY /* the last unused */
} CURLoption;
diff --git a/include/curl/easy.h b/include/curl/easy.h
index 2dbfb26b5..e0bf86169 100644
--- a/include/curl/easy.h
+++ b/include/curl/easy.h
@@ -41,6 +41,15 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...);
CURL_EXTERN CURLcode curl_easy_perform(CURL *curl);
CURL_EXTERN void curl_easy_cleanup(CURL *curl);
+/*
+ * curl-impersonate: Tell libcurl to impersonate a browser.
+ * This is a wrapper function that calls curl_easy_setopt()
+ * multiple times with all the parameters required. That's also why it was
+ * created as a separate API function and not just as another option to
+ * curl_easy_setopt().
+ */
+CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target);
+
/*
* NAME curl_easy_getinfo()
*
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 769363941..cd59ad4b2 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -32,7 +32,7 @@ EXTRA_DIST = Makefile.m32 config-win32.h config-win32ce.h \
firefox-db2pem.sh config-vxworks.h Makefile.vxworks checksrc.pl \
setup-win32.h .checksrc
-lib_LTLIBRARIES = libcurl.la
+lib_LTLIBRARIES = libcurl-impersonate-chrome.la
if BUILD_UNITTESTS
noinst_LTLIBRARIES = libcurlu.la
@@ -84,43 +84,43 @@ AM_CPPFLAGS += -DBUILDING_LIBCURL
AM_LDFLAGS =
AM_CFLAGS =
-libcurl_la_CPPFLAGS_EXTRA =
-libcurl_la_LDFLAGS_EXTRA =
-libcurl_la_CFLAGS_EXTRA =
+libcurl_impersonate_chrome_la_CPPFLAGS_EXTRA =
+libcurl_impersonate_chrome_la_LDFLAGS_EXTRA =
+libcurl_impersonate_chrome_la_CFLAGS_EXTRA =
if CURL_LT_SHLIB_USE_VERSION_INFO
-libcurl_la_LDFLAGS_EXTRA += $(VERSIONINFO)
+libcurl_impersonate_chrome_la_LDFLAGS_EXTRA += $(VERSIONINFO)
endif
if CURL_LT_SHLIB_USE_NO_UNDEFINED
-libcurl_la_LDFLAGS_EXTRA += -no-undefined
+libcurl_impersonate_chrome_la_LDFLAGS_EXTRA += -no-undefined
endif
if CURL_LT_SHLIB_USE_MIMPURE_TEXT
-libcurl_la_LDFLAGS_EXTRA += -mimpure-text
+libcurl_impersonate_chrome_la_LDFLAGS_EXTRA += -mimpure-text
endif
if CURL_LT_SHLIB_USE_VERSIONED_SYMBOLS
-libcurl_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers
+libcurl_impersonate_chrome_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers
else
# if symbol-hiding is enabled, hide them!
if DOING_CURL_SYMBOL_HIDING
-libcurl_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*'
+libcurl_impersonate_chrome_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*'
endif
endif
if USE_CPPFLAG_CURL_STATICLIB
-libcurl_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB
+libcurl_impersonate_chrome_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB
endif
if DOING_CURL_SYMBOL_HIDING
-libcurl_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS
-libcurl_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING)
+libcurl_impersonate_chrome_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS
+libcurl_impersonate_chrome_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING)
endif
-libcurl_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_la_CPPFLAGS_EXTRA)
-libcurl_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS)
-libcurl_la_CFLAGS = $(AM_CFLAGS) $(libcurl_la_CFLAGS_EXTRA)
+libcurl_impersonate_chrome_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_impersonate_chrome_la_CPPFLAGS_EXTRA)
+libcurl_impersonate_chrome_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_impersonate_chrome_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS)
+libcurl_impersonate_chrome_la_CFLAGS = $(AM_CFLAGS) $(libcurl_impersonate_chrome_la_CFLAGS_EXTRA)
libcurlu_la_CPPFLAGS = $(AM_CPPFLAGS) -DCURL_STATICLIB -DUNITTESTS
libcurlu_la_LDFLAGS = $(AM_LDFLAGS) -static $(LIBCURL_LIBS)
@@ -129,7 +129,7 @@ libcurlu_la_CFLAGS = $(AM_CFLAGS)
# Makefile.inc provides the CSOURCES and HHEADERS defines
include Makefile.inc
-libcurl_la_SOURCES = $(CSOURCES) $(HHEADERS)
+libcurl_impersonate_chrome_la_SOURCES = $(CSOURCES) $(HHEADERS)
libcurlu_la_SOURCES = $(CSOURCES) $(HHEADERS)
CHECKSRC = $(CS_$(V))
diff --git a/lib/easy.c b/lib/easy.c
index 20293a710..37cda0992 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -80,6 +80,7 @@
#include "dynbuf.h"
#include "altsvc.h"
#include "hsts.h"
+#include "strcase.h"
/* The last 3 #include files should be in this order */
#include "curl_printf.h"
@@ -282,6 +283,454 @@ void curl_global_cleanup(void)
init_flags = 0;
}
+/*
+ * curl-impersonate: Options to be set for each supported target browser.
+ * Note: this does not include the HTTP headers, which are handled separately
+ * in Curl_http().
+ */
+#define IMPERSONATE_MAX_HEADERS 32
+static const struct impersonate_opts {
+ const char *target;
+ int httpversion;
+ int ssl_version;
+ const char *ciphers;
+ /* Elliptic curves (TLS extension 10).
+ * Passed to CURLOPT_SSL_EC_CURVES */
+ const char *curves;
+ /* Signature hash algorithms (TLS extension 13).
+ * Passed to CURLOPT_SSL_SIG_HASH_ALGS */
+ const char *sig_hash_algs;
+ /* Enable TLS NPN extension. */
+ bool npn;
+ /* Enable TLS ALPN extension. */
+ bool alpn;
+ /* Enable TLS ALPS extension. */
+ bool alps;
+ /* Enable TLS session ticket extension. */
+ bool tls_session_ticket;
+ /* TLS certificate compression algorithms.
+ * (TLS extension 27) */
+ const char *cert_compression;
+ const char *http_headers[IMPERSONATE_MAX_HEADERS];
+ const char *http2_pseudo_headers_order;
+ /* Other TLS options will come here in the future once they are
+ * configurable through curl_easy_setopt() */
+} impersonations[] = {
+ {
+ .target = "chrome99",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "TLS_AES_128_GCM_SHA256,"
+ "TLS_AES_256_GCM_SHA384,"
+ "TLS_CHACHA20_POLY1305_SHA256,"
+ "ECDHE-ECDSA-AES128-GCM-SHA256,"
+ "ECDHE-RSA-AES128-GCM-SHA256,"
+ "ECDHE-ECDSA-AES256-GCM-SHA384,"
+ "ECDHE-RSA-AES256-GCM-SHA384,"
+ "ECDHE-ECDSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-AES128-SHA,"
+ "ECDHE-RSA-AES256-SHA,"
+ "AES128-GCM-SHA256,"
+ "AES256-GCM-SHA384,"
+ "AES128-SHA,"
+ "AES256-SHA",
+ .npn = false,
+ .alpn = true,
+ .alps = true,
+ .tls_session_ticket = true,
+ .cert_compression = "brotli",
+ .http_headers = {
+ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"99\", \"Google Chrome\";v=\"99\"",
+ "sec-ch-ua-mobile: ?0",
+ "sec-ch-ua-platform: \"Windows\"",
+ "Upgrade-Insecure-Requests: 1",
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-User: ?1",
+ "Sec-Fetch-Dest: document",
+ "Accept-Encoding: gzip, deflate, br",
+ "Accept-Language: en-US,en;q=0.9"
+ }
+ },
+ {
+ .target = "chrome100",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "TLS_AES_128_GCM_SHA256,"
+ "TLS_AES_256_GCM_SHA384,"
+ "TLS_CHACHA20_POLY1305_SHA256,"
+ "ECDHE-ECDSA-AES128-GCM-SHA256,"
+ "ECDHE-RSA-AES128-GCM-SHA256,"
+ "ECDHE-ECDSA-AES256-GCM-SHA384,"
+ "ECDHE-RSA-AES256-GCM-SHA384,"
+ "ECDHE-ECDSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-AES128-SHA,"
+ "ECDHE-RSA-AES256-SHA,"
+ "AES128-GCM-SHA256,"
+ "AES256-GCM-SHA384,"
+ "AES128-SHA,"
+ "AES256-SHA",
+ .npn = false,
+ .alpn = true,
+ .alps = true,
+ .tls_session_ticket = true,
+ .cert_compression = "brotli",
+ .http_headers = {
+ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"",
+ "sec-ch-ua-mobile: ?0",
+ "sec-ch-ua-platform: \"Windows\"",
+ "Upgrade-Insecure-Requests: 1",
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-User: ?1",
+ "Sec-Fetch-Dest: document",
+ "Accept-Encoding: gzip, deflate, br",
+ "Accept-Language: en-US,en;q=0.9"
+ }
+ },
+ {
+ .target = "chrome101",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "TLS_AES_128_GCM_SHA256,"
+ "TLS_AES_256_GCM_SHA384,"
+ "TLS_CHACHA20_POLY1305_SHA256,"
+ "ECDHE-ECDSA-AES128-GCM-SHA256,"
+ "ECDHE-RSA-AES128-GCM-SHA256,"
+ "ECDHE-ECDSA-AES256-GCM-SHA384,"
+ "ECDHE-RSA-AES256-GCM-SHA384,"
+ "ECDHE-ECDSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-AES128-SHA,"
+ "ECDHE-RSA-AES256-SHA,"
+ "AES128-GCM-SHA256,"
+ "AES256-GCM-SHA384,"
+ "AES128-SHA,"
+ "AES256-SHA",
+ .npn = false,
+ .alpn = true,
+ .alps = true,
+ .tls_session_ticket = true,
+ .cert_compression = "brotli",
+ .http_headers = {
+ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"101\", \"Google Chrome\";v=\"101\"",
+ "sec-ch-ua-mobile: ?0",
+ "sec-ch-ua-platform: \"Windows\"",
+ "Upgrade-Insecure-Requests: 1",
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-User: ?1",
+ "Sec-Fetch-Dest: document",
+ "Accept-Encoding: gzip, deflate, br",
+ "Accept-Language: en-US,en;q=0.9"
+ }
+ },
+ {
+ .target = "chrome99_android",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "TLS_AES_128_GCM_SHA256,"
+ "TLS_AES_256_GCM_SHA384,"
+ "TLS_CHACHA20_POLY1305_SHA256,"
+ "ECDHE-ECDSA-AES128-GCM-SHA256,"
+ "ECDHE-RSA-AES128-GCM-SHA256,"
+ "ECDHE-ECDSA-AES256-GCM-SHA384,"
+ "ECDHE-RSA-AES256-GCM-SHA384,"
+ "ECDHE-ECDSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-AES128-SHA,"
+ "ECDHE-RSA-AES256-SHA,"
+ "AES128-GCM-SHA256,"
+ "AES256-GCM-SHA384,"
+ "AES128-SHA,"
+ "AES256-SHA",
+ .npn = false,
+ .alpn = true,
+ .alps = true,
+ .tls_session_ticket = true,
+ .cert_compression = "brotli",
+ .http_headers = {
+ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"99\", \"Google Chrome\";v=\"99\"",
+ "sec-ch-ua-mobile: ?1",
+ "sec-ch-ua-platform: \"Android\"",
+ "Upgrade-Insecure-Requests: 1",
+ "User-Agent: Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-User: ?1",
+ "Sec-Fetch-Dest: document",
+ "Accept-Encoding: gzip, deflate, br",
+ "Accept-Language: en-US,en;q=0.9"
+ }
+ },
+ {
+ .target = "edge99",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "TLS_AES_128_GCM_SHA256,"
+ "TLS_AES_256_GCM_SHA384,"
+ "TLS_CHACHA20_POLY1305_SHA256,"
+ "ECDHE-ECDSA-AES128-GCM-SHA256,"
+ "ECDHE-RSA-AES128-GCM-SHA256,"
+ "ECDHE-ECDSA-AES256-GCM-SHA384,"
+ "ECDHE-RSA-AES256-GCM-SHA384,"
+ "ECDHE-ECDSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-AES128-SHA,"
+ "ECDHE-RSA-AES256-SHA,"
+ "AES128-GCM-SHA256,"
+ "AES256-GCM-SHA384,"
+ "AES128-SHA,"
+ "AES256-SHA",
+ .npn = false,
+ .alpn = true,
+ .alps = true,
+ .tls_session_ticket = true,
+ .cert_compression = "brotli",
+ .http_headers = {
+ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"99\", \"Microsoft Edge\";v=\"99\"",
+ "sec-ch-ua-mobile: ?0",
+ "sec-ch-ua-platform: \"Windows\"",
+ "Upgrade-Insecure-Requests: 1",
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36 Edg/99.0.1150.30",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-User: ?1",
+ "Sec-Fetch-Dest: document",
+ "Accept-Encoding: gzip, deflate, br",
+ "Accept-Language: en-US,en;q=0.9"
+ }
+ },
+ {
+ .target = "edge101",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "TLS_AES_128_GCM_SHA256,"
+ "TLS_AES_256_GCM_SHA384,"
+ "TLS_CHACHA20_POLY1305_SHA256,"
+ "ECDHE-ECDSA-AES128-GCM-SHA256,"
+ "ECDHE-RSA-AES128-GCM-SHA256,"
+ "ECDHE-ECDSA-AES256-GCM-SHA384,"
+ "ECDHE-RSA-AES256-GCM-SHA384,"
+ "ECDHE-ECDSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-CHACHA20-POLY1305,"
+ "ECDHE-RSA-AES128-SHA,"
+ "ECDHE-RSA-AES256-SHA,"
+ "AES128-GCM-SHA256,"
+ "AES256-GCM-SHA384,"
+ "AES128-SHA,"
+ "AES256-SHA",
+ .npn = false,
+ .alpn = true,
+ .alps = true,
+ .tls_session_ticket = true,
+ .cert_compression = "brotli",
+ .http_headers = {
+ "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"101\", \"Microsoft Edge\";v=\"101\"",
+ "sec-ch-ua-mobile: ?0",
+ "sec-ch-ua-platform: \"Windows\"",
+ "Upgrade-Insecure-Requests: 1",
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.47",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Sec-Fetch-Site: none",
+ "Sec-Fetch-Mode: navigate",
+ "Sec-Fetch-User: ?1",
+ "Sec-Fetch-Dest: document",
+ "Accept-Encoding: gzip, deflate, br",
+ "Accept-Language: en-US,en;q=0.9"
+ }
+ },
+ {
+ .target = "safari15_3",
+ .httpversion = CURL_HTTP_VERSION_2_0,
+ .ssl_version = CURL_SSLVERSION_TLSv1_0 | CURL_SSLVERSION_MAX_DEFAULT,
+ .ciphers =
+ "TLS_AES_128_GCM_SHA256,"
+ "TLS_AES_256_GCM_SHA384,"
+ "TLS_CHACHA20_POLY1305_SHA256,"
+ "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,"
+ "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,"
+ "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,"
+ "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,"
+ "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,"
+ "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,"
+ "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384,"
+ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,"
+ "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,"
+ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,"
+ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384,"
+ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,"
+ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,"
+ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,"
+ "TLS_RSA_WITH_AES_256_GCM_SHA384,"
+ "TLS_RSA_WITH_AES_128_GCM_SHA256,"
+ "TLS_RSA_WITH_AES_256_CBC_SHA256,"
+ "TLS_RSA_WITH_AES_128_CBC_SHA256,"
+ "TLS_RSA_WITH_AES_256_CBC_SHA,"
+ "TLS_RSA_WITH_AES_128_CBC_SHA,"
+ "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA,"
+ "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,"
+ "TLS_RSA_WITH_3DES_EDE_CBC_SHA,",
+ .curves = "X25519:P-256:P-384:P-521",
+ .sig_hash_algs =
+ "ecdsa_secp256r1_sha256,"
+ "rsa_pss_rsae_sha256,"
+ "rsa_pkcs1_sha256,"
+ "ecdsa_secp384r1_sha384,"
+ "ecdsa_sha1,"
+ "rsa_pss_rsae_sha384,"
+ "rsa_pss_rsae_sha384,"
+ "rsa_pkcs1_sha384,"
+ "rsa_pss_rsae_sha512,"
+ "rsa_pkcs1_sha512,"
+ "rsa_pkcs1_sha1",
+ .npn = false,
+ .alpn = true,
+ .alps = false,
+ .tls_session_ticket = false,
+ .http_headers = {
+ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15",
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language: en-us",
+ "Accept-Encoding: gzip, deflate, br"
+ },
+ .http2_pseudo_headers_order = "mspa"
+ }
+};
+
+#define NUM_IMPERSONATIONS \
+ sizeof(impersonations) / sizeof(impersonations[0])
+
+/*
+ * curl-impersonate:
+ * Call curl_easy_setopt() with all the needed options as defined in the
+ * 'impersonations' array.
+ * */
+CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target)
+{
+ int i;
+ int ret;
+ const struct impersonate_opts *opts = NULL;
+ struct curl_slist *headers = NULL;
+
+ for(i = 0; i < NUM_IMPERSONATIONS; i++) {
+ if (Curl_safe_strcasecompare(target, impersonations[i].target)) {
+ opts = &impersonations[i];
+ break;
+ }
+ }
+
+ if(!opts) {
+ DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n",
+ target));
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+ }
+
+ if(opts->httpversion != CURL_HTTP_VERSION_NONE) {
+ ret = curl_easy_setopt(data, CURLOPT_HTTP_VERSION, opts->httpversion);
+ if(ret)
+ return ret;
+ }
+
+ if (opts->ssl_version != CURL_SSLVERSION_DEFAULT) {
+ ret = curl_easy_setopt(data, CURLOPT_SSLVERSION, opts->ssl_version);
+ if(ret)
+ return ret;
+ }
+
+ if(opts->ciphers) {
+ ret = curl_easy_setopt(data, CURLOPT_SSL_CIPHER_LIST, opts->ciphers);
+ if (ret)
+ return ret;
+ }
+
+ if(opts->curves) {
+ ret = curl_easy_setopt(data, CURLOPT_SSL_EC_CURVES, opts->curves);
+ if(ret)
+ return ret;
+ }
+
+ if(opts->sig_hash_algs) {
+ ret = curl_easy_setopt(data, CURLOPT_SSL_SIG_HASH_ALGS,
+ opts->sig_hash_algs);
+ if(ret)
+ return ret;
+ }
+
+ ret = curl_easy_setopt(data, CURLOPT_SSL_ENABLE_NPN, opts->npn ? 1 : 0);
+ if(ret)
+ return ret;
+
+ ret = curl_easy_setopt(data, CURLOPT_SSL_ENABLE_ALPN, opts->alpn ? 1 : 0);
+ if(ret)
+ return ret;
+
+ ret = curl_easy_setopt(data, CURLOPT_SSL_ENABLE_ALPS, opts->alps ? 1 : 0);
+ if(ret)
+ return ret;
+
+ ret = curl_easy_setopt(data, CURLOPT_SSL_ENABLE_TICKET,
+ opts->tls_session_ticket ? 1 : 0);
+ if(ret)
+ return ret;
+
+ if(opts->cert_compression) {
+ ret = curl_easy_setopt(data,
+ CURLOPT_SSL_CERT_COMPRESSION,
+ opts->cert_compression);
+ if(ret)
+ return ret;
+ }
+
+ /* Build a linked list out of the static array of headers. */
+ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) {
+ if(opts->http_headers[i]) {
+ headers = curl_slist_append(headers, opts->http_headers[i]);
+ if(!headers) {
+ return CURLE_OUT_OF_MEMORY;
+ }
+ }
+ }
+
+ if(headers) {
+ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers);
+ curl_slist_free_all(headers);
+ if(ret)
+ return ret;
+ }
+
+ if(opts->http2_pseudo_headers_order) {
+ ret = curl_easy_setopt(data,
+ CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER,
+ opts->http2_pseudo_headers_order);
+ if(ret)
+ return ret;
+ }
+
+ /* Always enable all supported compressions. */
+ ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, "");
+ if(ret)
+ return ret;
+
+ return CURLE_OK;
+}
+
/*
* curl_easy_init() is the external interface to alloc, setup and init an
* easy handle that is returned. If anything goes wrong, NULL is returned.
@@ -290,6 +739,7 @@ struct Curl_easy *curl_easy_init(void)
{
CURLcode result;
struct Curl_easy *data;
+ char *target;
/* Make sure we inited the global SSL stuff */
if(!initialized) {
@@ -308,6 +758,22 @@ struct Curl_easy *curl_easy_init(void)
return NULL;
}
+ /*
+ * curl-impersonate: Hook into curl_easy_init() to set the required options
+ * from an environment variable.
+ * This is a bit hacky but allows seamless integration of libcurl-impersonate
+ * without code modifications to the app.
+ */
+ target = curl_getenv("CURL_IMPERSONATE");
+ if(target) {
+ result = curl_easy_impersonate(data, target);
+ free(target);
+ if(result) {
+ Curl_close(&data);
+ return NULL;
+ }
+ }
+
return data;
}
@@ -878,6 +1344,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
outcurl->state.referer_alloc = TRUE;
}
+ if(data->state.base_headers) {
+ outcurl->state.base_headers =
+ Curl_slist_duplicate(data->state.base_headers);
+ if(!outcurl->state.base_headers)
+ goto fail;
+ }
+
/* Reinitialize an SSL engine for the new handle
* note: the engine name has already been copied by dupset */
if(outcurl->set.str[STRING_SSL_ENGINE]) {
diff --git a/lib/easyoptions.c b/lib/easyoptions.c
index 04871ad1e..ce280eaa3 100644
--- a/lib/easyoptions.c
+++ b/lib/easyoptions.c
@@ -128,8 +128,11 @@ struct curl_easyoption Curl_easyopts[] = {
{"HSTS_CTRL", CURLOPT_HSTS_CTRL, CURLOT_LONG, 0},
{"HTTP09_ALLOWED", CURLOPT_HTTP09_ALLOWED, CURLOT_LONG, 0},
{"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0},
+ {"HTTP2_PSEUDO_HEADERS_ORDER", CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER,
+ CURLOT_STRING, 0},
{"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0},
{"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0},
+ {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0},
{"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0},
{"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0},
{"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0},
@@ -293,15 +296,19 @@ struct curl_easyoption Curl_easyopts[] = {
{"SSLKEYTYPE", CURLOPT_SSLKEYTYPE, CURLOT_STRING, 0},
{"SSLKEY_BLOB", CURLOPT_SSLKEY_BLOB, CURLOT_BLOB, 0},
{"SSLVERSION", CURLOPT_SSLVERSION, CURLOT_VALUES, 0},
+ {"SSL_CERT_COMPRESSION", CURLOPT_SSL_CERT_COMPRESSION, CURLOT_STRING, 0},
{"SSL_CIPHER_LIST", CURLOPT_SSL_CIPHER_LIST, CURLOT_STRING, 0},
{"SSL_CTX_DATA", CURLOPT_SSL_CTX_DATA, CURLOT_CBPTR, 0},
{"SSL_CTX_FUNCTION", CURLOPT_SSL_CTX_FUNCTION, CURLOT_FUNCTION, 0},
{"SSL_EC_CURVES", CURLOPT_SSL_EC_CURVES, CURLOT_STRING, 0},
{"SSL_ENABLE_ALPN", CURLOPT_SSL_ENABLE_ALPN, CURLOT_LONG, 0},
+ {"SSL_ENABLE_ALPS", CURLOPT_SSL_ENABLE_ALPS, CURLOT_LONG, 0},
{"SSL_ENABLE_NPN", CURLOPT_SSL_ENABLE_NPN, CURLOT_LONG, 0},
+ {"SSL_ENABLE_TICKET", CURLOPT_SSL_ENABLE_TICKET, CURLOT_LONG, 0},
{"SSL_FALSESTART", CURLOPT_SSL_FALSESTART, CURLOT_LONG, 0},
{"SSL_OPTIONS", CURLOPT_SSL_OPTIONS, CURLOT_VALUES, 0},
{"SSL_SESSIONID_CACHE", CURLOPT_SSL_SESSIONID_CACHE, CURLOT_LONG, 0},
+ {"SSL_SIG_HASH_ALGS", CURLOPT_SSL_SIG_HASH_ALGS, CURLOT_STRING, 0},
{"SSL_VERIFYHOST", CURLOPT_SSL_VERIFYHOST, CURLOT_LONG, 0},
{"SSL_VERIFYPEER", CURLOPT_SSL_VERIFYPEER, CURLOT_LONG, 0},
{"SSL_VERIFYSTATUS", CURLOPT_SSL_VERIFYSTATUS, CURLOT_LONG, 0},
@@ -360,6 +367,6 @@ struct curl_easyoption Curl_easyopts[] = {
*/
int Curl_easyopts_check(void)
{
- return ((CURLOPT_LASTENTRY%10000) != (315 + 1));
+ return ((CURLOPT_LASTENTRY%10000) != (320 + 1));
}
#endif
diff --git a/lib/http.c b/lib/http.c
index f08a343e3..2bbce4b23 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -84,6 +84,7 @@
#include "altsvc.h"
#include "hsts.h"
#include "c-hyper.h"
+#include "slist.h"
/* The last 3 #include files should be in this order */
#include "curl_printf.h"
@@ -1795,6 +1796,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
int numlists = 1; /* by default */
int i;
+ /*
+ * curl-impersonate: Use the merged list of headers if it exists (i.e. when
+ * the CURLOPT_HTTPBASEHEADER option was set.
+ */
+ struct curl_slist *noproxyheaders =
+ (data->state.merged_headers ?
+ data->state.merged_headers :
+ data->set.headers);
+
#ifndef CURL_DISABLE_PROXY
enum proxy_use proxy;
@@ -1806,10 +1816,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
switch(proxy) {
case HEADER_SERVER:
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
break;
case HEADER_PROXY:
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
if(data->set.sep_headers) {
h[1] = data->set.proxyheaders;
numlists++;
@@ -1819,12 +1829,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
if(data->set.sep_headers)
h[0] = data->set.proxyheaders;
else
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
break;
}
#else
(void)is_connect;
- h[0] = data->set.headers;
+ h[0] = noproxyheaders;
#endif
/* loop through one or two lists */
@@ -2059,6 +2069,92 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn,
*reqp = httpreq;
}
+/*
+ * curl-impersonate:
+ * Create a new linked list of headers.
+ * The new list is a merge between the "base" headers and the application given
+ * headers. The "base" headers contain curl-impersonate's list of headers
+ * used by default by the impersonated browser.
+ *
+ * The application given headers will override the "base" headers if supplied.
+ */
+CURLcode Curl_http_merge_headers(struct Curl_easy *data)
+{
+ int i;
+ int ret;
+ struct curl_slist *head;
+ struct curl_slist *dup = NULL;
+ struct curl_slist *new_list = NULL;
+
+ if (!data->state.base_headers)
+ return CURLE_OK;
+
+ /* Duplicate the list for temporary use. */
+ if (data->set.headers) {
+ dup = Curl_slist_duplicate(data->set.headers);
+ if(!dup)
+ return CURLE_OUT_OF_MEMORY;
+ }
+
+ for(head = data->state.base_headers; head; head = head->next) {
+ char *sep;
+ size_t prefix_len;
+ bool found = FALSE;
+ struct curl_slist *head2;
+
+ sep = strchr(head->data, ':');
+ if(!sep)
+ continue;
+
+ prefix_len = sep - head->data;
+
+ /* Check if this header was added by the application. */
+ for(head2 = dup; head2; head2 = head2->next) {
+ if(head2->data &&
+ strncasecompare(head2->data, head->data, prefix_len) &&
+ Curl_headersep(head2->data[prefix_len]) ) {
+ new_list = curl_slist_append(new_list, head2->data);
+ /* Free and set to NULL to mark that it's been added. */
+ Curl_safefree(head2->data);
+ found = TRUE;
+ break;
+ }
+ }
+
+ if (!found) {
+ new_list = curl_slist_append(new_list, head->data);
+ }
+
+ if (!new_list) {
+ ret = CURLE_OUT_OF_MEMORY;
+ goto fail;
+ }
+ }
+
+ /* Now go over any additional application-supplied headers. */
+ for(head = dup; head; head = head->next) {
+ if(head->data) {
+ new_list = curl_slist_append(new_list, head->data);
+ if(!new_list) {
+ ret = CURLE_OUT_OF_MEMORY;
+ goto fail;
+ }
+ }
+ }
+
+ curl_slist_free_all(dup);
+ /* Save the new, merged list separately, so it can be freed later. */
+ curl_slist_free_all(data->state.merged_headers);
+ data->state.merged_headers = new_list;
+
+ return CURLE_OK;
+
+fail:
+ Curl_safefree(dup);
+ curl_slist_free_all(new_list);
+ return ret;
+}
+
CURLcode Curl_http_useragent(struct Curl_easy *data)
{
/* The User-Agent string might have been allocated in url.c already, because
@@ -3063,6 +3159,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
http = data->req.p.http;
DEBUGASSERT(http);
+ /* curl-impersonate: Add HTTP headers to impersonate real browsers. */
+ result = Curl_http_merge_headers(data);
+ if (result)
+ return result;
+
result = Curl_http_host(data, conn);
if(result)
return result;
diff --git a/lib/http.h b/lib/http.h
index b4aaba2a2..0d716640c 100644
--- a/lib/http.h
+++ b/lib/http.h
@@ -278,7 +278,8 @@ struct http_conn {
struct h2settings settings;
/* list of settings that will be sent */
- nghttp2_settings_entry local_settings[3];
+ /* curl-impersonate: Align HTTP/2 settings to Chrome's */
+ nghttp2_settings_entry local_settings[4];
size_t local_settings_num;
#else
int unused; /* prevent a compiler warning */
diff --git a/lib/http2.c b/lib/http2.c
index e74400a4c..b0e6674f2 100644
--- a/lib/http2.c
+++ b/lib/http2.c
@@ -41,6 +41,7 @@
#include "curl_printf.h"
#include "curl_memory.h"
#include "memdebug.h"
+#include "rand.h"
#define H2_BUFSIZE 32768
@@ -56,7 +57,7 @@
#define NGHTTP2_HAS_SET_LOCAL_WINDOW_SIZE 1
#endif
-#define HTTP2_HUGE_WINDOW_SIZE (32 * 1024 * 1024) /* 32 MB */
+#define HTTP2_HUGE_WINDOW_SIZE (15 * 1024 * 1024) /* 15 MB */
#ifdef DEBUG_HTTP2
#define H2BUGF(x) x
@@ -1193,16 +1194,30 @@ static void populate_settings(struct Curl_easy *data,
{
nghttp2_settings_entry *iv = httpc->local_settings;
- iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
- iv[0].value = Curl_multi_max_concurrent_streams(data->multi);
+ /* curl-impersonate: Align HTTP/2 settings to Chrome's */
+ iv[0].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE;
+ iv[0].value = 0x10000;
- iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
- iv[1].value = HTTP2_HUGE_WINDOW_SIZE;
+ iv[1].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
+ iv[1].value = Curl_multi_max_concurrent_streams(data->multi);
- iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
- iv[2].value = data->multi->push_cb != NULL;
+ iv[2].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
+ iv[2].value = 0x600000;
- httpc->local_settings_num = 3;
+ iv[3].settings_id = NGHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE;
+ iv[3].value = 0x40000;
+
+ // iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
+ // iv[2].value = data->multi->push_cb != NULL;
+
+ // curl-impersonate:
+ // Up until Chrome 98, there was a randomly chosen setting number in the
+ // HTTP2 SETTINGS frame. This might be something similar to TLS GREASE.
+ // However, it seems to have been removed since.
+ // Curl_rand(data, (unsigned char *)&iv[4].settings_id, sizeof(iv[4].settings_id));
+ // Curl_rand(data, (unsigned char *)&iv[4].value, sizeof(iv[4].value));
+
+ httpc->local_settings_num = 4;
}
void Curl_http2_done(struct Curl_easy *data, bool premature)
@@ -1816,10 +1831,6 @@ static ssize_t http2_recv(struct Curl_easy *data, int sockindex,
return -1;
}
-/* Index where :authority header field will appear in request header
- field list. */
-#define AUTHORITY_DST_IDX 3
-
/* USHRT_MAX is 65535 == 0xffff */
#define HEADER_OVERFLOW(x) \
(x.namelen > 0xffff || x.valuelen > 0xffff - x.namelen)
@@ -1890,6 +1901,53 @@ static header_instruction inspect_header(const char *name, size_t namelen,
}
}
+/*
+ * curl-impersonate:
+ * Determine the position of HTTP/2 pseudo headers.
+ * The pseudo headers ":method", ":path", ":scheme", ":authority"
+ * are sent in different order by different browsers. An important part of the
+ * impersonation is ordering them like the browser does.
+ */
+static int http2_pseudo_header_index(struct Curl_easy *data,
+ const char *header,
+ size_t *index)
+{
+ char *off;
+ // Use the Chrome ordering by default:
+ // :method, :authority, :scheme, :path
+ char *order = "masp";
+ if(data->set.str[STRING_HTTP2_PSEUDO_HEADERS_ORDER])
+ order = data->set.str[STRING_HTTP2_PSEUDO_HEADERS_ORDER];
+
+ if(strlen(order) != 4)
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+
+ // :method should always be first
+ if(order[0] != 'm')
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+
+ // All pseudo-headers must be present
+ if(!strchr(order, 'm') ||
+ !strchr(order, 'a') ||
+ !strchr(order, 's') ||
+ !strchr(order, 'p'))
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+
+ if(strcasecompare(header, ":method"))
+ off = strchr(order, 'm');
+ else if(strcasecompare(header, ":authority"))
+ off = strchr(order, 'a');
+ else if(strcasecompare(header, ":scheme"))
+ off = strchr(order, 's');
+ else if(strcasecompare(header, ":path"))
+ off = strchr(order, 'p');
+ else
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+
+ *index = off - order;
+ return CURLE_OK;
+}
+
static ssize_t http2_send(struct Curl_easy *data, int sockindex,
const void *mem, size_t len, CURLcode *err)
{
@@ -1905,6 +1963,7 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
nghttp2_nv *nva = NULL;
size_t nheader;
size_t i;
+ size_t header_idx;
size_t authority_idx;
char *hdbuf = (char *)mem;
char *end, *line_end;
@@ -2010,12 +2069,21 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
end = memchr(hdbuf, ' ', line_end - hdbuf);
if(!end || end == hdbuf)
goto fail;
- nva[0].name = (unsigned char *)":method";
- nva[0].namelen = strlen((char *)nva[0].name);
- nva[0].value = (unsigned char *)hdbuf;
- nva[0].valuelen = (size_t)(end - hdbuf);
- nva[0].flags = NGHTTP2_NV_FLAG_NONE;
- if(HEADER_OVERFLOW(nva[0])) {
+ /* curl-impersonate: Set the index of ":method" based on libcurl option */
+ if(http2_pseudo_header_index(data, ":authority", &authority_idx))
+ goto fail;
+ if(http2_pseudo_header_index(data, ":method", &header_idx))
+ goto fail;
+ /* This is needed to overcome the fact that curl will only move the authority
+ * header into its place after all other headers have been placed. */
+ if(header_idx > authority_idx)
+ header_idx--;
+ nva[header_idx].name = (unsigned char *)":method";
+ nva[header_idx].namelen = strlen((char *)nva[header_idx].name);
+ nva[header_idx].value = (unsigned char *)hdbuf;
+ nva[header_idx].valuelen = (size_t)(end - hdbuf);
+ nva[header_idx].flags = NGHTTP2_NV_FLAG_NONE;
+ if(HEADER_OVERFLOW(nva[header_idx])) {
failf(data, "Failed sending HTTP request: Header overflow");
goto fail;
}
@@ -2032,25 +2100,35 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
}
if(!end || end == hdbuf)
goto fail;
- nva[1].name = (unsigned char *)":path";
- nva[1].namelen = strlen((char *)nva[1].name);
- nva[1].value = (unsigned char *)hdbuf;
- nva[1].valuelen = (size_t)(end - hdbuf);
- nva[1].flags = NGHTTP2_NV_FLAG_NONE;
- if(HEADER_OVERFLOW(nva[1])) {
+ /* curl-impersonate: Set the index of ":path" based on libcurl option */
+ if(http2_pseudo_header_index(data, ":path", &header_idx))
+ goto fail;
+ if(header_idx > authority_idx)
+ header_idx--;
+ nva[header_idx].name = (unsigned char *)":path";
+ nva[header_idx].namelen = strlen((char *)nva[header_idx].name);
+ nva[header_idx].value = (unsigned char *)hdbuf;
+ nva[header_idx].valuelen = (size_t)(end - hdbuf);
+ nva[header_idx].flags = NGHTTP2_NV_FLAG_NONE;
+ if(HEADER_OVERFLOW(nva[header_idx])) {
failf(data, "Failed sending HTTP request: Header overflow");
goto fail;
}
- nva[2].name = (unsigned char *)":scheme";
- nva[2].namelen = strlen((char *)nva[2].name);
+ /* curl-impersonate: Set the index of ":scheme" based on libcurl option */
+ if(http2_pseudo_header_index(data, ":scheme", &header_idx))
+ goto fail;
+ if(header_idx > authority_idx)
+ header_idx--;
+ nva[header_idx].name = (unsigned char *)":scheme";
+ nva[header_idx].namelen = strlen((char *)nva[header_idx].name);
if(conn->handler->flags & PROTOPT_SSL)
- nva[2].value = (unsigned char *)"https";
+ nva[header_idx].value = (unsigned char *)"https";
else
- nva[2].value = (unsigned char *)"http";
- nva[2].valuelen = strlen((char *)nva[2].value);
- nva[2].flags = NGHTTP2_NV_FLAG_NONE;
- if(HEADER_OVERFLOW(nva[2])) {
+ nva[header_idx].value = (unsigned char *)"http";
+ nva[header_idx].valuelen = strlen((char *)nva[header_idx].value);
+ nva[header_idx].flags = NGHTTP2_NV_FLAG_NONE;
+ if(HEADER_OVERFLOW(nva[header_idx])) {
failf(data, "Failed sending HTTP request: Header overflow");
goto fail;
}
@@ -2117,10 +2195,13 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
++i;
}
+ /* curl-impersonate: Set the index of ":authority" based on libcurl option */
+ if(http2_pseudo_header_index(data, ":authority", &header_idx))
+ goto fail;
/* :authority must come before non-pseudo header fields */
- if(authority_idx && authority_idx != AUTHORITY_DST_IDX) {
+ if(authority_idx && authority_idx != header_idx) {
nghttp2_nv authority = nva[authority_idx];
- for(i = authority_idx; i > AUTHORITY_DST_IDX; --i) {
+ for(i = authority_idx; i > header_idx; --i) {
nva[i] = nva[i - 1];
}
nva[i] = authority;
diff --git a/lib/http2.h b/lib/http2.h
index d6986d97f..fa5c90e7f 100644
--- a/lib/http2.h
+++ b/lib/http2.h
@@ -29,7 +29,8 @@
/* value for MAX_CONCURRENT_STREAMS we use until we get an updated setting
from the peer */
-#define DEFAULT_MAX_CONCURRENT_STREAMS 100
+/* curl-impersonate: Use 1000 concurrent streams like Chrome. */
+#define DEFAULT_MAX_CONCURRENT_STREAMS 1000
/*
* Store nghttp2 version info in this buffer.
diff --git a/lib/multi.c b/lib/multi.c
index f8dcc63b4..e6b728592 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -393,7 +393,8 @@ struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */
/* -1 means it not set by user, use the default value */
multi->maxconnects = -1;
- multi->max_concurrent_streams = 100;
+ /* curl-impersonate: Use 1000 concurrent streams like Chrome. */
+ multi->max_concurrent_streams = 1000;
multi->ipv6_works = Curl_ipv6works(NULL);
#ifdef USE_WINSOCK
diff --git a/lib/setopt.c b/lib/setopt.c
index 599ed5d99..3ac151feb 100644
--- a/lib/setopt.c
+++ b/lib/setopt.c
@@ -48,6 +48,7 @@
#include "multiif.h"
#include "altsvc.h"
#include "hsts.h"
+#include "slist.h"
/* The last 3 #include files should be in this order */
#include "curl_printf.h"
@@ -688,6 +689,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
va_arg(param, char *));
break;
+ case CURLOPT_HTTPBASEHEADER:
+ /*
+ * curl-impersonate:
+ * Set a list of "base" headers. These will be merged with any headers
+ * set by CURLOPT_HTTPHEADER. curl-impersonate uses this option in order
+ * to set a list of default browser headers.
+ *
+ * Unlike CURLOPT_HTTPHEADER,
+ * the list is copied and can be immediately freed by the user.
+ */
+ curl_slist_free_all(data->state.base_headers);
+ data->state.base_headers = \
+ Curl_slist_duplicate(va_arg(param, struct curl_slist *));
+ if (!data->state.base_headers)
+ result = CURLE_OUT_OF_MEMORY;
+ break;
+
case CURLOPT_HTTPHEADER:
/*
* Set a list with HTTP headers to use (or replace internals with)
@@ -2349,6 +2367,27 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
result = Curl_setstropt(&data->set.str[STRING_SSL_EC_CURVES],
va_arg(param, char *));
break;
+
+ case CURLOPT_SSL_SIG_HASH_ALGS:
+ /*
+ * Set the list of hash algorithms we want to use in the SSL connection.
+ * Specify comma-delimited list of algorithms to use.
+ */
+ result = Curl_setstropt(&data->set.str[STRING_SSL_SIG_HASH_ALGS],
+ va_arg(param, char *));
+ break;
+
+ case CURLOPT_SSL_CERT_COMPRESSION:
+ /*
+ * Set the list of ceritifcate compression algorithms we support in the TLS
+ * connection.
+ * Specify comma-delimited list of algorithms to use. Options are "zlib"
+ * and "brotli".
+ */
+ result = Curl_setstropt(&data->set.str[STRING_SSL_CERT_COMPRESSION],
+ va_arg(param, char *));
+ break;
+
#endif
case CURLOPT_IPRESOLVE:
arg = va_arg(param, long);
@@ -2871,6 +2910,16 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
case CURLOPT_SSL_ENABLE_ALPN:
data->set.ssl_enable_alpn = (0 != va_arg(param, long)) ? TRUE : FALSE;
break;
+ case CURLOPT_SSL_ENABLE_ALPS:
+ data->set.ssl_enable_alps = (0 != va_arg(param, long)) ? TRUE : FALSE;
+ break;
+ case CURLOPT_SSL_ENABLE_TICKET:
+ data->set.ssl_enable_ticket = (0 != va_arg(param, long)) ? TRUE : FALSE;
+ break;
+ case CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER:
+ result = Curl_setstropt(&data->set.str[STRING_HTTP2_PSEUDO_HEADERS_ORDER],
+ va_arg(param, char *));
+ break;
#ifdef USE_UNIX_SOCKETS
case CURLOPT_UNIX_SOCKET_PATH:
data->set.abstract_unix_socket = FALSE;
diff --git a/lib/transfer.c b/lib/transfer.c
index 22704fa15..1e100140c 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -102,7 +102,15 @@ char *Curl_checkheaders(const struct Curl_easy *data,
DEBUGASSERT(thislen);
DEBUGASSERT(thisheader[thislen-1] != ':');
- for(head = data->set.headers; head; head = head->next) {
+ /*
+ * curl-impersonate:
+ * Check if we have overriden the user-supplied list of headers.
+ */
+ head = data->set.headers;
+ if (data->state.merged_headers)
+ head = data->state.merged_headers;
+
+ for(; head; head = head->next) {
if(strncasecompare(head->data, thisheader, thislen) &&
Curl_headersep(head->data[thislen]) )
return head->data;
diff --git a/lib/url.c b/lib/url.c
index 9f1013554..0eff9c354 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -469,6 +469,11 @@ CURLcode Curl_close(struct Curl_easy **datap)
Curl_safefree(data->state.aptr.proxyuser);
Curl_safefree(data->state.aptr.proxypasswd);
+ /* curl-impersonate: Free the list set by CURLOPT_HTTPBASEHEADER. */
+ curl_slist_free_all(data->state.base_headers);
+ /* curl-impersonate: Free the dynamic list of headers. */
+ curl_slist_free_all(data->state.merged_headers);
+
#ifndef CURL_DISABLE_DOH
if(data->req.doh) {
Curl_dyn_free(&data->req.doh->probe[0].serverdoh);
@@ -622,6 +627,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data)
set->tcp_nodelay = TRUE;
set->ssl_enable_npn = TRUE;
set->ssl_enable_alpn = TRUE;
+ set->ssl_enable_ticket = TRUE;
set->expect_100_timeout = 1000L; /* Wait for a second by default. */
set->sep_headers = TRUE; /* separated header lists by default */
set->buffer_size = READBUFFER_SIZE;
@@ -3808,6 +3814,9 @@ static CURLcode create_conn(struct Curl_easy *data,
data->set.ssl.primary.cert_blob = data->set.blobs[BLOB_CERT];
data->set.ssl.primary.ca_info_blob = data->set.blobs[BLOB_CAINFO];
data->set.ssl.primary.curves = data->set.str[STRING_SSL_EC_CURVES];
+ data->set.ssl.primary.sig_hash_algs = data->set.str[STRING_SSL_SIG_HASH_ALGS];
+ data->set.ssl.primary.cert_compression =
+ data->set.str[STRING_SSL_CERT_COMPRESSION];
#ifndef CURL_DISABLE_PROXY
data->set.proxy_ssl.primary.CApath = data->set.str[STRING_SSL_CAPATH_PROXY];
@@ -3925,8 +3934,17 @@ static CURLcode create_conn(struct Curl_easy *data,
conn->bits.tls_enable_alpn = TRUE;
if(data->set.ssl_enable_npn)
conn->bits.tls_enable_npn = TRUE;
+
+ /* curl-impersonate: Turn on ALPS if ALPN is enabled and the bit is
+ * enabled. */
+ if(data->set.ssl_enable_alps)
+ conn->bits.tls_enable_alps = TRUE;
}
+ /* curl-impersonate: Add the TLS session ticket extension. */
+ if(data->set.ssl_enable_ticket)
+ conn->bits.tls_enable_ticket = TRUE;
+
if(waitpipe)
/* There is a connection that *might* become usable for multiplexing
"soon", and we wait for that */
diff --git a/lib/urldata.h b/lib/urldata.h
index cc9c88870..636ae6770 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -257,6 +257,8 @@ struct ssl_primary_config {
struct curl_blob *ca_info_blob;
struct curl_blob *issuercert_blob;
char *curves; /* list of curves to use */
+ char *sig_hash_algs; /* List of signature hash algorithms to use */
+ char *cert_compression; /* List of certificate compression algorithms. */
BIT(verifypeer); /* set TRUE if this is desired */
BIT(verifyhost); /* set TRUE if CN/SAN must match hostname */
BIT(verifystatus); /* set TRUE if certificate status must be checked */
@@ -517,6 +519,8 @@ struct ConnectBits {
BIT(tcp_fastopen); /* use TCP Fast Open */
BIT(tls_enable_npn); /* TLS NPN extension? */
BIT(tls_enable_alpn); /* TLS ALPN extension? */
+ BIT(tls_enable_alps); /* TLS ALPS extension? */
+ BIT(tls_enable_ticket); /* TLS session ticket extension? */
BIT(connect_only);
#ifndef CURL_DISABLE_DOH
BIT(doh);
@@ -1421,6 +1425,19 @@ struct UrlState {
CURLcode hresult; /* used to pass return codes back from hyper callbacks */
#endif
+ /*
+ * curl-impersonate:
+ * List of "base" headers set by CURLOPT_HTTPBASEHEADER.
+ */
+ struct curl_slist *base_headers;
+ /*
+ * curl-impersonate:
+ * Dynamically-constructed list of HTTP headers.
+ * This list is a merge of the default HTTP headers needed to impersonate a
+ * browser, together with any user-supplied headers.
+ */
+ struct curl_slist *merged_headers;
+
/* Dynamically allocated strings, MUST be freed before this struct is
killed. */
struct dynamically_allocated_data {
@@ -1579,6 +1596,9 @@ enum dupstring {
STRING_DNS_LOCAL_IP4,
STRING_DNS_LOCAL_IP6,
STRING_SSL_EC_CURVES,
+ STRING_SSL_SIG_HASH_ALGS,
+ STRING_SSL_CERT_COMPRESSION,
+ STRING_HTTP2_PSEUDO_HEADERS_ORDER,
/* -- end of null-terminated strings -- */
@@ -1849,6 +1869,8 @@ struct UserDefined {
BIT(tcp_fastopen); /* use TCP Fast Open */
BIT(ssl_enable_npn); /* TLS NPN extension? */
BIT(ssl_enable_alpn);/* TLS ALPN extension? */
+ BIT(ssl_enable_alps);/* TLS ALPS extension? */
+ BIT(ssl_enable_ticket); /* TLS session ticket extension */
BIT(path_as_is); /* allow dotdots? */
BIT(pipewait); /* wait for multiplex status before starting a new
connection */
diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c
index f836c63b0..c310f65ba 100644
--- a/lib/vtls/openssl.c
+++ b/lib/vtls/openssl.c
@@ -76,6 +76,13 @@
#include <openssl/buffer.h>
#include <openssl/pkcs12.h>
+#ifdef HAVE_ZLIB_H
+#include <zlib.h>
+#endif
+#ifdef HAVE_BROTLI
+#include <brotli/decode.h>
+#endif
+
#ifdef USE_AMISSL
#include "amigaos.h"
#endif
@@ -209,6 +216,10 @@
!defined(OPENSSL_IS_BORINGSSL))
#define HAVE_SSL_CTX_SET_CIPHERSUITES
#define HAVE_SSL_CTX_SET_POST_HANDSHAKE_AUTH
+#endif
+
+#if ((OPENSSL_VERSION_NUMBER >= 0x10101000L) && \
+ !defined(LIBRESSL_VERSION_NUMBER))
/* SET_EC_CURVES is available under the same preconditions: see
* https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set1_groups.html
*/
@@ -253,6 +264,113 @@
#define HAVE_OPENSSL_VERSION
#endif
+#if defined(OPENSSL_IS_BORINGSSL)
+#define HAVE_SSL_CTX_SET_VERIFY_ALGORITHM_PREFS
+
+/*
+ * kMaxSignatureAlgorithmNameLen and kSignatureAlgorithmNames
+ * Taken from BoringSSL, see ssl/ssl_privkey.cc
+ * */
+static const size_t kMaxSignatureAlgorithmNameLen = 23;
+
+static const struct {
+ uint16_t signature_algorithm;
+ const char *name;
+} kSignatureAlgorithmNames[] = {
+ {SSL_SIGN_RSA_PKCS1_MD5_SHA1, "rsa_pkcs1_md5_sha1"},
+ {SSL_SIGN_RSA_PKCS1_SHA1, "rsa_pkcs1_sha1"},
+ {SSL_SIGN_RSA_PKCS1_SHA256, "rsa_pkcs1_sha256"},
+ {SSL_SIGN_RSA_PKCS1_SHA384, "rsa_pkcs1_sha384"},
+ {SSL_SIGN_RSA_PKCS1_SHA512, "rsa_pkcs1_sha512"},
+ {SSL_SIGN_ECDSA_SHA1, "ecdsa_sha1"},
+ {SSL_SIGN_ECDSA_SECP256R1_SHA256, "ecdsa_secp256r1_sha256"},
+ {SSL_SIGN_ECDSA_SECP384R1_SHA384, "ecdsa_secp384r1_sha384"},
+ {SSL_SIGN_ECDSA_SECP521R1_SHA512, "ecdsa_secp521r1_sha512"},
+ {SSL_SIGN_RSA_PSS_RSAE_SHA256, "rsa_pss_rsae_sha256"},
+ {SSL_SIGN_RSA_PSS_RSAE_SHA384, "rsa_pss_rsae_sha384"},
+ {SSL_SIGN_RSA_PSS_RSAE_SHA512, "rsa_pss_rsae_sha512"},
+ {SSL_SIGN_ED25519, "ed25519"},
+};
+
+#define MAX_SIG_ALGS \
+ sizeof(kSignatureAlgorithmNames) / sizeof(kSignatureAlgorithmNames[0])
+
+/* Default signature hash algorithms taken from Chrome/Chromium.
+ * See kVerifyPeers @ net/socket/ssl_client_socket_impl.cc */
+static const uint16_t default_sig_algs[] = {
+ SSL_SIGN_ECDSA_SECP256R1_SHA256, SSL_SIGN_RSA_PSS_RSAE_SHA256,
+ SSL_SIGN_RSA_PKCS1_SHA256, SSL_SIGN_ECDSA_SECP384R1_SHA384,
+ SSL_SIGN_RSA_PSS_RSAE_SHA384, SSL_SIGN_RSA_PKCS1_SHA384,
+ SSL_SIGN_RSA_PSS_RSAE_SHA512, SSL_SIGN_RSA_PKCS1_SHA512,
+};
+
+#define DEFAULT_SIG_ALGS_LENGTH \
+ sizeof(default_sig_algs) / sizeof(default_sig_algs[0])
+
+static CURLcode parse_sig_algs(struct Curl_easy *data,
+ const char *sigalgs,
+ uint16_t *algs,
+ size_t *nalgs)
+{
+ *nalgs = 0;
+ while (sigalgs && sigalgs[0]) {
+ int i;
+ bool found = FALSE;
+ const char *end;
+ size_t len;
+ char algname[kMaxSignatureAlgorithmNameLen + 1];
+
+ end = strpbrk(sigalgs, ":,");
+ if (end)
+ len = end - sigalgs;
+ else
+ len = strlen(sigalgs);
+
+ if (len > kMaxSignatureAlgorithmNameLen) {
+ failf(data, "Bad signature hash algorithm list");
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+ }
+
+ if (!len) {
+ ++sigalgs;
+ continue;
+ }
+
+ if (*nalgs == MAX_SIG_ALGS) {
+ /* Reached the maximum number of possible algorithms, but more data
+ * available in the list. */
+ failf(data, "Bad signature hash algorithm list");
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+ }
+
+ memcpy(algname, sigalgs, len);
+ algname[len] = 0;
+
+ for (i = 0; i < MAX_SIG_ALGS; i++) {
+ if (strcasecompare(algname, kSignatureAlgorithmNames[i].name)) {
+ algs[*nalgs] = kSignatureAlgorithmNames[i].signature_algorithm;
+ (*nalgs)++;
+ found = TRUE;
+ break;
+ }
+ }
+
+ if (!found) {
+ failf(data, "Unknown signature hash algorithm: '%s'", algname);
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+ }
+
+ if (end)
+ sigalgs = ++end;
+ else
+ break;
+ }
+
+ return CURLE_OK;
+}
+
+#endif
+
struct ssl_backend_data {
struct Curl_easy *logger; /* transfer handle to pass trace logs to, only
using sockindex 0 */
@@ -2629,6 +2747,151 @@ static CURLcode load_cacert_from_memory(SSL_CTX *ctx,
return (count > 0 ? CURLE_OK : CURLE_SSL_CACERT_BADFILE);
}
+#ifdef HAVE_LIBZ
+int DecompressZlibCert(SSL *ssl,
+ CRYPTO_BUFFER** out,
+ size_t uncompressed_len,
+ const uint8_t* in,
+ size_t in_len)
+{
+ z_stream strm;
+ uint8_t* data;
+ CRYPTO_BUFFER* decompressed = CRYPTO_BUFFER_alloc(&data, uncompressed_len);
+ if(!decompressed) {
+ return 0;
+ }
+
+ strm.zalloc = NULL;
+ strm.zfree = NULL;
+ strm.opaque = NULL;
+ strm.next_in = (Bytef *)in;
+ strm.avail_in = in_len;
+ strm.next_out = (Bytef *)data;
+ strm.avail_out = uncompressed_len;
+
+ if(inflateInit(&strm) != Z_OK) {
+ CRYPTO_BUFFER_free(decompressed);
+ return 0;
+ }
+
+ if(inflate(&strm, Z_FINISH) != Z_STREAM_END ||
+ strm.avail_in != 0 ||
+ strm.avail_out != 0) {
+ inflateEnd(&strm);
+ CRYPTO_BUFFER_free(decompressed);
+ return 0;
+ }
+
+ inflateEnd(&strm);
+ *out = decompressed;
+ return 1;
+}
+#endif
+
+#ifdef HAVE_BROTLI
+
+/* Taken from Chromium and adapted to C,
+ * see net/ssl/cert_compression.cc
+ */
+int DecompressBrotliCert(SSL* ssl,
+ CRYPTO_BUFFER** out,
+ size_t uncompressed_len,
+ const uint8_t* in,
+ size_t in_len) {
+ uint8_t* data;
+ CRYPTO_BUFFER* decompressed = CRYPTO_BUFFER_alloc(&data, uncompressed_len);
+ if (!decompressed) {
+ return 0;
+ }
+
+ size_t output_size = uncompressed_len;
+ if (BrotliDecoderDecompress(in_len, in, &output_size, data) !=
+ BROTLI_DECODER_RESULT_SUCCESS ||
+ output_size != uncompressed_len) {
+ CRYPTO_BUFFER_free(decompressed);
+ return 0;
+ }
+
+ *out = decompressed;
+ return 1;
+}
+#endif
+
+#if defined(HAVE_LIBZ) || defined(HAVE_BROTLI)
+static struct {
+ char *alg_name;
+ uint16_t alg_id;
+ ssl_cert_compression_func_t compress;
+ ssl_cert_decompression_func_t decompress;
+} cert_compress_algs[] = {
+#ifdef HAVE_LIBZ
+ {"zlib", TLSEXT_cert_compression_zlib, NULL, DecompressZlibCert},
+#endif
+#ifdef HAVE_BROTLI
+ {"brotli", TLSEXT_cert_compression_brotli, NULL, DecompressBrotliCert},
+#endif
+};
+
+#define NUM_CERT_COMPRESSION_ALGS \
+ sizeof(cert_compress_algs) / sizeof(cert_compress_algs[0])
+
+/*
+ * curl-impersonate:
+ * Add support for TLS extension 27 - compress_certificate.
+ * This calls the BoringSSL-specific API SSL_CTX_add_cert_compression_alg
+ * for each algorithm specified in cert_compression, which is a comma separated list.
+ */
+static CURLcode add_cert_compression(struct Curl_easy *data,
+ SSL_CTX *ctx,
+ const char *algorithms)
+{
+ int i;
+ const char *s = algorithms;
+ char *alg_name;
+ size_t alg_name_len;
+ bool found;
+
+ while (s && s[0]) {
+ found = FALSE;
+
+ for(i = 0; i < NUM_CERT_COMPRESSION_ALGS; i++) {
+ alg_name = cert_compress_algs[i].alg_name;
+ alg_name_len = strlen(alg_name);
+ if(strlen(s) >= alg_name_len &&
+ strncasecompare(s, alg_name, alg_name_len) &&
+ (s[alg_name_len] == ',' || s[alg_name_len] == 0)) {
+ if(!SSL_CTX_add_cert_compression_alg(ctx,
+ cert_compress_algs[i].alg_id,
+ cert_compress_algs[i].compress,
+ cert_compress_algs[i].decompress)) {
+ failf(data, "Error adding certificate compression algorithm '%s'",
+ alg_name);
+ return CURLE_SSL_CIPHER;
+ }
+ s += alg_name_len;
+ if(*s == ',')
+ s += 1;
+ found = TRUE;
+ break;
+ }
+ }
+
+ if(!found) {
+ failf(data, "Invalid compression algorithm list");
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+ }
+ }
+
+ return CURLE_OK;
+}
+#else
+static CURLcode add_cert_compression(SSL_CTX *ctx, const char *algorithms)
+{
+ /* No compression algorithms are available. */
+ return CURLE_BAD_FUNCTION_ARGUMENT;
+}
+#endif
+
static CURLcode ossl_connect_step1(struct Curl_easy *data,
struct connectdata *conn, int sockindex)
{
@@ -2767,7 +3030,14 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
ctx_options = SSL_OP_ALL;
#ifdef SSL_OP_NO_TICKET
- ctx_options |= SSL_OP_NO_TICKET;
+ if(conn->bits.tls_enable_ticket) {
+ /* curl-impersonate:
+ * Turn off SSL_OP_NO_TICKET, we want TLS extension 35 (session_ticket)
+ * to be present in the client hello. */
+ ctx_options &= ~SSL_OP_NO_TICKET;
+ } else {
+ ctx_options |= SSL_OP_NO_TICKET;
+ }
#endif
#ifdef SSL_OP_NO_COMPRESSION
@@ -2912,6 +3182,35 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
}
#endif
+#ifdef HAVE_SSL_CTX_SET_VERIFY_ALGORITHM_PREFS
+ {
+ uint16_t algs[MAX_SIG_ALGS];
+ size_t nalgs;
+ /* curl-impersonate: Set the signature algorithms (TLS extension 13).
+ * See net/socket/ssl_client_socket_impl.cc in Chromium's source. */
+ char *sig_hash_algs = SSL_CONN_CONFIG(sig_hash_algs);
+ if (sig_hash_algs) {
+ CURLcode result = parse_sig_algs(data, sig_hash_algs, algs, &nalgs);
+ if (result)
+ return result;
+ if (!SSL_CTX_set_verify_algorithm_prefs(backend->ctx, algs, nalgs)) {
+ failf(data, "failed setting signature hash algorithms list: '%s'",
+ sig_hash_algs);
+ return CURLE_SSL_CIPHER;
+ }
+ } else {
+ /* Use defaults from Chrome. */
+ if (!SSL_CTX_set_verify_algorithm_prefs(backend->ctx,
+ default_sig_algs,
+ DEFAULT_SIG_ALGS_LENGTH)) {
+ failf(data, "failed setting signature hash algorithms list: '%s'",
+ sig_hash_algs);
+ return CURLE_SSL_CIPHER;
+ }
+ }
+ }
+#endif
+
#ifdef USE_OPENSSL_SRP
if(ssl_authtype == CURL_TLSAUTH_SRP) {
char * const ssl_username = SSL_SET_OPTION(username);
@@ -2937,6 +3236,20 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
}
#endif
+ /* curl-impersonate:
+ * Configure BoringSSL to behave like Chrome.
+ * See Constructor of SSLContext at net/socket/ssl_client_socket_impl.cc
+ * and SSLClientSocketImpl::Init()
+ * in the Chromium's source code. */
+
+ /* Enable TLS GREASE. */
+ SSL_CTX_set_grease_enabled(backend->ctx, 1);
+
+ if(SSL_CONN_CONFIG(cert_compression) &&
+ add_cert_compression(data,
+ backend->ctx,
+ SSL_CONN_CONFIG(cert_compression)))
+ return CURLE_SSL_CIPHER;
#if defined(USE_WIN32_CRYPTO)
/* Import certificates from the Windows root certificate store if requested.
@@ -3236,6 +3549,33 @@ static CURLcode ossl_connect_step1(struct Curl_easy *data,
SSL_set_connect_state(backend->handle);
+#if defined(HAS_ALPN) && defined(USE_HTTP2)
+ if(conn->bits.tls_enable_alpn &&
+ data->state.httpwant >= CURL_HTTP_VERSION_2 &&
+ conn->bits.tls_enable_alps) {
+ /* curl-impersonate: This adds the ALPS extension (17513).
+ * Chromium calls this function as well in SSLClientSocketImpl::Init().
+ * The 4th parameter is called "settings", and I don't know what it
+ * should contain. For now, use an empty string. */
+ SSL_add_application_settings(backend->handle, ALPN_H2, ALPN_H2_LENGTH,
+ NULL, 0);
+ infof(data, "ALPS, offering %s", ALPN_H2);
+ }
+#endif
+
+ SSL_set_options(backend->handle,
+ SSL_OP_LEGACY_SERVER_CONNECT);
+ SSL_set_mode(backend->handle,
+ SSL_MODE_CBC_RECORD_SPLITTING | SSL_MODE_ENABLE_FALSE_START);
+
+ /* curl-impersonate: Enable TLS extensions 5 - status_request and
+ * 18 - signed_certificate_timestamp. */
+ SSL_enable_signed_cert_timestamps(backend->handle);
+ SSL_enable_ocsp_stapling(backend->handle);
+
+ /* curl-impersonate: Some SSL settings copied over from Chrome. */
+ SSL_set_shed_handshake_config(backend->handle, 1);
+
backend->server_cert = 0x0;
#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME
if((0 == Curl_inet_pton(AF_INET, hostname, &addr)) &&
diff --git a/lib/vtls/vtls.c b/lib/vtls/vtls.c
index 6007bbba0..3c79e0d30 100644
--- a/lib/vtls/vtls.c
+++ b/lib/vtls/vtls.c
@@ -156,6 +156,9 @@ Curl_ssl_config_matches(struct ssl_primary_config *data,
Curl_safe_strcasecompare(data->cipher_list, needle->cipher_list) &&
Curl_safe_strcasecompare(data->cipher_list13, needle->cipher_list13) &&
Curl_safe_strcasecompare(data->curves, needle->curves) &&
+ Curl_safe_strcasecompare(data->sig_hash_algs, needle->sig_hash_algs) &&
+ Curl_safe_strcasecompare(data->cert_compression,
+ needle->cert_compression) &&
Curl_safe_strcasecompare(data->pinned_key, needle->pinned_key))
return TRUE;
@@ -186,6 +189,8 @@ Curl_clone_primary_ssl_config(struct ssl_primary_config *source,
CLONE_STRING(cipher_list13);
CLONE_STRING(pinned_key);
CLONE_STRING(curves);
+ CLONE_STRING(sig_hash_algs);
+ CLONE_STRING(cert_compression);
return TRUE;
}
@@ -205,6 +210,8 @@ void Curl_free_primary_ssl_config(struct ssl_primary_config *sslc)
Curl_safefree(sslc->ca_info_blob);
Curl_safefree(sslc->issuercert_blob);
Curl_safefree(sslc->curves);
+ Curl_safefree(sslc->sig_hash_algs);
+ Curl_safefree(sslc->cert_compression);
}
#ifdef USE_SSL
diff --git a/libcurl.pc.in b/libcurl.pc.in
index 8ac15d407..986d42b0e 100644
--- a/libcurl.pc.in
+++ b/libcurl.pc.in
@@ -34,6 +34,6 @@ Name: libcurl
URL: https://curl.se/
Description: Library to transfer files with ftp, http, etc.
Version: @CURLVERSION@
-Libs: -L${libdir} -lcurl @LIBCURL_NO_SHARED@
+Libs: -L${libdir} -lcurl-impersonate-chrome @LIBCURL_NO_SHARED@
Libs.private: @LIBCURL_LIBS@
Cflags: -I${includedir} @CPPFLAG_CURL_STATICLIB@
diff --git a/src/Makefile.am b/src/Makefile.am
index c8abc93b1..cf8dfdef7 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -41,7 +41,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/include \
-I$(top_srcdir)/lib \
-I$(top_srcdir)/src
-bin_PROGRAMS = curl
+bin_PROGRAMS = curl-impersonate-chrome
SUBDIRS = ../docs
@@ -52,7 +52,7 @@ endif
include Makefile.inc
# CURL_FILES comes from Makefile.inc
-curl_SOURCES = $(CURL_FILES)
+curl_impersonate_chrome_SOURCES = $(CURL_FILES)
# This might hold -Werror
CFLAGS += @CURL_CFLAG_EXTRAS@
@@ -61,9 +61,9 @@ CFLAGS += @CURL_CFLAG_EXTRAS@
LIBS = $(BLANK_AT_MAKETIME)
if USE_EXPLICIT_LIB_DEPS
-curl_LDADD = $(top_builddir)/lib/libcurl.la @LIBCURL_LIBS@
+curl_impersonate_chrome_LDADD = $(top_builddir)/lib/libcurl-impersonate-chrome.la @LIBCURL_LIBS@
else
-curl_LDADD = $(top_builddir)/lib/libcurl.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@
+curl_impersonate_chrome_LDADD = $(top_builddir)/lib/libcurl-impersonate-chrome.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@
endif
# if unit tests are enabled, build a static library to link them with
diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h
index 227b914e3..9f0d0b18b 100644
--- a/src/tool_cfgable.h
+++ b/src/tool_cfgable.h
@@ -165,8 +165,11 @@ struct OperationConfig {
bool crlf;
char *customrequest;
char *ssl_ec_curves;
+ char *ssl_sig_hash_algs;
+ char *ssl_cert_compression;
char *krblevel;
char *request_target;
+ char *http2_pseudo_headers_order;
long httpversion;
bool http09_allowed;
bool nobuffer;
@@ -274,6 +277,8 @@ struct OperationConfig {
char *oauth_bearer; /* OAuth 2.0 bearer token */
bool nonpn; /* enable/disable TLS NPN extension */
bool noalpn; /* enable/disable TLS ALPN extension */
+ bool alps; /* enable/disable TLS ALPS extension */
+ bool noticket; /* enable/disable TLS session ticket */
char *unix_socket_path; /* path to Unix domain socket */
bool abstract_unix_socket; /* path to an abstract Unix domain socket */
bool falsestart;
diff --git a/src/tool_getparam.c b/src/tool_getparam.c
index 7abbcc639..d2455b74f 100644
--- a/src/tool_getparam.c
+++ b/src/tool_getparam.c
@@ -279,6 +279,11 @@ static const struct LongShort aliases[]= {
{"EC", "etag-save", ARG_FILENAME},
{"ED", "etag-compare", ARG_FILENAME},
{"EE", "curves", ARG_STRING},
+ {"EG", "signature-hashes", ARG_STRING},
+ {"EH", "alps", ARG_BOOL},
+ {"EI", "cert-compression", ARG_STRING},
+ {"EJ", "tls-session-ticket", ARG_BOOL},
+ {"EK", "http2-pseudo-headers-order", ARG_STRING},
{"f", "fail", ARG_BOOL},
{"fa", "fail-early", ARG_BOOL},
{"fb", "styled-output", ARG_BOOL},
@@ -1794,6 +1799,31 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
GetStr(&config->ssl_ec_curves, nextarg);
break;
+ case 'G':
+ /* --signature-hashes */
+ GetStr(&config->ssl_sig_hash_algs, nextarg);
+ break;
+
+ case 'H':
+ /* --alps */
+ config->alps = toggle;
+ break;
+
+ case 'I':
+ /* --cert-compression */
+ GetStr(&config->ssl_cert_compression, nextarg);
+ break;
+
+ case 'J':
+ /* --tls-session-ticket */
+ config->noticket = (!toggle)?TRUE:FALSE;
+ break;
+
+ case 'K':
+ /* --http2-pseudo-headers-order */
+ GetStr(&config->http2_pseudo_headers_order, nextarg);
+ break;
+
default: /* unknown flag */
return PARAM_OPTION_UNKNOWN;
}
diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c
index 448fc7cb3..aa0c6203b 100644
--- a/src/tool_listhelp.c
+++ b/src/tool_listhelp.c
@@ -106,6 +106,18 @@ const struct helptxt helptext[] = {
{" --curves <algorithm list>",
"(EC) TLS key exchange algorithm(s) to request",
CURLHELP_TLS},
+ {" --signature-hashes <algorithm list>",
+ "TLS signature hash algorithm(s) to use",
+ CURLHELP_TLS},
+ {" --cert-compression <algorithm list>",
+ "TLS cert compressions algorithm(s) to use",
+ CURLHELP_TLS},
+ {" --no-tls-session-ticket",
+ "Disable the TLS session ticket extension",
+ CURLHELP_TLS},
+ {" --http2-pseudo-headers-order",
+ "Change the order of the HTTP2 pseudo headers",
+ CURLHELP_HTTP},
{"-d, --data <data>",
"HTTP POST data",
CURLHELP_IMPORTANT | CURLHELP_HTTP | CURLHELP_POST | CURLHELP_UPLOAD},
@@ -379,6 +391,9 @@ const struct helptxt helptext[] = {
{" --no-alpn",
"Disable the ALPN TLS extension",
CURLHELP_TLS | CURLHELP_HTTP},
+ {" --alps",
+ "Enable the ALPS TLS extension",
+ CURLHELP_TLS | CURLHELP_HTTP},
{"-N, --no-buffer",
"Disable buffering of the output stream",
CURLHELP_CURL},
diff --git a/src/tool_operate.c b/src/tool_operate.c
index fe2c43b55..7e487242e 100644
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@@ -1432,6 +1432,11 @@ static CURLcode single_transfer(struct GlobalConfig *global,
return result;
}
+ if(config->http2_pseudo_headers_order)
+ my_setopt_str(curl,
+ CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER,
+ config->http2_pseudo_headers_order);
+
} /* (built_in_protos & CURLPROTO_HTTP) */
my_setopt_str(curl, CURLOPT_FTPPORT, config->ftpport);
@@ -1520,6 +1525,14 @@ static CURLcode single_transfer(struct GlobalConfig *global,
if(config->ssl_ec_curves)
my_setopt_str(curl, CURLOPT_SSL_EC_CURVES, config->ssl_ec_curves);
+ if(config->ssl_sig_hash_algs)
+ my_setopt_str(curl, CURLOPT_SSL_SIG_HASH_ALGS,
+ config->ssl_sig_hash_algs);
+
+ if(config->ssl_cert_compression)
+ my_setopt_str(curl, CURLOPT_SSL_CERT_COMPRESSION,
+ config->ssl_cert_compression);
+
if(curlinfo->features & CURL_VERSION_SSL) {
/* Check if config->cert is a PKCS#11 URI and set the
* config->cert_type if necessary */
@@ -2061,6 +2074,14 @@ static CURLcode single_transfer(struct GlobalConfig *global,
my_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, 0L);
}
+ if(config->alps) {
+ my_setopt(curl, CURLOPT_SSL_ENABLE_ALPS, 1L);
+ }
+
+ if (config->noticket) {
+ my_setopt(curl, CURLOPT_SSL_ENABLE_TICKET, 0L);
+ }
+
/* new in 7.40.0, abstract support added in 7.53.0 */
if(config->unix_socket_path) {
if(config->abstract_unix_socket) {
diff --git a/src/tool_setopt.c b/src/tool_setopt.c
index 4c86eb321..eef8b187f 100644
--- a/src/tool_setopt.c
+++ b/src/tool_setopt.c
@@ -179,6 +179,7 @@ static const struct NameValue setopt_nv_CURLNONZERODEFAULTS[] = {
NV1(CURLOPT_SSL_VERIFYHOST, 1),
NV1(CURLOPT_SSL_ENABLE_NPN, 1),
NV1(CURLOPT_SSL_ENABLE_ALPN, 1),
+ NV1(CURLOPT_SSL_ENABLE_TICKET, 1),
NV1(CURLOPT_TCP_NODELAY, 1),
NV1(CURLOPT_PROXY_SSL_VERIFYPEER, 1),
NV1(CURLOPT_PROXY_SSL_VERIFYHOST, 1),
Reference in New Issue View Git Blame Copy Permalink