mirror of
https://github.com/lwthiker/curl-impersonate.git
synced 2025-08-04 10:52:29 +00:00

HTTP/2 includes various settings pertaining to stream priorities. Chrome and Firefox handle them differently, and this behavior was not mimicked in curl-impersonate well. With this commit, the stream settings set by curl-impersonate are identical to the real browsers. * With Chrome, the default stream weight is 256 and the "exclusive bit" is set to ON. * With Firefox, a complex tree of stream dependencies is created by default using PRIORITY frames. This behavior is now mimicked by curl-impersonate.
1343 lines
44 KiB
Diff
1343 lines
44 KiB
Diff
diff --git a/Makefile.am b/Makefile.am
|
|
index 3e55230ee..90b504b5c 100644
|
|
--- a/Makefile.am
|
|
+++ b/Makefile.am
|
|
@@ -192,13 +192,13 @@ CLEANFILES = $(VC6_LIBDSP) $(VC6_SRCDSP) $(VC7_LIBVCPROJ) $(VC7_SRCVCPROJ) \
|
|
$(VC11_LIBVCXPROJ) $(VC11_SRCVCXPROJ) $(VC12_LIBVCXPROJ) $(VC12_SRCVCXPROJ) \
|
|
$(VC14_LIBVCXPROJ) $(VC14_SRCVCXPROJ) $(VC15_LIBVCXPROJ) $(VC15_SRCVCXPROJ)
|
|
|
|
-bin_SCRIPTS = curl-config
|
|
+bin_SCRIPTS = curl-impersonate-ff-config
|
|
|
|
SUBDIRS = lib src
|
|
DIST_SUBDIRS = $(SUBDIRS) tests packages scripts include docs
|
|
|
|
pkgconfigdir = $(libdir)/pkgconfig
|
|
-pkgconfig_DATA = libcurl.pc
|
|
+pkgconfig_DATA = libcurl-impersonate-ff.pc
|
|
|
|
# List of files required to generate VC IDE .dsp, .vcproj and .vcxproj files
|
|
include lib/Makefile.inc
|
|
diff --git a/configure.ac b/configure.ac
|
|
index 63e320236..40f67f137 100644
|
|
--- a/configure.ac
|
|
+++ b/configure.ac
|
|
@@ -1331,7 +1331,8 @@ if test X"$OPT_BROTLI" != Xno; then
|
|
|
|
dnl if given with a prefix, we set -L and -I based on that
|
|
if test -n "$PREFIX_BROTLI"; then
|
|
- LIB_BROTLI="-lbrotlidec"
|
|
+ # curl-impersonate: Use static libbrotli
|
|
+ LIB_BROTLI="-lbrotlidec-static -lbrotlicommon-static"
|
|
LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff
|
|
CPP_BROTLI=-I${PREFIX_BROTLI}/include
|
|
DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff
|
|
@@ -1341,7 +1342,11 @@ if test X"$OPT_BROTLI" != Xno; then
|
|
CPPFLAGS="$CPPFLAGS $CPP_BROTLI"
|
|
LIBS="$LIB_BROTLI $LIBS"
|
|
|
|
- AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress)
|
|
+ AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress,
|
|
+ # curl-impersonate: Define 'action-if-found' explicitly to prevent
|
|
+ # -lbrotlidec from being added to LIBS (already added before)
|
|
+ AC_DEFINE(HAVE_LIBBROTLI, 1, [Define to 1 if libbrotli exists])
|
|
+ )
|
|
|
|
AC_CHECK_HEADERS(brotli/decode.h,
|
|
curl_brotli_msg="enabled (libbrotlidec)"
|
|
@@ -2573,15 +2578,15 @@ if test X"$want_nghttp2" != Xno; then
|
|
|
|
if test "$PKGCONFIG" != "no" ; then
|
|
LIB_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path])
|
|
- $PKGCONFIG --libs-only-l libnghttp2`
|
|
+ $PKGCONFIG --static --libs-only-l libnghttp2`
|
|
AC_MSG_NOTICE([-l is $LIB_H2])
|
|
|
|
CPP_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path]) dnl
|
|
- $PKGCONFIG --cflags-only-I libnghttp2`
|
|
+ $PKGCONFIG --static --cflags-only-I libnghttp2`
|
|
AC_MSG_NOTICE([-I is $CPP_H2])
|
|
|
|
LD_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path])
|
|
- $PKGCONFIG --libs-only-L libnghttp2`
|
|
+ $PKGCONFIG --static --libs-only-L libnghttp2`
|
|
AC_MSG_NOTICE([-L is $LD_H2])
|
|
|
|
LDFLAGS="$LDFLAGS $LD_H2"
|
|
@@ -4255,8 +4260,8 @@ AC_CONFIG_FILES([Makefile \
|
|
tests/unit/Makefile \
|
|
packages/Makefile \
|
|
packages/vms/Makefile \
|
|
- curl-config \
|
|
- libcurl.pc
|
|
+ curl-impersonate-ff-config:curl-config.in \
|
|
+ libcurl-impersonate-ff.pc:libcurl.pc.in
|
|
])
|
|
AC_OUTPUT
|
|
|
|
diff --git a/curl-config.in b/curl-config.in
|
|
index 8b4a29a9a..2c8b888d8 100644
|
|
--- a/curl-config.in
|
|
+++ b/curl-config.in
|
|
@@ -161,9 +161,9 @@ while test $# -gt 0; do
|
|
CURLLIBDIR=""
|
|
fi
|
|
if test "X@ENABLE_SHARED@" = "Xno"; then
|
|
- echo ${CURLLIBDIR}-lcurl @LIBCURL_LIBS@
|
|
+ echo ${CURLLIBDIR}-lcurl-impersonate-ff @LIBCURL_LIBS@
|
|
else
|
|
- echo ${CURLLIBDIR}-lcurl
|
|
+ echo ${CURLLIBDIR}-lcurl-impersonate-ff
|
|
fi
|
|
;;
|
|
--ssl-backends)
|
|
@@ -172,7 +172,7 @@ while test $# -gt 0; do
|
|
|
|
--static-libs)
|
|
if test "X@ENABLE_STATIC@" != "Xno" ; then
|
|
- echo @libdir@/libcurl.@libext@ @LDFLAGS@ @LIBCURL_LIBS@
|
|
+ echo @libdir@/libcurl-impersonate-ff.@libext@ @LDFLAGS@ @LIBCURL_LIBS@
|
|
else
|
|
echo "curl was built with static libraries disabled" >&2
|
|
exit 1
|
|
diff --git a/include/curl/curl.h b/include/curl/curl.h
|
|
index 7b69ce2d6..fe4bb36b9 100644
|
|
--- a/include/curl/curl.h
|
|
+++ b/include/curl/curl.h
|
|
@@ -2135,6 +2135,10 @@ typedef enum {
|
|
/* Set MIME option flags. */
|
|
CURLOPT(CURLOPT_MIME_OPTIONS, CURLOPTTYPE_LONG, 315),
|
|
|
|
+ /* curl-impersonate: A list of headers used by the impersonated browser.
|
|
+ * If given, merged with CURLOPT_HTTPHEADER. */
|
|
+ CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 316),
|
|
+
|
|
CURLOPT_LASTENTRY /* the last unused */
|
|
} CURLoption;
|
|
|
|
diff --git a/include/curl/easy.h b/include/curl/easy.h
|
|
index 2dbfb26b5..e0bf86169 100644
|
|
--- a/include/curl/easy.h
|
|
+++ b/include/curl/easy.h
|
|
@@ -41,6 +41,15 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...);
|
|
CURL_EXTERN CURLcode curl_easy_perform(CURL *curl);
|
|
CURL_EXTERN void curl_easy_cleanup(CURL *curl);
|
|
|
|
+/*
|
|
+ * curl-impersonate: Tell libcurl to impersonate a browser.
|
|
+ * This is a wrapper function that calls curl_easy_setopt()
|
|
+ * multiple times with all the parameters required. That's also why it was
|
|
+ * created as a separate API function and not just as another option to
|
|
+ * curl_easy_setopt().
|
|
+ */
|
|
+CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target);
|
|
+
|
|
/*
|
|
* NAME curl_easy_getinfo()
|
|
*
|
|
diff --git a/lib/Makefile.am b/lib/Makefile.am
|
|
index 769363941..6e2f1b829 100644
|
|
--- a/lib/Makefile.am
|
|
+++ b/lib/Makefile.am
|
|
@@ -32,7 +32,7 @@ EXTRA_DIST = Makefile.m32 config-win32.h config-win32ce.h \
|
|
firefox-db2pem.sh config-vxworks.h Makefile.vxworks checksrc.pl \
|
|
setup-win32.h .checksrc
|
|
|
|
-lib_LTLIBRARIES = libcurl.la
|
|
+lib_LTLIBRARIES = libcurl-impersonate-ff.la
|
|
|
|
if BUILD_UNITTESTS
|
|
noinst_LTLIBRARIES = libcurlu.la
|
|
@@ -84,43 +84,43 @@ AM_CPPFLAGS += -DBUILDING_LIBCURL
|
|
AM_LDFLAGS =
|
|
AM_CFLAGS =
|
|
|
|
-libcurl_la_CPPFLAGS_EXTRA =
|
|
-libcurl_la_LDFLAGS_EXTRA =
|
|
-libcurl_la_CFLAGS_EXTRA =
|
|
+libcurl_impersonate_ff_la_CPPFLAGS_EXTRA =
|
|
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA =
|
|
+libcurl_impersonate_ff_la_CFLAGS_EXTRA =
|
|
|
|
if CURL_LT_SHLIB_USE_VERSION_INFO
|
|
-libcurl_la_LDFLAGS_EXTRA += $(VERSIONINFO)
|
|
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += $(VERSIONINFO)
|
|
endif
|
|
|
|
if CURL_LT_SHLIB_USE_NO_UNDEFINED
|
|
-libcurl_la_LDFLAGS_EXTRA += -no-undefined
|
|
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -no-undefined
|
|
endif
|
|
|
|
if CURL_LT_SHLIB_USE_MIMPURE_TEXT
|
|
-libcurl_la_LDFLAGS_EXTRA += -mimpure-text
|
|
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -mimpure-text
|
|
endif
|
|
|
|
if CURL_LT_SHLIB_USE_VERSIONED_SYMBOLS
|
|
-libcurl_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers
|
|
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers
|
|
else
|
|
# if symbol-hiding is enabled, hide them!
|
|
if DOING_CURL_SYMBOL_HIDING
|
|
-libcurl_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*'
|
|
+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*'
|
|
endif
|
|
endif
|
|
|
|
if USE_CPPFLAG_CURL_STATICLIB
|
|
-libcurl_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB
|
|
+libcurl_impersonate_ff_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB
|
|
endif
|
|
|
|
if DOING_CURL_SYMBOL_HIDING
|
|
-libcurl_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS
|
|
-libcurl_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING)
|
|
+libcurl_impersonate_ff_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS
|
|
+libcurl_impersonate_ff_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING)
|
|
endif
|
|
|
|
-libcurl_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_la_CPPFLAGS_EXTRA)
|
|
-libcurl_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS)
|
|
-libcurl_la_CFLAGS = $(AM_CFLAGS) $(libcurl_la_CFLAGS_EXTRA)
|
|
+libcurl_impersonate_ff_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_impersonate_ff_la_CPPFLAGS_EXTRA)
|
|
+libcurl_impersonate_ff_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_impersonate_ff_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS)
|
|
+libcurl_impersonate_ff_la_CFLAGS = $(AM_CFLAGS) $(libcurl_impersonate_ff_la_CFLAGS_EXTRA)
|
|
|
|
libcurlu_la_CPPFLAGS = $(AM_CPPFLAGS) -DCURL_STATICLIB -DUNITTESTS
|
|
libcurlu_la_LDFLAGS = $(AM_LDFLAGS) -static $(LIBCURL_LIBS)
|
|
@@ -129,7 +129,7 @@ libcurlu_la_CFLAGS = $(AM_CFLAGS)
|
|
# Makefile.inc provides the CSOURCES and HHEADERS defines
|
|
include Makefile.inc
|
|
|
|
-libcurl_la_SOURCES = $(CSOURCES) $(HHEADERS)
|
|
+libcurl_impersonate_ff_la_SOURCES = $(CSOURCES) $(HHEADERS)
|
|
libcurlu_la_SOURCES = $(CSOURCES) $(HHEADERS)
|
|
|
|
CHECKSRC = $(CS_$(V))
|
|
diff --git a/lib/easy.c b/lib/easy.c
|
|
index 20293a710..66730b5fc 100644
|
|
--- a/lib/easy.c
|
|
+++ b/lib/easy.c
|
|
@@ -80,6 +80,7 @@
|
|
#include "dynbuf.h"
|
|
#include "altsvc.h"
|
|
#include "hsts.h"
|
|
+#include "strcase.h"
|
|
|
|
/* The last 3 #include files should be in this order */
|
|
#include "curl_printf.h"
|
|
@@ -282,6 +283,237 @@ void curl_global_cleanup(void)
|
|
init_flags = 0;
|
|
}
|
|
|
|
+/*
|
|
+ * curl-impersonate: Options to be set for each supported target browser.
|
|
+ * Note: this does not include the HTTP headers, which are handled separately
|
|
+ * in Curl_http().
|
|
+ */
|
|
+#define IMPERSONATE_MAX_HEADERS 32
|
|
+static const struct impersonate_opts {
|
|
+ const char *target;
|
|
+ int httpversion;
|
|
+ int ssl_version;
|
|
+ const char *ciphers;
|
|
+ const char *http_headers[IMPERSONATE_MAX_HEADERS];
|
|
+ /* Other TLS options will come here in the future once they are
|
|
+ * configurable through curl_easy_setopt() */
|
|
+} impersonations[] = {
|
|
+ {
|
|
+ .target = "ff91esr",
|
|
+ .httpversion = CURL_HTTP_VERSION_2_0,
|
|
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
|
|
+ .ciphers =
|
|
+ "aes_128_gcm_sha_256,"
|
|
+ "chacha20_poly1305_sha_256,"
|
|
+ "aes_256_gcm_sha_384,"
|
|
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
|
|
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
|
|
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
|
|
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
|
|
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
|
|
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
|
|
+ "ecdhe_ecdsa_aes_256_sha,"
|
|
+ "ecdhe_ecdsa_aes_128_sha,"
|
|
+ "ecdhe_rsa_aes_128_sha,"
|
|
+ "ecdhe_rsa_aes_256_sha,"
|
|
+ "rsa_aes_128_gcm_sha_256,"
|
|
+ "rsa_aes_256_gcm_sha_384,"
|
|
+ "rsa_aes_128_sha,"
|
|
+ "rsa_aes_256_sha,"
|
|
+ "rsa_3des_ede_cbc_sha",
|
|
+ .http_headers = {
|
|
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0",
|
|
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
+ "Accept-Language: en-US,en;q=0.5",
|
|
+ "Accept-Encoding: gzip, deflate, br",
|
|
+ "Upgrade-Insecure-Requests: 1",
|
|
+ "Sec-Fetch-Dest: document",
|
|
+ "Sec-Fetch-Mode: navigate",
|
|
+ "Sec-Fetch-Site: none",
|
|
+ "Sec-Fetch-User: ?1",
|
|
+ "TE: Trailers"
|
|
+ }
|
|
+ },
|
|
+ {
|
|
+ .target = "ff95",
|
|
+ .httpversion = CURL_HTTP_VERSION_2_0,
|
|
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
|
|
+ .ciphers =
|
|
+ "aes_128_gcm_sha_256,"
|
|
+ "chacha20_poly1305_sha_256,"
|
|
+ "aes_256_gcm_sha_384,"
|
|
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
|
|
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
|
|
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
|
|
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
|
|
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
|
|
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
|
|
+ "ecdhe_ecdsa_aes_256_sha,"
|
|
+ "ecdhe_ecdsa_aes_128_sha,"
|
|
+ "ecdhe_rsa_aes_128_sha,"
|
|
+ "ecdhe_rsa_aes_256_sha,"
|
|
+ "rsa_aes_128_gcm_sha_256,"
|
|
+ "rsa_aes_256_gcm_sha_384,"
|
|
+ "rsa_aes_128_sha,"
|
|
+ "rsa_aes_256_sha",
|
|
+ .http_headers = {
|
|
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0",
|
|
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
+ "Accept-Language: en-US,en;q=0.5",
|
|
+ "Accept-Encoding: gzip, deflate, br",
|
|
+ "Upgrade-Insecure-Requests: 1",
|
|
+ "Sec-Fetch-Dest: document",
|
|
+ "Sec-Fetch-Mode: navigate",
|
|
+ "Sec-Fetch-Site: none",
|
|
+ "Sec-Fetch-User: ?1",
|
|
+ "TE: Trailers"
|
|
+ }
|
|
+ },
|
|
+ {
|
|
+ .target = "ff98",
|
|
+ .httpversion = CURL_HTTP_VERSION_2_0,
|
|
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
|
|
+ .ciphers =
|
|
+ "aes_128_gcm_sha_256,"
|
|
+ "chacha20_poly1305_sha_256,"
|
|
+ "aes_256_gcm_sha_384,"
|
|
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
|
|
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
|
|
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
|
|
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
|
|
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
|
|
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
|
|
+ "ecdhe_ecdsa_aes_256_sha,"
|
|
+ "ecdhe_ecdsa_aes_128_sha,"
|
|
+ "ecdhe_rsa_aes_128_sha,"
|
|
+ "ecdhe_rsa_aes_256_sha,"
|
|
+ "rsa_aes_128_gcm_sha_256,"
|
|
+ "rsa_aes_256_gcm_sha_384,"
|
|
+ "rsa_aes_128_sha,"
|
|
+ "rsa_aes_256_sha",
|
|
+ .http_headers = {
|
|
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:98.0) Gecko/20100101 Firefox/98.0",
|
|
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
+ "Accept-Language: en-US,en;q=0.5",
|
|
+ "Accept-Encoding: gzip, deflate, br",
|
|
+ "Upgrade-Insecure-Requests: 1",
|
|
+ "Sec-Fetch-Dest: document",
|
|
+ "Sec-Fetch-Mode: navigate",
|
|
+ "Sec-Fetch-Site: none",
|
|
+ "Sec-Fetch-User: ?1",
|
|
+ "TE: Trailers"
|
|
+ }
|
|
+ },
|
|
+ {
|
|
+ .target = "ff100",
|
|
+ .httpversion = CURL_HTTP_VERSION_2_0,
|
|
+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT,
|
|
+ .ciphers =
|
|
+ "aes_128_gcm_sha_256,"
|
|
+ "chacha20_poly1305_sha_256,"
|
|
+ "aes_256_gcm_sha_384,"
|
|
+ "ecdhe_ecdsa_aes_128_gcm_sha_256,"
|
|
+ "ecdhe_rsa_aes_128_gcm_sha_256,"
|
|
+ "ecdhe_ecdsa_chacha20_poly1305_sha_256,"
|
|
+ "ecdhe_rsa_chacha20_poly1305_sha_256,"
|
|
+ "ecdhe_ecdsa_aes_256_gcm_sha_384,"
|
|
+ "ecdhe_rsa_aes_256_gcm_sha_384,"
|
|
+ "ecdhe_ecdsa_aes_256_sha,"
|
|
+ "ecdhe_ecdsa_aes_128_sha,"
|
|
+ "ecdhe_rsa_aes_128_sha,"
|
|
+ "ecdhe_rsa_aes_256_sha,"
|
|
+ "rsa_aes_128_gcm_sha_256,"
|
|
+ "rsa_aes_256_gcm_sha_384,"
|
|
+ "rsa_aes_128_sha,"
|
|
+ "rsa_aes_256_sha",
|
|
+ .http_headers = {
|
|
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0",
|
|
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
+ "Accept-Language: en-US,en;q=0.5",
|
|
+ "Accept-Encoding: gzip, deflate, br",
|
|
+ "Upgrade-Insecure-Requests: 1",
|
|
+ "Sec-Fetch-Dest: document",
|
|
+ "Sec-Fetch-Mode: navigate",
|
|
+ "Sec-Fetch-Site: none",
|
|
+ "Sec-Fetch-User: ?1",
|
|
+ "TE: Trailers"
|
|
+ }
|
|
+ }
|
|
+};
|
|
+
|
|
+#define NUM_IMPERSONATIONS \
|
|
+ sizeof(impersonations) / sizeof(impersonations[0])
|
|
+
|
|
+/*
|
|
+ * curl-impersonate:
|
|
+ * Call curl_easy_setopt() with all the needed options as defined in the
|
|
+ * 'impersonations' array.
|
|
+ * */
|
|
+CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target)
|
|
+{
|
|
+ int i;
|
|
+ int ret;
|
|
+ const struct impersonate_opts *opts = NULL;
|
|
+ struct curl_slist *headers = NULL;
|
|
+
|
|
+ for(i = 0; i < NUM_IMPERSONATIONS; i++) {
|
|
+ if (Curl_strncasecompare(target,
|
|
+ impersonations[i].target,
|
|
+ strlen(impersonations[i].target))) {
|
|
+ opts = &impersonations[i];
|
|
+ break;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ if(!opts) {
|
|
+ DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n",
|
|
+ target));
|
|
+ return CURLE_BAD_FUNCTION_ARGUMENT;
|
|
+ }
|
|
+
|
|
+ if(opts->httpversion != CURL_HTTP_VERSION_NONE) {
|
|
+ ret = curl_easy_setopt(data, CURLOPT_HTTP_VERSION, opts->httpversion);
|
|
+ if(ret)
|
|
+ return ret;
|
|
+ }
|
|
+
|
|
+ if (opts->ssl_version != CURL_SSLVERSION_DEFAULT) {
|
|
+ ret = curl_easy_setopt(data, CURLOPT_SSLVERSION, opts->ssl_version);
|
|
+ if(ret)
|
|
+ return ret;
|
|
+ }
|
|
+
|
|
+ if(opts->ciphers) {
|
|
+ ret = curl_easy_setopt(data, CURLOPT_SSL_CIPHER_LIST, opts->ciphers);
|
|
+ if (ret)
|
|
+ return ret;
|
|
+ }
|
|
+
|
|
+ /* Build a linked list out of the static array of headers. */
|
|
+ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) {
|
|
+ if(opts->http_headers[i]) {
|
|
+ headers = curl_slist_append(headers, opts->http_headers[i]);
|
|
+ if(!headers) {
|
|
+ return CURLE_OUT_OF_MEMORY;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+
|
|
+ if(headers) {
|
|
+ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers);
|
|
+ curl_slist_free_all(headers);
|
|
+ if(ret)
|
|
+ return ret;
|
|
+ }
|
|
+
|
|
+ /* Always enable all supported compressions. */
|
|
+ ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, "");
|
|
+ if(ret)
|
|
+ return ret;
|
|
+
|
|
+ return CURLE_OK;
|
|
+}
|
|
+
|
|
/*
|
|
* curl_easy_init() is the external interface to alloc, setup and init an
|
|
* easy handle that is returned. If anything goes wrong, NULL is returned.
|
|
@@ -290,6 +522,7 @@ struct Curl_easy *curl_easy_init(void)
|
|
{
|
|
CURLcode result;
|
|
struct Curl_easy *data;
|
|
+ char *target;
|
|
|
|
/* Make sure we inited the global SSL stuff */
|
|
if(!initialized) {
|
|
@@ -308,6 +541,22 @@ struct Curl_easy *curl_easy_init(void)
|
|
return NULL;
|
|
}
|
|
|
|
+ /*
|
|
+ * curl-impersonate: Hook into curl_easy_init() to set the required options
|
|
+ * from an environment variable.
|
|
+ * This is a bit hacky but allows seamless integration of libcurl-impersonate
|
|
+ * without code modifications to the app.
|
|
+ */
|
|
+ target = curl_getenv("CURL_IMPERSONATE");
|
|
+ if(target) {
|
|
+ result = curl_easy_impersonate(data, target);
|
|
+ free(target);
|
|
+ if(result) {
|
|
+ Curl_close(&data);
|
|
+ return NULL;
|
|
+ }
|
|
+ }
|
|
+
|
|
return data;
|
|
}
|
|
|
|
@@ -878,6 +1127,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
|
|
outcurl->state.referer_alloc = TRUE;
|
|
}
|
|
|
|
+ if(data->state.base_headers) {
|
|
+ outcurl->state.base_headers =
|
|
+ Curl_slist_duplicate(data->state.base_headers);
|
|
+ if(!outcurl->state.base_headers)
|
|
+ goto fail;
|
|
+ }
|
|
+
|
|
/* Reinitialize an SSL engine for the new handle
|
|
* note: the engine name has already been copied by dupset */
|
|
if(outcurl->set.str[STRING_SSL_ENGINE]) {
|
|
@@ -967,6 +1223,8 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
|
|
*/
|
|
void curl_easy_reset(struct Curl_easy *data)
|
|
{
|
|
+ char *target;
|
|
+
|
|
Curl_free_request_state(data);
|
|
|
|
/* zero out UserDefined data: */
|
|
@@ -991,6 +1249,12 @@ void curl_easy_reset(struct Curl_easy *data)
|
|
#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH)
|
|
Curl_http_auth_cleanup_digest(data);
|
|
#endif
|
|
+
|
|
+ target = curl_getenv("CURL_IMPERSONATE");
|
|
+ if(target) {
|
|
+ curl_easy_impersonate(data, target);
|
|
+ free(target);
|
|
+ }
|
|
}
|
|
|
|
/*
|
|
diff --git a/lib/easyoptions.c b/lib/easyoptions.c
|
|
index 04871ad1e..cd5998146 100644
|
|
--- a/lib/easyoptions.c
|
|
+++ b/lib/easyoptions.c
|
|
@@ -130,6 +130,7 @@ struct curl_easyoption Curl_easyopts[] = {
|
|
{"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0},
|
|
{"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0},
|
|
{"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0},
|
|
+ {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0},
|
|
{"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0},
|
|
{"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0},
|
|
{"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0},
|
|
diff --git a/lib/http.c b/lib/http.c
|
|
index f08a343e3..2bbce4b23 100644
|
|
--- a/lib/http.c
|
|
+++ b/lib/http.c
|
|
@@ -84,6 +84,7 @@
|
|
#include "altsvc.h"
|
|
#include "hsts.h"
|
|
#include "c-hyper.h"
|
|
+#include "slist.h"
|
|
|
|
/* The last 3 #include files should be in this order */
|
|
#include "curl_printf.h"
|
|
@@ -1795,6 +1796,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
|
|
int numlists = 1; /* by default */
|
|
int i;
|
|
|
|
+ /*
|
|
+ * curl-impersonate: Use the merged list of headers if it exists (i.e. when
|
|
+ * the CURLOPT_HTTPBASEHEADER option was set.
|
|
+ */
|
|
+ struct curl_slist *noproxyheaders =
|
|
+ (data->state.merged_headers ?
|
|
+ data->state.merged_headers :
|
|
+ data->set.headers);
|
|
+
|
|
#ifndef CURL_DISABLE_PROXY
|
|
enum proxy_use proxy;
|
|
|
|
@@ -1806,10 +1816,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
|
|
|
|
switch(proxy) {
|
|
case HEADER_SERVER:
|
|
- h[0] = data->set.headers;
|
|
+ h[0] = noproxyheaders;
|
|
break;
|
|
case HEADER_PROXY:
|
|
- h[0] = data->set.headers;
|
|
+ h[0] = noproxyheaders;
|
|
if(data->set.sep_headers) {
|
|
h[1] = data->set.proxyheaders;
|
|
numlists++;
|
|
@@ -1819,12 +1829,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data,
|
|
if(data->set.sep_headers)
|
|
h[0] = data->set.proxyheaders;
|
|
else
|
|
- h[0] = data->set.headers;
|
|
+ h[0] = noproxyheaders;
|
|
break;
|
|
}
|
|
#else
|
|
(void)is_connect;
|
|
- h[0] = data->set.headers;
|
|
+ h[0] = noproxyheaders;
|
|
#endif
|
|
|
|
/* loop through one or two lists */
|
|
@@ -2059,6 +2069,92 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn,
|
|
*reqp = httpreq;
|
|
}
|
|
|
|
+/*
|
|
+ * curl-impersonate:
|
|
+ * Create a new linked list of headers.
|
|
+ * The new list is a merge between the "base" headers and the application given
|
|
+ * headers. The "base" headers contain curl-impersonate's list of headers
|
|
+ * used by default by the impersonated browser.
|
|
+ *
|
|
+ * The application given headers will override the "base" headers if supplied.
|
|
+ */
|
|
+CURLcode Curl_http_merge_headers(struct Curl_easy *data)
|
|
+{
|
|
+ int i;
|
|
+ int ret;
|
|
+ struct curl_slist *head;
|
|
+ struct curl_slist *dup = NULL;
|
|
+ struct curl_slist *new_list = NULL;
|
|
+
|
|
+ if (!data->state.base_headers)
|
|
+ return CURLE_OK;
|
|
+
|
|
+ /* Duplicate the list for temporary use. */
|
|
+ if (data->set.headers) {
|
|
+ dup = Curl_slist_duplicate(data->set.headers);
|
|
+ if(!dup)
|
|
+ return CURLE_OUT_OF_MEMORY;
|
|
+ }
|
|
+
|
|
+ for(head = data->state.base_headers; head; head = head->next) {
|
|
+ char *sep;
|
|
+ size_t prefix_len;
|
|
+ bool found = FALSE;
|
|
+ struct curl_slist *head2;
|
|
+
|
|
+ sep = strchr(head->data, ':');
|
|
+ if(!sep)
|
|
+ continue;
|
|
+
|
|
+ prefix_len = sep - head->data;
|
|
+
|
|
+ /* Check if this header was added by the application. */
|
|
+ for(head2 = dup; head2; head2 = head2->next) {
|
|
+ if(head2->data &&
|
|
+ strncasecompare(head2->data, head->data, prefix_len) &&
|
|
+ Curl_headersep(head2->data[prefix_len]) ) {
|
|
+ new_list = curl_slist_append(new_list, head2->data);
|
|
+ /* Free and set to NULL to mark that it's been added. */
|
|
+ Curl_safefree(head2->data);
|
|
+ found = TRUE;
|
|
+ break;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ if (!found) {
|
|
+ new_list = curl_slist_append(new_list, head->data);
|
|
+ }
|
|
+
|
|
+ if (!new_list) {
|
|
+ ret = CURLE_OUT_OF_MEMORY;
|
|
+ goto fail;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ /* Now go over any additional application-supplied headers. */
|
|
+ for(head = dup; head; head = head->next) {
|
|
+ if(head->data) {
|
|
+ new_list = curl_slist_append(new_list, head->data);
|
|
+ if(!new_list) {
|
|
+ ret = CURLE_OUT_OF_MEMORY;
|
|
+ goto fail;
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+
|
|
+ curl_slist_free_all(dup);
|
|
+ /* Save the new, merged list separately, so it can be freed later. */
|
|
+ curl_slist_free_all(data->state.merged_headers);
|
|
+ data->state.merged_headers = new_list;
|
|
+
|
|
+ return CURLE_OK;
|
|
+
|
|
+fail:
|
|
+ Curl_safefree(dup);
|
|
+ curl_slist_free_all(new_list);
|
|
+ return ret;
|
|
+}
|
|
+
|
|
CURLcode Curl_http_useragent(struct Curl_easy *data)
|
|
{
|
|
/* The User-Agent string might have been allocated in url.c already, because
|
|
@@ -3063,6 +3159,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
|
|
http = data->req.p.http;
|
|
DEBUGASSERT(http);
|
|
|
|
+ /* curl-impersonate: Add HTTP headers to impersonate real browsers. */
|
|
+ result = Curl_http_merge_headers(data);
|
|
+ if (result)
|
|
+ return result;
|
|
+
|
|
result = Curl_http_host(data, conn);
|
|
if(result)
|
|
return result;
|
|
diff --git a/lib/http2.c b/lib/http2.c
|
|
index e74400a4c..1dd2593a5 100644
|
|
--- a/lib/http2.c
|
|
+++ b/lib/http2.c
|
|
@@ -56,7 +56,7 @@
|
|
#define NGHTTP2_HAS_SET_LOCAL_WINDOW_SIZE 1
|
|
#endif
|
|
|
|
-#define HTTP2_HUGE_WINDOW_SIZE (32 * 1024 * 1024) /* 32 MB */
|
|
+#define HTTP2_HUGE_WINDOW_SIZE (12 * 1024 * 1024) /* 12 MB */
|
|
|
|
#ifdef DEBUG_HTTP2
|
|
#define H2BUGF(x) x
|
|
@@ -75,13 +75,20 @@ static int h2_process_pending_input(struct Curl_easy *data,
|
|
struct http_conn *httpc,
|
|
CURLcode *err);
|
|
|
|
+
|
|
+/*
|
|
+ * curl-impersonate: Set the HTTP/2 stream weight to the one used by Firefox
|
|
+ * by default to fetch html resources.
|
|
+ */
|
|
+#define FIREFOX_DEFAULT_STREAM_WEIGHT (42)
|
|
+
|
|
/*
|
|
* Curl_http2_init_state() is called when the easy handle is created and
|
|
* allows for HTTP/2 specific init of state.
|
|
*/
|
|
void Curl_http2_init_state(struct UrlState *state)
|
|
{
|
|
- state->stream_weight = NGHTTP2_DEFAULT_WEIGHT;
|
|
+ state->stream_weight = FIREFOX_DEFAULT_STREAM_WEIGHT;
|
|
}
|
|
|
|
/*
|
|
@@ -90,7 +97,7 @@ void Curl_http2_init_state(struct UrlState *state)
|
|
*/
|
|
void Curl_http2_init_userset(struct UserDefined *set)
|
|
{
|
|
- set->stream_weight = NGHTTP2_DEFAULT_WEIGHT;
|
|
+ set->stream_weight = FIREFOX_DEFAULT_STREAM_WEIGHT;
|
|
}
|
|
|
|
static int http2_getsock(struct Curl_easy *data,
|
|
@@ -1193,14 +1200,18 @@ static void populate_settings(struct Curl_easy *data,
|
|
{
|
|
nghttp2_settings_entry *iv = httpc->local_settings;
|
|
|
|
- iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
|
|
- iv[0].value = Curl_multi_max_concurrent_streams(data->multi);
|
|
+ /* curl-impersonate: Align HTTP/2 settings to Firefox's */
|
|
+ iv[0].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE;
|
|
+ iv[0].value = 0x10000;
|
|
|
|
iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
|
|
- iv[1].value = HTTP2_HUGE_WINDOW_SIZE;
|
|
+ iv[1].value = 0x20000;
|
|
|
|
- iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
|
|
- iv[2].value = data->multi->push_cb != NULL;
|
|
+ iv[2].settings_id = NGHTTP2_SETTINGS_MAX_FRAME_SIZE;
|
|
+ iv[2].value = 0x4000;
|
|
+
|
|
+ // iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
|
|
+ // iv[2].value = data->multi->push_cb != NULL;
|
|
|
|
httpc->local_settings_num = 3;
|
|
}
|
|
@@ -1566,12 +1577,18 @@ static ssize_t http2_handle_stream_close(struct connectdata *conn,
|
|
* struct.
|
|
*/
|
|
|
|
+/*
|
|
+ * curl-impersonate: By default Firefox uses stream 13 as the "parent" of the
|
|
+ * stream that fetches the main html resource of the web page.
|
|
+ */
|
|
+#define FIREFOX_DEFAULT_STREAM_DEP (13)
|
|
+
|
|
static void h2_pri_spec(struct Curl_easy *data,
|
|
nghttp2_priority_spec *pri_spec)
|
|
{
|
|
struct HTTP *depstream = (data->set.stream_depends_on?
|
|
data->set.stream_depends_on->req.p.http:NULL);
|
|
- int32_t depstream_id = depstream? depstream->stream_id:0;
|
|
+ int32_t depstream_id = depstream? depstream->stream_id:FIREFOX_DEFAULT_STREAM_DEP;
|
|
nghttp2_priority_spec_init(pri_spec, depstream_id, data->set.stream_weight,
|
|
data->set.stream_depends_e);
|
|
data->state.stream_weight = data->set.stream_weight;
|
|
@@ -1818,7 +1835,8 @@ static ssize_t http2_recv(struct Curl_easy *data, int sockindex,
|
|
|
|
/* Index where :authority header field will appear in request header
|
|
field list. */
|
|
-#define AUTHORITY_DST_IDX 3
|
|
+/* curl-impersonate: Put the ":authority" header in the second place. */
|
|
+#define AUTHORITY_DST_IDX 2
|
|
|
|
/* USHRT_MAX is 65535 == 0xffff */
|
|
#define HEADER_OVERFLOW(x) \
|
|
@@ -2276,6 +2294,73 @@ CURLcode Curl_http2_setup(struct Curl_easy *data,
|
|
return CURLE_OK;
|
|
}
|
|
|
|
+/*
|
|
+ * curl-impersonate: Start with stream id 15 as Firefox does.
|
|
+ */
|
|
+#define FIREFOX_DEFAULT_STREAM_ID (15)
|
|
+
|
|
+static CURLcode http2_set_stream_priority(struct Curl_easy *data,
|
|
+ int32_t stream_id,
|
|
+ int32_t dep_stream_id,
|
|
+ int32_t weight)
|
|
+{
|
|
+ int rv;
|
|
+ nghttp2_priority_spec pri_spec;
|
|
+ struct connectdata *conn = data->conn;
|
|
+ struct http_conn *httpc = &conn->proto.httpc;
|
|
+
|
|
+ nghttp2_priority_spec_init(&pri_spec, dep_stream_id, weight, 0);
|
|
+ rv = nghttp2_submit_priority(httpc->h2, NGHTTP2_FLAG_NONE,
|
|
+ stream_id, &pri_spec);
|
|
+ if(rv) {
|
|
+ failf(data, "nghttp2_submit_priority() failed: %s(%d)",
|
|
+ nghttp2_strerror(rv), rv);
|
|
+ return CURLE_HTTP2;
|
|
+ }
|
|
+
|
|
+ return CURLE_OK;
|
|
+}
|
|
+
|
|
+
|
|
+/*
|
|
+ * curl-impersonate: Firefox uses an elaborate scheme of http/2 streams to
|
|
+ * split the load for html/js/css/images. It builds a tree of streams with
|
|
+ * different weights (priorities) by default and communicates this to the
|
|
+ * server. Imitate that behavior.
|
|
+ */
|
|
+static CURLcode http2_set_stream_priorities(struct Curl_easy *data)
|
|
+{
|
|
+ CURLcode result;
|
|
+ struct connectdata *conn = data->conn;
|
|
+ struct http_conn *httpc = &conn->proto.httpc;
|
|
+
|
|
+ result = http2_set_stream_priority(data, 3, 0, 201);
|
|
+ if(result)
|
|
+ return result;
|
|
+
|
|
+ result = http2_set_stream_priority(data, 5, 0, 101);
|
|
+ if(result)
|
|
+ return result;
|
|
+
|
|
+ result = http2_set_stream_priority(data, 7, 0, 0);
|
|
+ if(result)
|
|
+ return result;
|
|
+
|
|
+ result = http2_set_stream_priority(data, 9, 7, 0);
|
|
+ if(result)
|
|
+ return result;
|
|
+
|
|
+ result = http2_set_stream_priority(data, 11, 3, 0);
|
|
+ if(result)
|
|
+ return result;
|
|
+
|
|
+ result = http2_set_stream_priority(data, 13, 0, 241);
|
|
+ if(result)
|
|
+ return result;
|
|
+
|
|
+ return CURLE_OK;
|
|
+}
|
|
+
|
|
CURLcode Curl_http2_switched(struct Curl_easy *data,
|
|
const char *mem, size_t nread)
|
|
{
|
|
@@ -2284,6 +2369,7 @@ CURLcode Curl_http2_switched(struct Curl_easy *data,
|
|
struct http_conn *httpc = &conn->proto.httpc;
|
|
int rv;
|
|
struct HTTP *stream = data->req.p.http;
|
|
+ nghttp2_priority_spec pri_spec;
|
|
|
|
result = Curl_http2_setup(data, conn);
|
|
if(result)
|
|
@@ -2338,6 +2424,13 @@ CURLcode Curl_http2_switched(struct Curl_easy *data,
|
|
return CURLE_HTTP2;
|
|
}
|
|
|
|
+ result = http2_set_stream_priorities(data);
|
|
+ if(result)
|
|
+ return result;
|
|
+
|
|
+ /* Best effort to set the request's stream id to 15, like Firefox does. */
|
|
+ nghttp2_session_set_next_stream_id(httpc->h2, FIREFOX_DEFAULT_STREAM_ID);
|
|
+
|
|
/* we are going to copy mem to httpc->inbuf. This is required since
|
|
mem is part of buffer pointed by stream->mem, and callbacks
|
|
called by nghttp2_session_mem_recv() will write stream specific
|
|
diff --git a/lib/setopt.c b/lib/setopt.c
|
|
index 599ed5d99..1baa48e70 100644
|
|
--- a/lib/setopt.c
|
|
+++ b/lib/setopt.c
|
|
@@ -48,6 +48,7 @@
|
|
#include "multiif.h"
|
|
#include "altsvc.h"
|
|
#include "hsts.h"
|
|
+#include "slist.h"
|
|
|
|
/* The last 3 #include files should be in this order */
|
|
#include "curl_printf.h"
|
|
@@ -688,6 +689,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
|
|
va_arg(param, char *));
|
|
break;
|
|
|
|
+ case CURLOPT_HTTPBASEHEADER:
|
|
+ /*
|
|
+ * curl-impersonate:
|
|
+ * Set a list of "base" headers. These will be merged with any headers
|
|
+ * set by CURLOPT_HTTPHEADER. curl-impersonate uses this option in order
|
|
+ * to set a list of default browser headers.
|
|
+ *
|
|
+ * Unlike CURLOPT_HTTPHEADER,
|
|
+ * the list is copied and can be immediately freed by the user.
|
|
+ */
|
|
+ curl_slist_free_all(data->state.base_headers);
|
|
+ data->state.base_headers = \
|
|
+ Curl_slist_duplicate(va_arg(param, struct curl_slist *));
|
|
+ if (!data->state.base_headers)
|
|
+ result = CURLE_OUT_OF_MEMORY;
|
|
+ break;
|
|
+
|
|
case CURLOPT_HTTPHEADER:
|
|
/*
|
|
* Set a list with HTTP headers to use (or replace internals with)
|
|
diff --git a/lib/transfer.c b/lib/transfer.c
|
|
index 22704fa15..1e100140c 100644
|
|
--- a/lib/transfer.c
|
|
+++ b/lib/transfer.c
|
|
@@ -102,7 +102,15 @@ char *Curl_checkheaders(const struct Curl_easy *data,
|
|
DEBUGASSERT(thislen);
|
|
DEBUGASSERT(thisheader[thislen-1] != ':');
|
|
|
|
- for(head = data->set.headers; head; head = head->next) {
|
|
+ /*
|
|
+ * curl-impersonate:
|
|
+ * Check if we have overriden the user-supplied list of headers.
|
|
+ */
|
|
+ head = data->set.headers;
|
|
+ if (data->state.merged_headers)
|
|
+ head = data->state.merged_headers;
|
|
+
|
|
+ for(; head; head = head->next) {
|
|
if(strncasecompare(head->data, thisheader, thislen) &&
|
|
Curl_headersep(head->data[thislen]) )
|
|
return head->data;
|
|
diff --git a/lib/url.c b/lib/url.c
|
|
index 9f1013554..f0f266797 100644
|
|
--- a/lib/url.c
|
|
+++ b/lib/url.c
|
|
@@ -469,6 +469,11 @@ CURLcode Curl_close(struct Curl_easy **datap)
|
|
Curl_safefree(data->state.aptr.proxyuser);
|
|
Curl_safefree(data->state.aptr.proxypasswd);
|
|
|
|
+ /* curl-impersonate: Free the list set by CURLOPT_HTTPBASEHEADER. */
|
|
+ curl_slist_free_all(data->state.base_headers);
|
|
+ /* curl-impersonate: Free the dynamic list of headers. */
|
|
+ curl_slist_free_all(data->state.merged_headers);
|
|
+
|
|
#ifndef CURL_DISABLE_DOH
|
|
if(data->req.doh) {
|
|
Curl_dyn_free(&data->req.doh->probe[0].serverdoh);
|
|
diff --git a/lib/urldata.h b/lib/urldata.h
|
|
index cc9c88870..a35a20e10 100644
|
|
--- a/lib/urldata.h
|
|
+++ b/lib/urldata.h
|
|
@@ -1421,6 +1421,19 @@ struct UrlState {
|
|
CURLcode hresult; /* used to pass return codes back from hyper callbacks */
|
|
#endif
|
|
|
|
+ /*
|
|
+ * curl-impersonate:
|
|
+ * List of "base" headers set by CURLOPT_HTTPBASEHEADER.
|
|
+ */
|
|
+ struct curl_slist *base_headers;
|
|
+ /*
|
|
+ * curl-impersonate:
|
|
+ * Dynamically-constructed list of HTTP headers.
|
|
+ * This list is a merge of the default HTTP headers needed to impersonate a
|
|
+ * browser, together with any user-supplied headers.
|
|
+ */
|
|
+ struct curl_slist *merged_headers;
|
|
+
|
|
/* Dynamically allocated strings, MUST be freed before this struct is
|
|
killed. */
|
|
struct dynamically_allocated_data {
|
|
diff --git a/lib/vtls/nss.c b/lib/vtls/nss.c
|
|
index 2b44f0512..eec2bf76f 100644
|
|
--- a/lib/vtls/nss.c
|
|
+++ b/lib/vtls/nss.c
|
|
@@ -143,6 +143,7 @@ static const struct cipher_s cipherlist[] = {
|
|
{"dhe_dss_3des_sha", SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA},
|
|
{"dhe_rsa_des_sha", SSL_DHE_RSA_WITH_DES_CBC_SHA},
|
|
{"dhe_dss_des_sha", SSL_DHE_DSS_WITH_DES_CBC_SHA},
|
|
+ {"rsa_3des_ede_cbc_sha", TLS_RSA_WITH_3DES_EDE_CBC_SHA},
|
|
/* TLS 1.0: Exportable 56-bit Cipher Suites. */
|
|
{"rsa_des_56_sha", TLS_RSA_EXPORT1024_WITH_DES_CBC_SHA},
|
|
{"rsa_rc4_56_sha", TLS_RSA_EXPORT1024_WITH_RC4_56_SHA},
|
|
@@ -378,6 +379,95 @@ static SECStatus set_ciphers(struct Curl_easy *data, PRFileDesc *model,
|
|
return SECSuccess;
|
|
}
|
|
|
|
+/* See nsSSLIOLayerSetOptions@nsNSSIOLayer.cpp, Firefox source code */
|
|
+const SSLNamedGroup named_groups[] = {
|
|
+ ssl_grp_ec_curve25519, ssl_grp_ec_secp256r1, ssl_grp_ec_secp384r1,
|
|
+ ssl_grp_ec_secp521r1, ssl_grp_ffdhe_2048, ssl_grp_ffdhe_3072};
|
|
+
|
|
+#define NUM_OF_NAMED_GROUPS sizeof(named_groups)/sizeof(named_groups[0])
|
|
+
|
|
+static SECStatus set_named_groups(PRFileDesc *model)
|
|
+{
|
|
+ /* This aligns TLS extension 10 (supported_groups) to what Firefox does. */
|
|
+ return SSL_NamedGroupConfig(model, named_groups, NUM_OF_NAMED_GROUPS);
|
|
+}
|
|
+
|
|
+static const SSLSignatureScheme signatures[] = {
|
|
+ ssl_sig_ecdsa_secp256r1_sha256, ssl_sig_ecdsa_secp384r1_sha384,
|
|
+ ssl_sig_ecdsa_secp521r1_sha512, ssl_sig_rsa_pss_sha256,
|
|
+ ssl_sig_rsa_pss_sha384, ssl_sig_rsa_pss_sha512,
|
|
+ ssl_sig_rsa_pkcs1_sha256, ssl_sig_rsa_pkcs1_sha384,
|
|
+ ssl_sig_rsa_pkcs1_sha512, ssl_sig_ecdsa_sha1,
|
|
+ ssl_sig_rsa_pkcs1_sha1
|
|
+};
|
|
+
|
|
+#define NUM_OF_SIGNATURES sizeof(signatures)/sizeof(signatures[0])
|
|
+
|
|
+static SECStatus set_additional_key_shares(PRFileDesc *model)
|
|
+{
|
|
+ /* This aligns TLS extension 51 (key_share) to what Firefox does. */
|
|
+ return SSL_SendAdditionalKeyShares(model, 1);
|
|
+}
|
|
+
|
|
+static SECStatus set_signatures(PRFileDesc *model)
|
|
+{
|
|
+ /* Align TLS extension 13 (signature_algorithms) to what Firefox does. */
|
|
+ return SSL_SignatureSchemePrefSet(model, signatures, NUM_OF_SIGNATURES);
|
|
+}
|
|
+
|
|
+static SECStatus set_ssl_options(PRFileDesc *model)
|
|
+{
|
|
+ SECStatus s;
|
|
+
|
|
+ /* Enable TLS 1.3 compat mode. Firefox does this, as can be seen at
|
|
+ * nsSSLIOLayerSetOptions()@nsNSSIOLayer.cpp.
|
|
+ * This has the side effect of NSS faking a TLS session ID.
|
|
+ * See ssl3_CreateClientHelloPreamble()@ssl3con.c
|
|
+ */
|
|
+ s = SSL_OptionSet(model, SSL_ENABLE_TLS13_COMPAT_MODE, PR_TRUE);
|
|
+ if (s != SECSuccess) {
|
|
+ return s;
|
|
+ }
|
|
+
|
|
+ /* Firefox sets the following options. I don't know what they do. */
|
|
+ s = SSL_OptionSet(model, SSL_REQUIRE_SAFE_NEGOTIATION, false);
|
|
+ if (s != SECSuccess) {
|
|
+ return s;
|
|
+ }
|
|
+ s = SSL_OptionSet(model, SSL_ENABLE_EXTENDED_MASTER_SECRET, true);
|
|
+ if (s != SECSuccess) {
|
|
+ return s;
|
|
+ }
|
|
+ s = SSL_OptionSet(model, SSL_ENABLE_HELLO_DOWNGRADE_CHECK, true);
|
|
+ if (s != SECSuccess) {
|
|
+ return s;
|
|
+ }
|
|
+ s = SSL_OptionSet(model, SSL_ENABLE_0RTT_DATA, true);
|
|
+ if (s != SECSuccess) {
|
|
+ return s;
|
|
+ }
|
|
+
|
|
+ /* This adds TLS extension 34 to the Client Hello. */
|
|
+ s = SSL_OptionSet(model, SSL_ENABLE_DELEGATED_CREDENTIALS, true);
|
|
+ if (s != SECSuccess) {
|
|
+ return s;
|
|
+ }
|
|
+
|
|
+ /* This adds TLS extension 5 (status_request) to the Client Hello. */
|
|
+ s = SSL_OptionSet(model, SSL_ENABLE_OCSP_STAPLING, true);
|
|
+ if (s != SECSuccess) {
|
|
+ return s;
|
|
+ }
|
|
+
|
|
+ /* Remove TLS extension 18 (signed_certificate_timestamp) */
|
|
+ s = SSL_OptionSet(model, SSL_ENABLE_SIGNED_CERT_TIMESTAMPS, false);
|
|
+ if (s != SECSuccess) {
|
|
+ return s;
|
|
+ }
|
|
+
|
|
+ return SSL_OptionSet(model, SSL_HANDSHAKE_AS_CLIENT, true);
|
|
+}
|
|
+
|
|
/*
|
|
* Return true if at least one cipher-suite is enabled. Used to determine
|
|
* if we need to call NSS_SetDomesticPolicy() to enable the default ciphers.
|
|
@@ -1320,6 +1410,7 @@ static CURLcode nss_load_module(SECMODModule **pmod, const char *library,
|
|
|
|
if(module)
|
|
SECMOD_DestroyModule(module);
|
|
+
|
|
return CURLE_FAILED_INIT;
|
|
}
|
|
|
|
@@ -1921,6 +2012,12 @@ static CURLcode nss_setup_connect(struct Curl_easy *data,
|
|
if(SSL_OptionSet(model, SSL_NO_CACHE, ssl_no_cache) != SECSuccess)
|
|
goto error;
|
|
|
|
+ if(SSL_SET_OPTION(primary.sessionid)) {
|
|
+ if(SSL_OptionSet(model, SSL_ENABLE_SESSION_TICKETS,
|
|
+ PR_TRUE) != SECSuccess)
|
|
+ goto error;
|
|
+ }
|
|
+
|
|
/* enable/disable the requested SSL version(s) */
|
|
if(nss_init_sslver(&sslver, data, conn) != CURLE_OK)
|
|
goto error;
|
|
@@ -1960,6 +2057,14 @@ static CURLcode nss_setup_connect(struct Curl_easy *data,
|
|
}
|
|
}
|
|
|
|
+ if (set_named_groups(model) != SECSuccess ||
|
|
+ set_additional_key_shares(model) != SECSuccess ||
|
|
+ set_signatures(model) != SECSuccess ||
|
|
+ set_ssl_options(model) != SECSuccess) {
|
|
+ result = CURLE_SSL_CIPHER;
|
|
+ goto error;
|
|
+ }
|
|
+
|
|
if(!SSL_CONN_CONFIG(verifypeer) && SSL_CONN_CONFIG(verifyhost))
|
|
infof(data, "warning: ignoring value of ssl.verifyhost");
|
|
|
|
@@ -2113,6 +2218,10 @@ static CURLcode nss_setup_connect(struct Curl_easy *data,
|
|
int cur = 0;
|
|
unsigned char protocols[128];
|
|
|
|
+ protocols[cur++] = ALPN_HTTP_1_1_LENGTH;
|
|
+ memcpy(&protocols[cur], ALPN_HTTP_1_1, ALPN_HTTP_1_1_LENGTH);
|
|
+ cur += ALPN_HTTP_1_1_LENGTH;
|
|
+
|
|
#ifdef USE_HTTP2
|
|
if(data->state.httpwant >= CURL_HTTP_VERSION_2
|
|
#ifndef CURL_DISABLE_PROXY
|
|
@@ -2124,9 +2233,6 @@ static CURLcode nss_setup_connect(struct Curl_easy *data,
|
|
cur += ALPN_H2_LENGTH;
|
|
}
|
|
#endif
|
|
- protocols[cur++] = ALPN_HTTP_1_1_LENGTH;
|
|
- memcpy(&protocols[cur], ALPN_HTTP_1_1, ALPN_HTTP_1_1_LENGTH);
|
|
- cur += ALPN_HTTP_1_1_LENGTH;
|
|
|
|
if(SSL_SetNextProtoNego(backend->handle, protocols, cur) != SECSuccess)
|
|
goto error;
|
|
diff --git a/libcurl.pc.in b/libcurl.pc.in
|
|
index 8ac15d407..68d01b219 100644
|
|
--- a/libcurl.pc.in
|
|
+++ b/libcurl.pc.in
|
|
@@ -34,6 +34,6 @@ Name: libcurl
|
|
URL: https://curl.se/
|
|
Description: Library to transfer files with ftp, http, etc.
|
|
Version: @CURLVERSION@
|
|
-Libs: -L${libdir} -lcurl @LIBCURL_NO_SHARED@
|
|
+Libs: -L${libdir} -lcurl-impersonate-ff @LIBCURL_NO_SHARED@
|
|
Libs.private: @LIBCURL_LIBS@
|
|
Cflags: -I${includedir} @CPPFLAG_CURL_STATICLIB@
|
|
diff --git a/m4/curl-nss.m4 b/m4/curl-nss.m4
|
|
index 397ba71b1..d2a8fc1f2 100644
|
|
--- a/m4/curl-nss.m4
|
|
+++ b/m4/curl-nss.m4
|
|
@@ -74,7 +74,107 @@ if test "x$OPT_NSS" != xno; then
|
|
# Without pkg-config, we'll kludge in some defaults
|
|
AC_MSG_WARN([Using hard-wired libraries and compilation flags for NSS.])
|
|
addld="-L$OPT_NSS/lib"
|
|
- addlib="-lssl3 -lsmime3 -lnss3 -lplds4 -lplc4 -lnspr4"
|
|
+
|
|
+ # curl-impersonate: Link NSS statically.
|
|
+ # NSS is poorly documented in this regard and a lot of trial and error
|
|
+ # was made to come up with the correct list of linking flags. The
|
|
+ # libraries have circular dependencies which makes their order extremely
|
|
+ # difficult to find out.
|
|
+
|
|
+ # Some references:
|
|
+ # https://github.com/mozilla/application-services/blob/b2690fd2e4cc3e8e10b6868ab0de8b79c89d3a93/components/support/rc_crypto/nss/nss_build_common/src/lib.rs#L94
|
|
+ # and
|
|
+ # https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/freebl/freebl.gyp
|
|
+
|
|
+ # On Linux we can use special linker flags to force static linking
|
|
+ # (-l:libplc4.a etc.), otherwise the linker will prefer to use
|
|
+ # libplc4.so. On other systems the dynamic libraries would have to be
|
|
+ # removed manually from the NSS directory before building curl.
|
|
+ case $host_os in
|
|
+ linux*)
|
|
+ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -l:libplc4.a -l:libplds4.a -l:libnspr4.a -lsqlite"
|
|
+ ;;
|
|
+ darwin*)
|
|
+ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4"
|
|
+ ;;
|
|
+ *)
|
|
+ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4 -lsqlite"
|
|
+ ;;
|
|
+ esac
|
|
+
|
|
+ case $host_cpu in
|
|
+ arm)
|
|
+ addlib="$addlib -larmv8_c_lib"
|
|
+ ;;
|
|
+ aarch64)
|
|
+ addlib="$addlib -larmv8_c_lib -lgcm-aes-aarch64_c_lib"
|
|
+ ;;
|
|
+ x86)
|
|
+ addlib="$addlib -lgcm-aes-x86_c_lib"
|
|
+ ;;
|
|
+ x86_64)
|
|
+ addlib="$addlib -lgcm-aes-x86_c_lib -lhw-acc-crypto-avx -lhw-acc-crypto-avx2 -lsha-x86_c_lib"
|
|
+ case $host_os in
|
|
+ linux*)
|
|
+ addlib="$addlib -lintel-gcm-wrap_c_lib -lintel-gcm-s_lib"
|
|
+ ;;
|
|
+ esac
|
|
+ ;;
|
|
+ esac
|
|
+
|
|
+ # curl-impersonate:
|
|
+ # On Linux these linker flags are necessary to resolve
|
|
+ # the symbol mess and circular dependencies of NSS .a libraries
|
|
+ # to make the AC_CHECK_LIB test below pass.
|
|
+ case $host_os in
|
|
+ linux*)
|
|
+ addlib="-Wl,--start-group $addlib -Wl,--end-group"
|
|
+ ;;
|
|
+ esac
|
|
+
|
|
+ # External dependencies for nss
|
|
+ case $host_os in
|
|
+ linux*)
|
|
+ addlib="$addlib -pthread -ldl"
|
|
+ ;;
|
|
+ darwin*)
|
|
+ addlib="$addlib -lsqlite3"
|
|
+ ;;
|
|
+ esac
|
|
+
|
|
+ # Attempt to locate libnssckbi.
|
|
+ # This library file contains the trusted certificates and nss loads it
|
|
+ # at runtime using dlopen. If it's not in a path findable by dlopen
|
|
+ # we have to add that path explicitly using -rpath so it may find it.
|
|
+ # On Ubuntu and Mac M1 it is in a non-standard location.
|
|
+ AC_MSG_CHECKING([if libnssckbi is in a non-standard location])
|
|
+ case $host_os in
|
|
+ linux*)
|
|
+ search_paths="/usr/lib/$host /usr/lib/$host/nss"
|
|
+ search_paths="$search_paths /usr/lib/$host_cpu-$host_os"
|
|
+ search_paths="$search_paths /usr/lib/$host_cpu-$host_os/nss"
|
|
+ search_ext="so"
|
|
+ ;;
|
|
+ darwin*)
|
|
+ search_paths="/opt/homebrew/lib"
|
|
+ search_ext="dylib"
|
|
+ ;;
|
|
+ esac
|
|
+
|
|
+ found="no"
|
|
+ for path in $search_paths; do
|
|
+ if test -f "$path/libnssckbi.$search_ext"; then
|
|
+ AC_MSG_RESULT([$path])
|
|
+ addld="$addld -Wl,-rpath,$path"
|
|
+ found="yes"
|
|
+ break
|
|
+ fi
|
|
+ done
|
|
+
|
|
+ if test "$found" = "no"; then
|
|
+ AC_MSG_RESULT([no])
|
|
+ fi
|
|
+
|
|
addcflags="-I$OPT_NSS/include"
|
|
version="unknown"
|
|
nssprefix=$OPT_NSS
|
|
@@ -91,7 +191,7 @@ if test "x$OPT_NSS" != xno; then
|
|
fi
|
|
|
|
dnl The function SSL_VersionRangeSet() is needed to enable TLS > 1.0
|
|
- AC_CHECK_LIB(nss3, SSL_VersionRangeSet,
|
|
+ AC_CHECK_LIB(nss_static, SSL_VersionRangeSet,
|
|
[
|
|
AC_DEFINE(USE_NSS, 1, [if NSS is enabled])
|
|
AC_SUBST(USE_NSS, [1])
|
|
@@ -101,9 +201,7 @@ if test "x$OPT_NSS" != xno; then
|
|
test nss != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes
|
|
],
|
|
[
|
|
- LDFLAGS="$CLEANLDFLAGS"
|
|
- LIBS="$CLEANLIBS"
|
|
- CPPFLAGS="$CLEANCPPFLAGS"
|
|
+ AC_MSG_ERROR([Failed linking NSS statically])
|
|
])
|
|
|
|
if test "x$USE_NSS" = "xyes"; then
|
|
diff --git a/src/Makefile.am b/src/Makefile.am
|
|
index c8abc93b1..fcecb10d0 100644
|
|
--- a/src/Makefile.am
|
|
+++ b/src/Makefile.am
|
|
@@ -41,7 +41,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/include \
|
|
-I$(top_srcdir)/lib \
|
|
-I$(top_srcdir)/src
|
|
|
|
-bin_PROGRAMS = curl
|
|
+bin_PROGRAMS = curl-impersonate-ff
|
|
|
|
SUBDIRS = ../docs
|
|
|
|
@@ -52,7 +52,7 @@ endif
|
|
include Makefile.inc
|
|
|
|
# CURL_FILES comes from Makefile.inc
|
|
-curl_SOURCES = $(CURL_FILES)
|
|
+curl_impersonate_ff_SOURCES = $(CURL_FILES)
|
|
|
|
# This might hold -Werror
|
|
CFLAGS += @CURL_CFLAG_EXTRAS@
|
|
@@ -61,9 +61,9 @@ CFLAGS += @CURL_CFLAG_EXTRAS@
|
|
LIBS = $(BLANK_AT_MAKETIME)
|
|
|
|
if USE_EXPLICIT_LIB_DEPS
|
|
-curl_LDADD = $(top_builddir)/lib/libcurl.la @LIBCURL_LIBS@
|
|
+curl_impersonate_ff_LDADD = $(top_builddir)/lib/libcurl-impersonate-ff.la @LIBCURL_LIBS@
|
|
else
|
|
-curl_LDADD = $(top_builddir)/lib/libcurl.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@
|
|
+curl_impersonate_ff_LDADD = $(top_builddir)/lib/libcurl-impersonate-ff.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@
|
|
endif
|
|
|
|
# if unit tests are enabled, build a static library to link them with
|