Merge pull request #4 from lwthiker/impersonate_chrome

Impersonate Chrome by building curl with BoringSSL and modifying its TLS and HTTP/2 parameters.
2025-08-08 12:49:36 +00:00 · 2022-02-19 15:12:15 +02:00
parent f08db5c1cc 1b41b6de84
commit 6f7273fb88
9 changed files with 337 additions and 0 deletions
--- a/chrome/Dockerfile
+++ b/chrome/Dockerfile
@@ -0,0 +1,79 @@
+# Use same base as the existing Dockerfile
+FROM python:3.10.1-slim-buster
+
+WORKDIR /build
+
+# Dependencies for downloading and building BoringSSL
+RUN apt-get update && \
+    apt-get install -y git g++ cmake golang-go ninja-build curl unzip zlib1g-dev libbrotli-dev
+
+# BoringSSL doesn't have versions. Choose a commit that is used in a stable
+# Chromium version.
+ARG BORING_SSL_COMMIT=3a667d10e94186fd503966f5638e134fe9fb4080
+RUN curl -L https://github.com/google/boringssl/archive/${BORING_SSL_COMMIT}.zip -o boringssl.zip && \
+    unzip boringssl && \
+    mv boringssl-${BORING_SSL_COMMIT} boringssl
+
+# Compile BoringSSL.
+# See https://boringssl.googlesource.com/boringssl/+/HEAD/BUILDING.md
+RUN cd boringssl && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_POSITION_INDEPENDENT_CODE=on -GNinja .. && \
+    ninja
+
+# Fix the directory structure so that curl can compile against it.
+# See https://everything.curl.dev/source/build/tls/boringssl 
+RUN mkdir boringssl/build/lib && \
+    ln -s ../crypto/libcrypto.a boringssl/build/lib/libcrypto.a && \
+    ln -s ../ssl/libssl.a boringssl/build/lib/libssl.a && \
+    cp -R boringssl/include boringssl/build
+
+ARG NGHTTP2_VERSION=nghttp2-1.46.0
+ARG NGHTTP2_URL=https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2
+
+# The following are needed because we are going to change some autoconf scripts,
+# both for libnghttp2 and curl.
+RUN apt-get install -y autoconf automake autotools-dev pkg-config libtool
+
+# Download nghttp2 for HTTP/2.0 support.
+RUN curl -o ${NGHTTP2_VERSION}.tar.bz2 -L ${NGHTTP2_URL}
+RUN tar xf ${NGHTTP2_VERSION}.tar.bz2
+
+# Patch nghttp2 pkg config file to support static builds.
+COPY patches/libnghttp2-*.patch ${NGHTTP2_VERSION}/
+RUN cd ${NGHTTP2_VERSION} && \
+    for p in $(ls libnghttp2-*.patch); do patch -p1 < $p; done && \
+    autoreconf -i && automake && autoconf
+
+# Compile nghttp2
+RUN cd ${NGHTTP2_VERSION} && \
+    ./configure && \
+    make && make install
+
+# Download curl.
+ARG CURL_VERSION=curl-7.81.0
+RUN curl -o ${CURL_VERSION}.tar.xz https://curl.se/download/${CURL_VERSION}.tar.xz
+RUN tar xf ${CURL_VERSION}.tar.xz
+
+# Patch curl and re-generate the configure script
+COPY patches/curl-*.patch ${CURL_VERSION}/
+RUN cd ${CURL_VERSION} && \
+    for p in $(ls curl-*.patch); do patch -p1 < $p; done && \
+    autoreconf -fi
+
+# Compile curl with BoringSSL & nghttp2.
+# Enable keylogfile for debugging of TLS traffic.
+RUN cd ${CURL_VERSION} && \
+    ./configure --with-openssl=/build/boringssl/build --enable-static --disable-shared --with-nghttp2=/usr/local LIBS="-pthread" CFLAGS="-I/build/boringssl/build" USE_CURL_SSLKEYLOGFILE=true && \
+    make
+
+# 'xxd' is needed for the wrapper script
+RUN apt-get install -y xxd
+
+RUN mkdir out && \
+    cp ${CURL_VERSION}/src/curl out/curl-impersonate-ch
+
+# Wrapper script
+COPY curl_chrome* out/
+
+RUN chmod +x out/*
--- a/chrome/curl_chrome98
+++ b/chrome/curl_chrome98
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Find the directory of this script
+dir=`echo "$0" | sed 's%/[^/]*$%%'`
+
+# The list of ciphers can be obtained by looking at the Client Hello message in
+# Wireshark, then converting it using this reference
+# https://wiki.mozilla.org/Security/Cipher_Suites
+"$dir/curl-impersonate-ch" \
+    --ciphers TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA \
+    -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"' \
+    -H 'sec-ch-ua-mobile: ?0' \
+    -H 'sec-ch-ua-platform: "Windows"' \
+    -H 'Upgrade-Insecure-Requests: 1' \
+    -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' \
+    -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
+    -H 'Sec-Fetch-Site: none' \
+    -H 'Sec-Fetch-Mode: navigate' \
+    -H 'Sec-Fetch-User: ?1' \
+    -H 'Sec-Fetch-Dest: document' \
+    -H 'Accept-Encoding: gzip, deflate, br' \
+    -H 'Accept-Language: en-US,en;q=0.9' \
+    --http2 --false-start --tlsv1.2 --compressed \
+    $@
--- a/chrome/patches/curl-configure.patch
+++ b/chrome/patches/curl-configure.patch
@@ -0,0 +1,17 @@
+--- curl-7.81.0-original/configure.ac	2022-01-03 18:36:46.000000000 +0200
+++ curl-7.81.0/configure.ac	2022-02-17 13:40:02.248497926 +0200
+@@ -2575,3 +2575,3 @@
+     LIB_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path])
+-      $PKGCONFIG --libs-only-l libnghttp2`
+      $PKGCONFIG --static --libs-only-l libnghttp2`
+     AC_MSG_NOTICE([-l is $LIB_H2])
+@@ -2579,3 +2579,3 @@
+     CPP_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path]) dnl
+-      $PKGCONFIG --cflags-only-I libnghttp2`
+      $PKGCONFIG --static --cflags-only-I libnghttp2`
+     AC_MSG_NOTICE([-I is $CPP_H2])
+@@ -2583,3 +2583,3 @@
+     LD_H2=`CURL_EXPORT_PCDIR([$want_nghttp2_path])
+-      $PKGCONFIG --libs-only-L libnghttp2`
+      $PKGCONFIG --static --libs-only-L libnghttp2`
+     AC_MSG_NOTICE([-L is $LD_H2])
--- a/chrome/patches/curl-http-h.patch
+++ b/chrome/patches/curl-http-h.patch
@@ -0,0 +1,8 @@
+--- curl-7.81.0-original/lib/http.h	2022-01-03 18:36:46.000000000 +0200
+++ curl-7.81.0/lib/http.h	2022-02-19 00:44:48.347052308 +0200
+@@ -280,3 +280,4 @@
+   /* list of settings that will be sent */
+-  nghttp2_settings_entry local_settings[3];
+  /* curl-impersonate: Align HTTP/2 settings to Chrome's */
+  nghttp2_settings_entry local_settings[5];
+   size_t local_settings_num;
--- a/chrome/patches/curl-http2-c.patch
+++ b/chrome/patches/curl-http2-c.patch
@@ -0,0 +1,80 @@
+--- curl-7.81.0-original/lib/http2.c	2022-01-03 18:36:46.000000000 +0200
+++ curl-7.81.0/lib/http2.c	2022-02-19 00:43:56.613992732 +0200
+@@ -43,2 +43,3 @@
+ #include "memdebug.h"
+#include "rand.h"
+ 
+@@ -1195,12 +1196,23 @@
+ 
+-  iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
+-  iv[0].value = Curl_multi_max_concurrent_streams(data->multi);
+  /* curl-impersonate: Align HTTP/2 settings to Chrome's */
+  iv[0].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE;
+  iv[0].value = 0x10000;
+
+  iv[1].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
+  iv[1].value = Curl_multi_max_concurrent_streams(data->multi);
+
+  iv[2].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
+  iv[2].value = 0x600000;
+
+  iv[3].settings_id = NGHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE;
+  iv[3].value = 0x40000;
+
+  // iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
+  // iv[2].value = data->multi->push_cb != NULL;
+  
+  // Looks like random setting set by Chrome, maybe similar to TLS GREASE. */
+  Curl_rand(data, (unsigned char *)&iv[4].settings_id, sizeof(iv[4].settings_id));
+  Curl_rand(data, (unsigned char *)&iv[4].value, sizeof(iv[4].value));
+ 
+-  iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
+-  iv[1].value = HTTP2_HUGE_WINDOW_SIZE;
+-
+-  iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
+-  iv[2].value = data->multi->push_cb != NULL;
+-
+-  httpc->local_settings_num = 3;
+  httpc->local_settings_num = 5;
+ }
+@@ -1820,3 +1832,4 @@
+    field list. */
+-#define AUTHORITY_DST_IDX 3
+/* curl-impersonate: Put the ":authority" header in the first place. */
+#define AUTHORITY_DST_IDX 1
+ 
+@@ -2034,8 +2047,9 @@
+     goto fail;
+-  nva[1].name = (unsigned char *)":path";
+-  nva[1].namelen = strlen((char *)nva[1].name);
+-  nva[1].value = (unsigned char *)hdbuf;
+-  nva[1].valuelen = (size_t)(end - hdbuf);
+-  nva[1].flags = NGHTTP2_NV_FLAG_NONE;
+-  if(HEADER_OVERFLOW(nva[1])) {
+  /* curl-impersonate: Switch the places of ":path" and ":scheme". */
+  nva[2].name = (unsigned char *)":path";
+  nva[2].namelen = strlen((char *)nva[2].name);
+  nva[2].value = (unsigned char *)hdbuf;
+  nva[2].valuelen = (size_t)(end - hdbuf);
+  nva[2].flags = NGHTTP2_NV_FLAG_NONE;
+  if(HEADER_OVERFLOW(nva[2])) {
+     failf(data, "Failed sending HTTP request: Header overflow");
+@@ -2044,11 +2058,11 @@
+ 
+-  nva[2].name = (unsigned char *)":scheme";
+-  nva[2].namelen = strlen((char *)nva[2].name);
+  nva[1].name = (unsigned char *)":scheme";
+  nva[1].namelen = strlen((char *)nva[1].name);
+   if(conn->handler->flags & PROTOPT_SSL)
+-    nva[2].value = (unsigned char *)"https";
+    nva[1].value = (unsigned char *)"https";
+   else
+-    nva[2].value = (unsigned char *)"http";
+-  nva[2].valuelen = strlen((char *)nva[2].value);
+-  nva[2].flags = NGHTTP2_NV_FLAG_NONE;
+-  if(HEADER_OVERFLOW(nva[2])) {
+    nva[1].value = (unsigned char *)"http";
+  nva[1].valuelen = strlen((char *)nva[1].value);
+  nva[1].flags = NGHTTP2_NV_FLAG_NONE;
+  if(HEADER_OVERFLOW(nva[1])) {
+     failf(data, "Failed sending HTTP request: Header overflow");
--- a/chrome/patches/curl-http2-h.patch
+++ b/chrome/patches/curl-http2-h.patch
@@ -0,0 +1,8 @@
+--- curl-7.81.0-original/lib/http2.h	2021-12-10 09:40:37.000000000 +0200
+++ curl-7.81.0/lib/http2.h	2022-02-19 00:45:53.440376589 +0200
+@@ -31,3 +31,4 @@
+    from the peer */
+-#define DEFAULT_MAX_CONCURRENT_STREAMS 100
+/* curl-impersonate: Use 1000 concurrent streams like Chrome. */
+#define DEFAULT_MAX_CONCURRENT_STREAMS 1000
+ 
--- a/chrome/patches/curl-multi-c.patch
+++ b/chrome/patches/curl-multi-c.patch
@@ -0,0 +1,8 @@
+--- curl-7.81.0-original/lib/multi.c	2022-01-03 18:36:46.000000000 +0200
+++ curl-7.81.0/lib/multi.c	2022-02-18 22:43:54.939227658 +0200
+@@ -395,3 +395,4 @@
+   multi->maxconnects = -1;
+-  multi->max_concurrent_streams = 100;
+  /* curl-impersonate: Use 1000 concurrent streams like Chrome. */
+  multi->max_concurrent_streams = 1000;
+   multi->ipv6_works = Curl_ipv6works(NULL);
--- a/chrome/patches/curl-openssl-c.patch
+++ b/chrome/patches/curl-openssl-c.patch
@@ -0,0 +1,105 @@
+--- curl-7.81.0-original/lib/vtls/openssl.c	2022-01-03 18:36:46.000000000 +0200
+++ curl-7.81.0/lib/vtls/openssl.c	2022-02-18 20:02:23.559231773 +0200
+@@ -78,2 +78,4 @@
+ 
+#include <brotli/decode.h>
+
+ #ifdef USE_AMISSL
+@@ -2631,2 +2633,27 @@
+ 
+/* Taken from Chromium and adapted to C,
+ * see net/ssl/cert_compression.cc
+ */
+int DecompressBrotliCert(SSL* ssl,
+                         CRYPTO_BUFFER** out,
+                         size_t uncompressed_len,
+                         const uint8_t* in,
+                         size_t in_len) {
+  uint8_t* data;
+  CRYPTO_BUFFER* decompressed = CRYPTO_BUFFER_alloc(&data, uncompressed_len);
+  if (!decompressed) {
+    return 0;
+  }
+
+  size_t output_size = uncompressed_len;
+  if (BrotliDecoderDecompress(in_len, in, &output_size, data) !=
+          BROTLI_DECODER_RESULT_SUCCESS ||
+      output_size != uncompressed_len) {
+    return 0;
+  }
+
+  *out = decompressed;
+  return 1;
+}
+
+ static CURLcode ossl_connect_step1(struct Curl_easy *data,
+@@ -2769,3 +2796,6 @@
+ #ifdef SSL_OP_NO_TICKET
+-  ctx_options |= SSL_OP_NO_TICKET;
+  /* curl-impersonate patch.
+   * Turn off SSL_OP_NO_TICKET, we want TLS extension 35 (session_ticket)
+   * to be sent. */
+  ctx_options &= ~SSL_OP_NO_TICKET;
+ #endif
+@@ -2823,4 +2853,7 @@
+ #ifdef HAS_NPN
+  /* curl-impersonate: Do not enable the NPN extension. */
+  /*
+   if(conn->bits.tls_enable_npn)
+     SSL_CTX_set_next_proto_select_cb(backend->ctx, select_next_proto_cb, data);
+  */
+ #endif
+@@ -2939,2 +2972,15 @@
+ 
+  /* curl-impersonate:
+   * Configure BoringSSL to behave like Chrome. 
+   * See Constructor of SSLContext at net/socket/ssl_client_socket_impl.cc
+   * and SSLClientSocketImpl::Init()
+   * in the Chromium's source code. */
+
+  /* Enable TLS GREASE. */
+  SSL_CTX_set_grease_enabled(backend->ctx, 1);
+
+  /* Add support for TLS extension 27 - compress_certificate.
+   * Add Brotli decompression. See Chromium net/ssl/cert_compression.cc */
+  SSL_CTX_add_cert_compression_alg(backend->ctx,
+          TLSEXT_cert_compression_brotli, NULL, DecompressBrotliCert);
+ 
+@@ -3238,2 +3284,37 @@
+ 
+#ifdef USE_HTTP2
+  /* curl-impersonate: This adds the ALPS extension (17513).
+   * Chromium calls this function as well in SSLClientSocketImpl::Init().
+   * The 4th parameter is called "settings", and I don't know what it
+   * should contain. For now, use an empty string. */
+  SSL_add_application_settings(backend->handle, "h2", 2, NULL, 0);
+#endif
+
+  SSL_set_options(backend->handle,
+      SSL_OP_LEGACY_SERVER_CONNECT);
+  SSL_set_mode(backend->handle,
+      SSL_MODE_CBC_RECORD_SPLITTING | SSL_MODE_ENABLE_FALSE_START);
+
+  /* curl-impersonate: Enable TLS extensions 5 - status_request and
+   * 18 - signed_certificate_timestamp. */
+  SSL_enable_signed_cert_timestamps(backend->handle);
+  SSL_enable_ocsp_stapling(backend->handle);
+
+  /* curl-impersonate: Some SSL settings copied over from Chrome. */
+  SSL_set_shed_handshake_config(backend->handle, 1);
+
+  /* curl-impersonate: Set the signature algorithms.
+   * (TLS extension 13).
+   * See net/socket/ssl_client_socket_impl.cc in Chromium's source. */
+  static const uint16_t kVerifyPrefs[] = {
+    SSL_SIGN_ECDSA_SECP256R1_SHA256, SSL_SIGN_RSA_PSS_RSAE_SHA256,
+    SSL_SIGN_RSA_PKCS1_SHA256,       SSL_SIGN_ECDSA_SECP384R1_SHA384,
+    SSL_SIGN_RSA_PSS_RSAE_SHA384,    SSL_SIGN_RSA_PKCS1_SHA384,
+    SSL_SIGN_RSA_PSS_RSAE_SHA512,    SSL_SIGN_RSA_PKCS1_SHA512,
+  };
+  if (!SSL_set_verify_algorithm_prefs(backend->handle, kVerifyPrefs,
+              sizeof(kVerifyPrefs) / sizeof(kVerifyPrefs[0]))) {
+    return CURLE_SSL_CIPHER;
+  }
+
+   backend->server_cert = 0x0;
--- a/chrome/patches/libnghttp2-pc.patch
+++ b/chrome/patches/libnghttp2-pc.patch
@@ -0,0 +1,8 @@
+--- nghttp2-1.46.0-original/lib/libnghttp2.pc.in	2021-10-19 12:31:47.000000000 +0300
+++ nghttp2-1.46.0/lib/libnghttp2.pc.in	2022-02-17 13:44:46.722604316 +0200
+@@ -31,3 +31,4 @@
+ Version: @VERSION@
+-Libs: -L${libdir} -lnghttp2
+Libs: -L${libdir}
+Libs.private: -l:libnghttp2.a
+ Cflags: -I${includedir}