hoelee/curl-impersonate
1
0
Fork 0
mirror of https://github.com/lwthiker/curl-impersonate.git synced 2025-08-08 04:42:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Set HTTP/2 stream settings correctly

Browse Source 
HTTP/2 includes various settings pertaining to stream priorities. Chrome
and Firefox handle them differently, and this behavior was not mimicked
in curl-impersonate well. With this commit, the stream settings set
by curl-impersonate are identical to the real browsers.

* With Chrome, the default stream weight is 256 and the "exclusive bit"
is set to ON.
* With Firefox, a complex tree of stream dependencies is created
by default using PRIORITY frames. This behavior is now mimicked by
curl-impersonate.
This commit is contained in:
lwthiker
2022-05-29 16:45:17 +03:00
parent 33b63f49e5
commit 6080446ef4
2 changed files with 197 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
chrome/patches/curl-impersonate.patch
View File

@@ -997,7 +997,7 @@ index b4aaba2a2..0d716640c 100644
#else
int unused; /* prevent a compiler warning */
diff --git a/lib/http2.c b/lib/http2.c
index e74400a4c..b0e6674f2 100644
index e74400a4c..aec67c975 100644
--- a/lib/http2.c
+++ b/lib/http2.c
@@ -41,6 +41,7 @@
@@ -1017,7 +1017,39 @@ index e74400a4c..b0e6674f2 100644
#ifdef DEBUG_HTTP2
#define H2BUGF(x) x
@@ -1193,16 +1194,30 @@ static void populate_settings(struct Curl_easy *data,
@@ -75,13 +76,20 @@ static int h2_process_pending_input(struct Curl_easy *data,
struct http_conn *httpc,
CURLcode *err);
+/*
+ * curl-impersonate: Use Chrome's default HTTP/2 stream weight.
+ */
+
+#define CHROME_DEFAULT_STREAM_WEIGHT (256)
+
/*
* Curl_http2_init_state() is called when the easy handle is created and
* allows for HTTP/2 specific init of state.
*/
void Curl_http2_init_state(struct UrlState *state)
{
- state->stream_weight = NGHTTP2_DEFAULT_WEIGHT;
+ state->stream_weight = CHROME_DEFAULT_STREAM_WEIGHT;
+ state->stream_depends_e = TRUE;
}
/*
@@ -90,7 +98,8 @@ void Curl_http2_init_state(struct UrlState *state)
*/
void Curl_http2_init_userset(struct UserDefined *set)
{
- set->stream_weight = NGHTTP2_DEFAULT_WEIGHT;
+ set->stream_weight = CHROME_DEFAULT_STREAM_WEIGHT;
+ set->stream_depends_e = TRUE;
}
static int http2_getsock(struct Curl_easy *data,
@@ -1193,16 +1202,30 @@ static void populate_settings(struct Curl_easy *data,
{
nghttp2_settings_entry *iv = httpc->local_settings;
@@ -1026,21 +1058,21 @@ index e74400a4c..b0e6674f2 100644
+ /* curl-impersonate: Align HTTP/2 settings to Chrome's */
+ iv[0].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE;
+ iv[0].value = 0x10000;
- iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
- iv[1].value = HTTP2_HUGE_WINDOW_SIZE;
+
+ iv[1].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
+ iv[1].value = Curl_multi_max_concurrent_streams(data->multi);
- iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
- iv[2].value = data->multi->push_cb != NULL;
- iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
- iv[1].value = HTTP2_HUGE_WINDOW_SIZE;
+ iv[2].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
+ iv[2].value = 0x600000;
- httpc->local_settings_num = 3;
- iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
- iv[2].value = data->multi->push_cb != NULL;
+ iv[3].settings_id = NGHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE;
+ iv[3].value = 0x40000;
+
- httpc->local_settings_num = 3;
+ // iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
+ // iv[2].value = data->multi->push_cb != NULL;
+
@@ -1055,7 +1087,7 @@ index e74400a4c..b0e6674f2 100644
}
void Curl_http2_done(struct Curl_easy *data, bool premature)
@@ -1816,10 +1831,6 @@ static ssize_t http2_recv(struct Curl_easy *data, int sockindex,
@@ -1816,10 +1839,6 @@ static ssize_t http2_recv(struct Curl_easy *data, int sockindex,
return -1;
}
@@ -1066,7 +1098,7 @@ index e74400a4c..b0e6674f2 100644
/* USHRT_MAX is 65535 == 0xffff */
#define HEADER_OVERFLOW(x) \
(x.namelen > 0xffff || x.valuelen > 0xffff - x.namelen)
@@ -1890,6 +1901,53 @@ static header_instruction inspect_header(const char *name, size_t namelen,
@@ -1890,6 +1909,53 @@ static header_instruction inspect_header(const char *name, size_t namelen,
}
}
@@ -1120,7 +1152,7 @@ index e74400a4c..b0e6674f2 100644
static ssize_t http2_send(struct Curl_easy *data, int sockindex,
const void *mem, size_t len, CURLcode *err)
{
@@ -1905,6 +1963,7 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
@@ -1905,6 +1971,7 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
nghttp2_nv *nva = NULL;
size_t nheader;
size_t i;
@@ -1128,7 +1160,7 @@ index e74400a4c..b0e6674f2 100644
size_t authority_idx;
char *hdbuf = (char *)mem;
char *end, *line_end;
@@ -2010,12 +2069,21 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
@@ -2010,12 +2077,21 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
end = memchr(hdbuf, ' ', line_end - hdbuf);
if(!end || end == hdbuf)
goto fail;
@@ -1156,7 +1188,7 @@ index e74400a4c..b0e6674f2 100644
failf(data, "Failed sending HTTP request: Header overflow");
goto fail;
}
@@ -2032,25 +2100,35 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
@@ -2032,25 +2108,35 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
}
if(!end || end == hdbuf)
goto fail;
@@ -1205,7 +1237,7 @@ index e74400a4c..b0e6674f2 100644
failf(data, "Failed sending HTTP request: Header overflow");
goto fail;
}
@@ -2117,10 +2195,13 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
@@ -2117,10 +2203,13 @@ static ssize_t http2_send(struct Curl_easy *data, int sockindex,
++i;
}

153
firefox/patches/curl-impersonate.patch
View File

@@ -706,7 +706,7 @@ index f08a343e3..2bbce4b23 100644
if(result)
return result;
diff --git a/lib/http2.c b/lib/http2.c
index e74400a4c..5df654d04 100644
index e74400a4c..1dd2593a5 100644
--- a/lib/http2.c
+++ b/lib/http2.c
@@ -56,7 +56,7 @@
@@ -718,7 +718,38 @@ index e74400a4c..5df654d04 100644
#ifdef DEBUG_HTTP2
#define H2BUGF(x) x
@@ -1193,14 +1193,18 @@ static void populate_settings(struct Curl_easy *data,
@@ -75,13 +75,20 @@ static int h2_process_pending_input(struct Curl_easy *data,
struct http_conn *httpc,
CURLcode *err);
+
+/*
+ * curl-impersonate: Set the HTTP/2 stream weight to the one used by Firefox
+ * by default to fetch html resources.
+ */
+#define FIREFOX_DEFAULT_STREAM_WEIGHT (42)
+
/*
* Curl_http2_init_state() is called when the easy handle is created and
* allows for HTTP/2 specific init of state.
*/
void Curl_http2_init_state(struct UrlState *state)
{
- state->stream_weight = NGHTTP2_DEFAULT_WEIGHT;
+ state->stream_weight = FIREFOX_DEFAULT_STREAM_WEIGHT;
}
/*
@@ -90,7 +97,7 @@ void Curl_http2_init_state(struct UrlState *state)
*/
void Curl_http2_init_userset(struct UserDefined *set)
{
- set->stream_weight = NGHTTP2_DEFAULT_WEIGHT;
+ set->stream_weight = FIREFOX_DEFAULT_STREAM_WEIGHT;
}
static int http2_getsock(struct Curl_easy *data,
@@ -1193,14 +1200,18 @@ static void populate_settings(struct Curl_easy *data,
{
nghttp2_settings_entry *iv = httpc->local_settings;
@@ -742,7 +773,27 @@ index e74400a4c..5df654d04 100644
httpc->local_settings_num = 3;
}
@@ -1818,7 +1822,8 @@ static ssize_t http2_recv(struct Curl_easy *data, int sockindex,
@@ -1566,12 +1577,18 @@ static ssize_t http2_handle_stream_close(struct connectdata *conn,
* struct.
*/
+/*
+ * curl-impersonate: By default Firefox uses stream 13 as the "parent" of the
+ * stream that fetches the main html resource of the web page.
+ */
+#define FIREFOX_DEFAULT_STREAM_DEP (13)
+
static void h2_pri_spec(struct Curl_easy *data,
nghttp2_priority_spec *pri_spec)
{
struct HTTP *depstream = (data->set.stream_depends_on?
data->set.stream_depends_on->req.p.http:NULL);
- int32_t depstream_id = depstream? depstream->stream_id:0;
+ int32_t depstream_id = depstream? depstream->stream_id:FIREFOX_DEFAULT_STREAM_DEP;
nghttp2_priority_spec_init(pri_spec, depstream_id, data->set.stream_weight,
data->set.stream_depends_e);
data->state.stream_weight = data->set.stream_weight;
@@ -1818,7 +1835,8 @@ static ssize_t http2_recv(struct Curl_easy *data, int sockindex,
/* Index where :authority header field will appear in request header
field list. */
@@ -752,6 +803,102 @@ index e74400a4c..5df654d04 100644
/* USHRT_MAX is 65535 == 0xffff */
#define HEADER_OVERFLOW(x) \
@@ -2276,6 +2294,73 @@ CURLcode Curl_http2_setup(struct Curl_easy *data,
return CURLE_OK;
}
+/*
+ * curl-impersonate: Start with stream id 15 as Firefox does.
+ */
+#define FIREFOX_DEFAULT_STREAM_ID (15)
+
+static CURLcode http2_set_stream_priority(struct Curl_easy *data,
+ int32_t stream_id,
+ int32_t dep_stream_id,
+ int32_t weight)
+{
+ int rv;
+ nghttp2_priority_spec pri_spec;
+ struct connectdata *conn = data->conn;
+ struct http_conn *httpc = &conn->proto.httpc;
+
+ nghttp2_priority_spec_init(&pri_spec, dep_stream_id, weight, 0);
+ rv = nghttp2_submit_priority(httpc->h2, NGHTTP2_FLAG_NONE,
+ stream_id, &pri_spec);
+ if(rv) {
+ failf(data, "nghttp2_submit_priority() failed: %s(%d)",
+ nghttp2_strerror(rv), rv);
+ return CURLE_HTTP2;
+ }
+
+ return CURLE_OK;
+}
+
+
+/*
+ * curl-impersonate: Firefox uses an elaborate scheme of http/2 streams to
+ * split the load for html/js/css/images. It builds a tree of streams with
+ * different weights (priorities) by default and communicates this to the
+ * server. Imitate that behavior.
+ */
+static CURLcode http2_set_stream_priorities(struct Curl_easy *data)
+{
+ CURLcode result;
+ struct connectdata *conn = data->conn;
+ struct http_conn *httpc = &conn->proto.httpc;
+
+ result = http2_set_stream_priority(data, 3, 0, 201);
+ if(result)
+ return result;
+
+ result = http2_set_stream_priority(data, 5, 0, 101);
+ if(result)
+ return result;
+
+ result = http2_set_stream_priority(data, 7, 0, 0);
+ if(result)
+ return result;
+
+ result = http2_set_stream_priority(data, 9, 7, 0);
+ if(result)
+ return result;
+
+ result = http2_set_stream_priority(data, 11, 3, 0);
+ if(result)
+ return result;
+
+ result = http2_set_stream_priority(data, 13, 0, 241);
+ if(result)
+ return result;
+
+ return CURLE_OK;
+}
+
CURLcode Curl_http2_switched(struct Curl_easy *data,
const char *mem, size_t nread)
{
@@ -2284,6 +2369,7 @@ CURLcode Curl_http2_switched(struct Curl_easy *data,
struct http_conn *httpc = &conn->proto.httpc;
int rv;
struct HTTP *stream = data->req.p.http;
+ nghttp2_priority_spec pri_spec;
result = Curl_http2_setup(data, conn);
if(result)
@@ -2338,6 +2424,13 @@ CURLcode Curl_http2_switched(struct Curl_easy *data,
return CURLE_HTTP2;
}
+ result = http2_set_stream_priorities(data);
+ if(result)
+ return result;
+
+ /* Best effort to set the request's stream id to 15, like Firefox does. */
+ nghttp2_session_set_next_stream_id(httpc->h2, FIREFOX_DEFAULT_STREAM_ID);
+
/* we are going to copy mem to httpc->inbuf. This is required since
mem is part of buffer pointed by stream->mem, and callbacks
called by nghttp2_session_mem_recv() will write stream specific
diff --git a/lib/setopt.c b/lib/setopt.c
index 599ed5d99..1baa48e70 100644
--- a/lib/setopt.c