From 922a8bb72e1a99e3f4e5f080aa6798d7cd33f123 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sun, 17 Apr 2022 12:47:27 +0300 Subject: [PATCH] Support encoded content in curl_easy_impersonate Set CURLOPT_ACCEPT_ENCODING to an empty string in curl_easy_impersonate() to enable decompression of encoded responses using all built-in compressions. This is similar to adding '--compressed' in the command line curl and is necessary since curl_easy_impersonate() adds the 'Accept-Encoding' header which may cause the server to respond with compressed content. --- chrome/patches/curl-impersonate.patch | 15 ++++++---- firefox/patches/curl-impersonate.patch | 15 ++++++---- tests/test_impersonate.py | 39 ++++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/chrome/patches/curl-impersonate.patch b/chrome/patches/curl-impersonate.patch index 3801b2f..33de539 100644 --- a/chrome/patches/curl-impersonate.patch +++ b/chrome/patches/curl-impersonate.patch @@ -244,7 +244,7 @@ index 769363941..cd59ad4b2 100644 CHECKSRC = $(CS_$(V)) diff --git a/lib/easy.c b/lib/easy.c -index 20293a710..0cea8af50 100644 +index 20293a710..23141fe16 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -80,6 +80,7 @@ @@ -255,7 +255,7 @@ index 20293a710..0cea8af50 100644 /* The last 3 #include files should be in this order */ #include "curl_printf.h" -@@ -282,6 +283,409 @@ void curl_global_cleanup(void) +@@ -282,6 +283,414 @@ void curl_global_cleanup(void) init_flags = 0; } @@ -659,13 +659,18 @@ index 20293a710..0cea8af50 100644 + return ret; + } + ++ /* Always enable all supported compressions. */ ++ ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, ""); ++ if(ret) ++ return ret; ++ + return CURLE_OK; +} + /* * curl_easy_init() is the external interface to alloc, setup and init an * easy handle that is returned. If anything goes wrong, NULL is returned. -@@ -290,6 +694,7 @@ struct Curl_easy *curl_easy_init(void) +@@ -290,6 +699,7 @@ struct Curl_easy *curl_easy_init(void) { CURLcode result; struct Curl_easy *data; @@ -673,7 +678,7 @@ index 20293a710..0cea8af50 100644 /* Make sure we inited the global SSL stuff */ if(!initialized) { -@@ -308,6 +713,22 @@ struct Curl_easy *curl_easy_init(void) +@@ -308,6 +718,22 @@ struct Curl_easy *curl_easy_init(void) return NULL; } @@ -696,7 +701,7 @@ index 20293a710..0cea8af50 100644 return data; } -@@ -878,6 +1299,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) +@@ -878,6 +1304,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) outcurl->state.referer_alloc = TRUE; } diff --git a/firefox/patches/curl-impersonate.patch b/firefox/patches/curl-impersonate.patch index cb6e364..3d81794 100644 --- a/firefox/patches/curl-impersonate.patch +++ b/firefox/patches/curl-impersonate.patch @@ -216,7 +216,7 @@ index 769363941..6e2f1b829 100644 CHECKSRC = $(CS_$(V)) diff --git a/lib/easy.c b/lib/easy.c -index 20293a710..b97ac204e 100644 +index 20293a710..f08403bc0 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -80,6 +80,7 @@ @@ -227,7 +227,7 @@ index 20293a710..b97ac204e 100644 /* The last 3 #include files should be in this order */ #include "curl_printf.h" -@@ -282,6 +283,197 @@ void curl_global_cleanup(void) +@@ -282,6 +283,202 @@ void curl_global_cleanup(void) init_flags = 0; } @@ -419,13 +419,18 @@ index 20293a710..b97ac204e 100644 + return ret; + } + ++ /* Always enable all supported compressions. */ ++ ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, ""); ++ if(ret) ++ return ret; ++ + return CURLE_OK; +} + /* * curl_easy_init() is the external interface to alloc, setup and init an * easy handle that is returned. If anything goes wrong, NULL is returned. -@@ -290,6 +482,7 @@ struct Curl_easy *curl_easy_init(void) +@@ -290,6 +487,7 @@ struct Curl_easy *curl_easy_init(void) { CURLcode result; struct Curl_easy *data; @@ -433,7 +438,7 @@ index 20293a710..b97ac204e 100644 /* Make sure we inited the global SSL stuff */ if(!initialized) { -@@ -308,6 +501,22 @@ struct Curl_easy *curl_easy_init(void) +@@ -308,6 +506,22 @@ struct Curl_easy *curl_easy_init(void) return NULL; } @@ -456,7 +461,7 @@ index 20293a710..b97ac204e 100644 return data; } -@@ -878,6 +1087,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) +@@ -878,6 +1092,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) outcurl->state.referer_alloc = TRUE; } diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index 503f1a5..75c84a1 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -3,6 +3,7 @@ import io import re import logging import subprocess +import tempfile import yaml import dpkt @@ -254,7 +255,8 @@ class TestImpersonation: p.terminate() p.wait(timeout=10) - def _run_curl(self, curl_binary, env_vars, extra_args, url): + def _run_curl(self, curl_binary, env_vars, extra_args, url, + output="/dev/null"): env = os.environ.copy() if env_vars: env.update(env_vars) @@ -266,7 +268,7 @@ class TestImpersonation: args = [ curl_binary, - "-o", "/dev/null", + "-o", output, "--local-port", f"{self.LOCAL_PORTS[0]}-{self.LOCAL_PORTS[1]}" ] if extra_args: @@ -460,3 +462,36 @@ class TestImpersonation: equals, msg = sig.equals(expected_sig, reason=True) assert equals, msg + + @pytest.mark.parametrize( + "curl_binary, env_vars, ld_preload, expected_signature", + CURL_BINARIES_AND_SIGNATURES + ) + def test_content_encoding(self, + pytestconfig, + curl_binary, + env_vars, + ld_preload, + expected_signature): + """ + Ensure the output of curl-impersonate is correct, i.e. that compressed + responses are decoded correctly. + """ + curl_binary = os.path.join( + pytestconfig.getoption("install_dir"), "bin", curl_binary + ) + if ld_preload: + env_vars["LD_PRELOAD"] = os.path.join( + pytestconfig.getoption("install_dir"), "lib", ld_preload + ) + + output = tempfile.mkstemp()[1] + ret = self._run_curl(curl_binary, + env_vars=env_vars, + extra_args=None, + url=self.TEST_URL, + output=output) + assert ret == 0 + + with open(output, "r") as f: + assert "" in f.read()