Support encoded content in curl_easy_impersonate

Set CURLOPT_ACCEPT_ENCODING to an empty string in
curl_easy_impersonate() to enable decompression of encoded responses
using all built-in compressions. This is similar to adding
'--compressed' in the command line curl and is necessary since
curl_easy_impersonate() adds the 'Accept-Encoding' header which may
cause the server to respond with compressed content.

chrome/patches/curl-impersonate.patch

@@ -244,7 +244,7 @@ index 769363941..cd59ad4b2 100644
  
  CHECKSRC = $(CS_$(V))
 diff --git a/lib/easy.c b/lib/easy.c
-index 20293a710..0cea8af50 100644
+index 20293a710..23141fe16 100644
 --- a/lib/easy.c
 +++ b/lib/easy.c
 @@ -80,6 +80,7 @@
@@ -255,7 +255,7 @@ index 20293a710..0cea8af50 100644
  
  /* The last 3 #include files should be in this order */
  #include "curl_printf.h"
-@@ -282,6 +283,409 @@ void curl_global_cleanup(void)
+@@ -282,6 +283,414 @@ void curl_global_cleanup(void)
    init_flags  = 0;
  }
  
@@ -659,13 +659,18 @@ index 20293a710..0cea8af50 100644
 +      return ret;
 +  }
 +
++  /* Always enable all supported compressions. */
++  ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, "");
++  if(ret)
++    return ret;
++
 +  return CURLE_OK;
 +}
 +
  /*
   * curl_easy_init() is the external interface to alloc, setup and init an
   * easy handle that is returned. If anything goes wrong, NULL is returned.
-@@ -290,6 +694,7 @@ struct Curl_easy *curl_easy_init(void)
+@@ -290,6 +699,7 @@ struct Curl_easy *curl_easy_init(void)
  {
    CURLcode result;
    struct Curl_easy *data;
@@ -673,7 +678,7 @@ index 20293a710..0cea8af50 100644
  
    /* Make sure we inited the global SSL stuff */
    if(!initialized) {
-@@ -308,6 +713,22 @@ struct Curl_easy *curl_easy_init(void)
+@@ -308,6 +718,22 @@ struct Curl_easy *curl_easy_init(void)
      return NULL;
    }
  
@@ -696,7 +701,7 @@ index 20293a710..0cea8af50 100644
    return data;
  }
  
-@@ -878,6 +1299,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
+@@ -878,6 +1304,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
      outcurl->state.referer_alloc = TRUE;
    }
  

firefox/patches/curl-impersonate.patch

@@ -216,7 +216,7 @@ index 769363941..6e2f1b829 100644
  
  CHECKSRC = $(CS_$(V))
 diff --git a/lib/easy.c b/lib/easy.c
-index 20293a710..b97ac204e 100644
+index 20293a710..f08403bc0 100644
 --- a/lib/easy.c
 +++ b/lib/easy.c
 @@ -80,6 +80,7 @@
@@ -227,7 +227,7 @@ index 20293a710..b97ac204e 100644
  
  /* The last 3 #include files should be in this order */
  #include "curl_printf.h"
-@@ -282,6 +283,197 @@ void curl_global_cleanup(void)
+@@ -282,6 +283,202 @@ void curl_global_cleanup(void)
    init_flags  = 0;
  }
  
@@ -419,13 +419,18 @@ index 20293a710..b97ac204e 100644
 +      return ret;
 +  }
 +
++  /* Always enable all supported compressions. */
++  ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, "");
++  if(ret)
++    return ret;
++
 +  return CURLE_OK;
 +}
 +
  /*
   * curl_easy_init() is the external interface to alloc, setup and init an
   * easy handle that is returned. If anything goes wrong, NULL is returned.
-@@ -290,6 +482,7 @@ struct Curl_easy *curl_easy_init(void)
+@@ -290,6 +487,7 @@ struct Curl_easy *curl_easy_init(void)
  {
    CURLcode result;
    struct Curl_easy *data;
@@ -433,7 +438,7 @@ index 20293a710..b97ac204e 100644
  
    /* Make sure we inited the global SSL stuff */
    if(!initialized) {
-@@ -308,6 +501,22 @@ struct Curl_easy *curl_easy_init(void)
+@@ -308,6 +506,22 @@ struct Curl_easy *curl_easy_init(void)
      return NULL;
    }
  
@@ -456,7 +461,7 @@ index 20293a710..b97ac204e 100644
    return data;
  }
  
-@@ -878,6 +1087,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
+@@ -878,6 +1092,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
      outcurl->state.referer_alloc = TRUE;
    }
  

tests/test_impersonate.py

@@ -3,6 +3,7 @@ import io
 import re
 import logging
 import subprocess
+import tempfile
 
 import yaml
 import dpkt
@@ -254,7 +255,8 @@ class TestImpersonation:
         p.terminate()
         p.wait(timeout=10)
 
-    def _run_curl(self, curl_binary, env_vars, extra_args, url):
+    def _run_curl(self, curl_binary, env_vars, extra_args, url,
+                  output="/dev/null"):
         env = os.environ.copy()
         if env_vars:
             env.update(env_vars)
@@ -266,7 +268,7 @@ class TestImpersonation:
 
         args = [
             curl_binary,
-            "-o", "/dev/null",
+            "-o", output,
             "--local-port", f"{self.LOCAL_PORTS[0]}-{self.LOCAL_PORTS[1]}"
         ]
         if extra_args:
@@ -460,3 +462,36 @@ class TestImpersonation:
 
         equals, msg = sig.equals(expected_sig, reason=True)
         assert equals, msg
+
+    @pytest.mark.parametrize(
+        "curl_binary, env_vars, ld_preload, expected_signature",
+        CURL_BINARIES_AND_SIGNATURES
+    )
+    def test_content_encoding(self,
+                              pytestconfig,
+                              curl_binary,
+                              env_vars,
+                              ld_preload,
+                              expected_signature):
+        """
+        Ensure the output of curl-impersonate is correct, i.e. that compressed
+        responses are decoded correctly.
+        """
+        curl_binary = os.path.join(
+            pytestconfig.getoption("install_dir"), "bin", curl_binary
+        )
+        if ld_preload:
+            env_vars["LD_PRELOAD"] = os.path.join(
+                pytestconfig.getoption("install_dir"), "lib", ld_preload
+            )
+
+        output = tempfile.mkstemp()[1]
+        ret = self._run_curl(curl_binary,
+                             env_vars=env_vars,
+                             extra_args=None,
+                             url=self.TEST_URL,
+                             output=output)
+        assert ret == 0
+
+        with open(output, "r") as f:
+            assert "<!DOCTYPE html>" in f.read()