Allow disabling built-in HTTP headers

Let the user disable the built-in list of HTTP headers when using
libcurl-impersonate, either directly or when replacing it at runtime
with LD_PRELOAD. This is intended to give the user more precise control
over the content and order of the HTTP headers.

To support this, the curl_easy_impersonate() now has an added argument
that can be set to 0, in which case the built-in list of HTTP headers
used by libcurl-impersonate will not be automatically sent. Instead,
the user is expected to supply all the headers by themselves using the
standard CURLOPT_HTTPHEADER libcurl option.

When using LD_PRELOAD to inject libcurl-impersonate, one can disable
the built-in headers by setting the CURL_IMPERSONATE_HEADERS
environment variable to "no".

tests/minicurl.c

@@ -18,6 +18,7 @@
 
 /* Support up to 16 URLs */
 #define MAX_URLS 16
+
 /* Command line options. */
 struct opts {
     char *outfile;
@@ -25,6 +26,7 @@ struct opts {
     uint16_t local_port_end;
     bool insecure;
     char *urls[MAX_URLS];
+    struct curl_slist *headers;
 };
 
 int parse_ports_range(char *str, uint16_t *start, uint16_t *end)
@@ -63,6 +65,7 @@ int parse_opts(int argc, char **argv, struct opts *opts)
     int c;
     int r;
     int i;
+    struct curl_slist *tmp;
 
     memset(opts, 0, sizeof(*opts));
 
@@ -71,10 +74,12 @@ int parse_opts(int argc, char **argv, struct opts *opts)
     while (1) {
         int option_index = 0;
         static struct option long_options[] = {
-            {"local-port", required_argument, NULL, 'l'}
+            {"header", required_argument, NULL, 'H'},
+            {"local-port", required_argument, NULL, 'l'},
+            {0, 0, NULL, 0}
         };
 
-        c = getopt_long(argc, argv, "o:k", long_options, &option_index);
+        c = getopt_long(argc, argv, "o:kH:", long_options, &option_index);
         if (c == -1) {
             break;
         }
@@ -94,16 +99,29 @@ int parse_opts(int argc, char **argv, struct opts *opts)
         case 'k':
             opts->insecure = true;
             break;
+        case 'H':
+            tmp = curl_slist_append(opts->headers, optarg);
+            if (!tmp) {
+                fprintf(stderr, "curl_slist_append() failed\n");
+                if (opts->headers) {
+                    curl_slist_free_all(opts->headers);
+                }
+                return 1;
+            }
+            opts->headers = tmp;
+            break;
+        case '?':
+            break;
         }
     }
 
     /* No URL supplied. */
+    i = 0;
     if (optind >= argc) {
         return 1;
     }
 
     /* The rest of the options are URLs */
-    i = 0;
     while (optind < argc) {
         opts->urls[i++] = argv[optind++];
     }
@@ -111,6 +129,13 @@ int parse_opts(int argc, char **argv, struct opts *opts)
     return 0;
 }
 
+void clean_opts(struct opts *opts)
+{
+    if (opts->headers) {
+        curl_slist_free_all(opts->headers);
+    }
+}
+
 /* Set all options except for the URL. */
 int set_opts(CURL *curl, struct opts *opts, FILE *file)
 {
@@ -160,6 +185,14 @@ int set_opts(CURL *curl, struct opts *opts, FILE *file)
         }
     }
 
+    if (opts->headers) {
+        c = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, opts->headers);
+        if (c) {
+            fprintf(stderr, "curl_easy_setopt(CURLOPT_HTTPHEADER) failed\n");
+            return 1;
+        }
+    }
+
     return 0;
 }
 
@@ -180,7 +213,7 @@ int main(int argc, char *argv[])
         file = fopen(opts.outfile, "w");
         if (!file) {
             fprintf(stderr, "Failed opening %s for writing\n", opts.outfile);
-            exit(1);
+            goto out_clean_opts;
         }
     } else {
         file = stdout;
@@ -231,5 +264,7 @@ out_close:
     if (file) {
         fclose(file);
     }
+out_clean_opts:
+    clean_opts(&opts);
     return c;
 }

tests/test_impersonate.py

@@ -8,6 +8,7 @@ import logging
 import pathlib
 import subprocess
 import tempfile
+import itertools
 from typing import List
 
 import yaml
@@ -628,3 +629,75 @@ class TestImpersonation:
                 "<html>" in body or
                 "<!doctype html>" in body
             )
+
+    @pytest.mark.parametrize(
+        "curl_binary, env_vars, ld_preload",
+        [
+            (
+                "minicurl",
+                {
+                    "CURL_IMPERSONATE": "chrome101",
+                    "CURL_IMPERSONATE_HEADERS": "no"
+                },
+                "libcurl-impersonate-chrome"
+            ),
+            (
+                "minicurl",
+                {
+                    "CURL_IMPERSONATE": "ff102",
+                    "CURL_IMPERSONATE_HEADERS": "no"
+                },
+                "libcurl-impersonate-ff",
+            )
+        ]
+    )
+    async def test_no_builtin_headers(self,
+                                      pytestconfig,
+                                      nghttpd,
+                                      curl_binary,
+                                      env_vars,
+                                      ld_preload):
+        """
+        Ensure the built-in headers of libcurl-impersonate are not added when
+        the CURL_IMPERSONATE_HEADERS environment variable is set to "no".
+        """
+        curl_binary = os.path.join(
+            pytestconfig.getoption("install_dir"), "bin", curl_binary
+        )
+
+        if not sys.platform.startswith("linux"):
+            pytest.skip()
+
+        self._set_ld_preload(env_vars, os.path.join(
+            pytestconfig.getoption("install_dir"), "lib", ld_preload
+        ))
+
+        # Use some custom headers with a specific order.
+        # We will test that the headers are sent in the exact given order, as
+        # it is important for users to be able to control the exact headers
+        # content and order.
+        headers = [
+            "X-Hello: World",
+            "Accept: application/json",
+            "X-Goodbye: World",
+            "Accept-Encoding: deflate, gzip, br"
+            "X-Foo: Bar",
+            "User-Agent: curl-impersonate"
+        ]
+        header_args = list(itertools.chain(*[
+            ["-H", header]
+            for header in  headers
+        ]))
+
+        ret = self._run_curl(curl_binary,
+                             env_vars=env_vars,
+                             extra_args=["-k"] + header_args,
+                             urls=["https://localhost:8443"])
+        assert ret == 0
+
+        output = await self._read_proc_output(nghttpd, timeout=2)
+
+        assert len(output) > 0
+        _, output_headers = self._parse_nghttpd2_output(output)
+        for i, header in enumerate(output_headers):
+            assert header.lower() == headers[i].lower()