diff --git a/tests/Dockerfile b/tests/Dockerfile index 28d5595..d7b1801 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -9,7 +9,7 @@ FROM python:3.10.1-slim-buster WORKDIR /tests RUN apt-get update && \ - apt-get install -y tcpdump libbrotli1 libnss3 gcc libcurl4-openssl-dev + apt-get install -y tcpdump libbrotli1 libnss3 gcc libcurl4-openssl-dev nghttp2-server COPY requirements.txt requirements.txt diff --git a/tests/README.md b/tests/README.md index 8b414b2..2c19b93 100644 --- a/tests/README.md +++ b/tests/README.md @@ -15,10 +15,10 @@ docker run --rm curl-impersonate-tests This simply runs `pytest` in the container. You can pass additional flags to `pytest` such as `--log-cli-level DEBUG`. ## How the tests work -For each supported browser, a packet capture is started while `curl-impersonate` is run with the relevant wrapper script. The Client Hello message is extracted from the capture, and compared against the known signature of the browser. +For each supported browser, the following tests are performed: +* A packet capture is started while `curl-impersonate` is run with the relevant wrapper script. The Client Hello message is extracted from the capture and compared against the known signature of the browser. +* `curl-impersonate` is run, connecting to a local `nghttpd` server (a simple HTTP/2 server). The HTTP/2 pseudo-headers and headers are extracted from the output log of `nghttpd` and compared to the known headers of the browser. ## What's missing The following tests are still missing: -* Test that `curl-impersonate` sends the HTTP headers in the same order as the browser. -* Test that `curl-impersonate` sends the HTTP/2 pseudo-headers in the same order as the browser. * Test that `curl-impersonate` sends the same HTTP/2 SETTINGS as the browser. diff --git a/tests/minicurl.c b/tests/minicurl.c index 57887e2..f1a4907 100644 --- a/tests/minicurl.c +++ b/tests/minicurl.c @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -20,6 +21,7 @@ struct opts { char *outfile; uint16_t local_port_start; uint16_t local_port_end; + bool insecure; char *url; }; @@ -61,13 +63,15 @@ int parse_opts(int argc, char **argv, struct opts *opts) memset(opts, 0, sizeof(*opts)); + opts->insecure = false; + while (1) { int option_index = 0; static struct option long_options[] = { {"local-port", required_argument, NULL, 'l'} }; - c = getopt_long(argc, argv, "o:", long_options, &option_index); + c = getopt_long(argc, argv, "o:k", long_options, &option_index); if (c == -1) { break; } @@ -84,6 +88,9 @@ int parse_opts(int argc, char **argv, struct opts *opts) case 'o': opts->outfile = optarg; break; + case 'k': + opts->insecure = true; + break; } } @@ -167,6 +174,19 @@ int main(int argc, char *argv[]) } } + if (opts.insecure) { + c = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + if (c) { + fprintf(stderr, "curl_easy_setopt(CURLOPT_SSL_VERIFYPEER) failed\n"); + goto out; + } + c = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); + if (c) { + fprintf(stderr, "curl_easy_setopt(CURLOPT_SSL_VERIFYHOST) failed\n"); + goto out; + } + } + c = curl_easy_setopt(curl, CURLOPT_URL, opts.url); if (c) { fprintf(stderr, "curl_easy_setopt(CURLOPT_URL) failed\n"); diff --git a/tests/signature.py b/tests/signature.py index 016cb85..868fd20 100644 --- a/tests/signature.py +++ b/tests/signature.py @@ -848,19 +848,129 @@ class TLSClientHelloSignature(): ) -class BrowserSignature(): +class HTTP2Signature: + """ + The HTTP/2 signature of a browser. + + In HTTP/2 multiple parameters can be used to fingerprint the browser. + Currently this class contains the following parameters: + * The order of the HTTP/2 pseudo-headers. + * The "regular" HTTP headers sent by the browser upon first connection to a + website. + """ + def __init__(self, + pseudo_headers: List[str], + headers: List[str]): + self.pseudo_headers = pseudo_headers + self.headers = headers + + def _equals(self, other: 'HTTP2Signature', reason: bool = False): + if set(self.pseudo_headers) != set(other.pseudo_headers): + symdiff = list(set(self.pseudo_headers).symmetric_difference( + other.pseudo_headers + )) + msg = (f"HTTP/2 pseudo-headers differ: " + f"Symmetric difference {symdiff}") + return False, msg + + if self.pseudo_headers != other.pseudo_headers: + msg = (f"HTTP/2 pseudo-headers differ in order: " + f"{self.pseudo_headers} != {other.pseudo_headers}") + return False, msg + + if self.headers != other.headers: + msg = (f"HTTP/2 headers differ: " + f"{self.headers} != {other.headers}") + return False, msg + + return True, None + + def equals(self, other: 'HTTP2Signature', reason: bool = False): + """Checks whether two browsers have the same HTTP/2 signature. + + Parameters + ---------- + other : HTTP2Signature + The signature of the other browser. + reason : bool + If True, returns an additional string describing the reason of the + difference in case of a difference, and None otherwise. + """ + equal, msg = self._equals(other) + if reason: + return equal, msg + else: + return equal + + def to_dict(self): + """Serialize to a dict object.""" + return { + "pseudo_headers": self.pseudo_headers, + "headers": self.headers + } + + @classmethod + def from_dict(cls, d): + """Unserialize a HTTP2Signature from a dict. + + Parameters + ---------- + d : dict + HTTP/2 signature encoded to a Python dict. + + Returns + ------- + sig : HTTP2Signature + Signature constructed based on the dict representation. + """ + return HTTP2Signature(**d) + + +class BrowserSignature: """ Represents the network signature of a specific browser based on multiple network parameters. - Currently includes only the signature of the Client Hello message, but - designed to include other parameters (HTTP headers, HTTP2 settings, etc.) + Attributes + ---------- + tls_client_hello : TLSClientHelloSignature + The signature of the browser's TLS Client Hello message. + Can be None, in which case it is ignored. + http2 : HTTP2Signature + The HTTP/2 signature of the browser. + Can be None, in which case it is ignored. """ - def __init__(self, tls_client_hello: TLSClientHelloSignature): + def __init__(self, + tls_client_hello: TLSClientHelloSignature = None, + http2: HTTP2Signature = None): self.tls_client_hello = tls_client_hello + self.http2 = http2 - def equals(self, other: 'BrowserSignature', reason=False): + def _equals(self, other: 'BrowserSignature'): + # If one is None, so must be the other + if (self.tls_client_hello is None) != (other.tls_client_hello is None): + return False, "TLS signature present in one but not the other" + + if self.tls_client_hello is not None: + equal, msg = self.tls_client_hello.equals( + other.tls_client_hello, reason=True + ) + if not equal: + return equal, msg + + # If one is None, so must be the other + if (self.http2 is None) != (other.http2 is None): + return False, "HTTP2 signature present in one but not the other" + + if self.http2 is not None: + equal, msg = self.http2.equals(other.http2, reason=True) + if not equal: + return equal, msg + + return True, None + + def equals(self, other: 'BrowserSignature', reason: bool = False): """Checks whether two browsers have the same network signatures. Parameters @@ -871,21 +981,34 @@ class BrowserSignature(): If True, returns an additional string describing the reason of the difference in case of a difference, and None otherwise. """ - return self.tls_client_hello.equals(other.tls_client_hello, reason) + equal, msg = self._equals(other) + if reason: + return equal, msg + else: + return equal def to_dict(self): """Serialize to a dict object.""" - return { - "tls_client_hello": self.tls_client_hello.to_dict() - } + d = {} + if self.tls_client_hello is not None: + d["tls_client_hello"] = self.tls_client_hello.to_dict() + if self.http2 is not None: + d["http2"] = self.http2.to_dict() + return d @classmethod def from_dict(cls, d): """Unserialize a BrowserSignature from a dict.""" - tls_client_hello = None if d.get("tls_client_hello"): tls_client_hello=TLSClientHelloSignature.from_dict( d["tls_client_hello"] ) + else: + tls_client_hello = None - return BrowserSignature(tls_client_hello=tls_client_hello) + if d.get("http2"): + http2 = HTTP2Signature.from_dict(d["http2"]) + else: + http2 = None + + return BrowserSignature(tls_client_hello=tls_client_hello, http2=http2) diff --git a/tests/signatures.yaml b/tests/signatures.yaml index 63274b3..1f13b04 100644 --- a/tests/signatures.yaml +++ b/tests/signatures.yaml @@ -1,9 +1,11 @@ # Browser signatures database # # Each signature refers to the browser's behavior upon browsing to a site -# not cached or visited before. Each signature contains the various parameters -# in the TLS Client Hello message, and is designed to accomodate other -# parameters as well (such as HTTP headers, HTTP/2 settings). +# not cached or visited before. +# Each signature contains: +# * The parameters in the TLS client hello message. +# * The HTTP/2 HEADERS frame sent by the browser. +# * (planned) The HTTP/2 SETTINGS frame sent by the browser. --- name: chrome_98.0.4758.102_win10 browser: @@ -81,6 +83,25 @@ signature: length: 1 data: !!binary AA== - type: padding + http2: + pseudo_headers: + - ':method' + - ':authority' + - ':scheme' + - ':path' + headers: + - 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"' + - 'sec-ch-ua-mobile: ?0' + - 'sec-ch-ua-platform: "Windows"' + - 'upgrade-insecure-requests: 1' + - 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' + - 'sec-fetch-site: none' + - 'sec-fetch-mode: navigate' + - 'sec-fetch-user: ?1' + - 'sec-fetch-dest: document' + - 'accept-encoding: gzip, deflate, br' + - 'accept-language: en-US,en;q=0.9' --- name: edge_98.0.1108.62_win10 browser: @@ -158,6 +179,25 @@ signature: length: 1 data: !!binary AA== - type: padding + http2: + pseudo_headers: + - ':method' + - ':authority' + - ':scheme' + - ':path' + headers: + - 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="98", "Microsoft Edge";v="98"' + - 'sec-ch-ua-mobile: ?0' + - 'sec-ch-ua-platform: "Windows"' + - 'upgrade-insecure-requests: 1' + - 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.62' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' + - 'sec-fetch-site: none' + - 'sec-fetch-mode: navigate' + - 'sec-fetch-user: ?1' + - 'sec-fetch-dest: document' + - 'accept-encoding: gzip, deflate, br' + - 'accept-language: en-US,en;q=0.9' --- name: firefox_91.6.0esr_win10 browser: @@ -229,6 +269,23 @@ signature: length: 2 record_size_limit: 16385 - type: padding + http2: + pseudo_headers: + - ':method' + - ':path' + - ':authority' + - ':scheme' + headers: + - 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' + - 'accept-language: en-US,en;q=0.5' + - 'accept-encoding: gzip, deflate, br' + - 'upgrade-insecure-requests: 1' + - 'sec-fetch-dest: document' + - 'sec-fetch-mode: navigate' + - 'sec-fetch-site: none' + - 'sec-fetch-user: ?1' + - 'te: trailers' --- name: firefox_95.0.2_win10 browser: @@ -300,6 +357,23 @@ signature: length: 2 record_size_limit: 16385 - type: padding + http2: + pseudo_headers: + - ':method' + - ':path' + - ':authority' + - ':scheme' + headers: + - 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' + - 'accept-language: en-US,en;q=0.5' + - 'accept-encoding: gzip, deflate, br' + - 'upgrade-insecure-requests: 1' + - 'sec-fetch-dest: document' + - 'sec-fetch-mode: navigate' + - 'sec-fetch-site: none' + - 'sec-fetch-user: ?1' + - 'te: trailers' --- name: safari_15.3_macos11.6.4 browser: @@ -372,3 +446,14 @@ signature: length: 1 data: !!binary AA== - type: padding + http2: + pseudo_headers: + - ':method' + - ':scheme' + - ':path' + - ':authority' + headers: + - 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + - 'accept-language: en-us' + - 'accept-encoding: gzip, deflate, br' diff --git a/tests/ssl/dhparam.pem b/tests/ssl/dhparam.pem new file mode 100644 index 0000000..c4562ca --- /dev/null +++ b/tests/ssl/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAv9gTtz/gXIU5qQgb14Ki7i9l0UbRJX0bssggCQcKSr+SrKaWWE9N +FrJzCHFHIk2vY2/XOHs2XWj3BN7xn1y0JwEQQAp6R8h8iXSZzQAzovpt14qrYbgk +JEgx9TYDyj/tq4L0c4Wkwyjz9gKDFc6HppjYXJBMw7iW+cZ8q08easZ+2aoDNikT +UoX0x3hrzJwd/BIHvhNnEa87vP+9cohUZGMGLFKykykXrraTprRxZ6bQrb42RMmI +9X+vjxtBIh1l1Y6UtsnjhkTgFvJA6r3RZ/w3SVmWyFsoIOjrnoLza4sYQpdMpnTV +dyY0tu1ur7sCAWIuZsqZGUyYzUBARKgk0wIBAg== +-----END DH PARAMETERS----- diff --git a/tests/ssl/server.crt b/tests/ssl/server.crt new file mode 100644 index 0000000..65c87ec --- /dev/null +++ b/tests/ssl/server.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUNBtR47HxG8+YmIuj424xy6xxJ5EwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAzMDQxNDM1NTNaFw0yMzAz +MDQxNDM1NTNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDWBLG1Uto4ncPUq525H4op3LNu4MuztBPeMqAwkOlM +8v9XCoP54/Fkg0UebOmzHmJHNGx5dUBdnU92N/b/yMH5UnSxJj0Yd0Zht2iCjVob +Idv4p6zxrUtDXydDtp+0GOVZ2pRi6raljcfE/V7ny3P6XCGxWxeyNTahu42Kt7pJ +N9LmIAvXCwR3GUOQJH2v8L4YxHdXxjWcGR1Fcv7oq691+4sbJS2J5BMkhIa2/5IR +jeZXHUrXbMDmr6AVUlLoniTOB+SdWxSSdHxC/FsfpTPTahn6gFdwYJPlwtxSkeDv +6mRoGSYNyhTy8Oquax7rhlycfiUVeSEVKBeczclcggnJAgMBAAGjUzBRMB0GA1Ud +DgQWBBQndv4aJrTkFWRaszsbFvnMcXotHzAfBgNVHSMEGDAWgBQndv4aJrTkFWRa +szsbFvnMcXotHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQDD +RfFcGK1HiqMN6vSEJ0xbjsQWc7SjufVoBVP2mAtCVcCvBDbHIFNEI2KN9XGSvVX7 +U19oOBimmq/Qjy7HCiGtAw1jgaYrvHYxowFCx8Wpof/uqIjE79Wp7lAhuIE9XHF9 +B/j38CrkzK0XcaAQZfVGS/WsHvCxu6O0L+9CpvFSyTz2er/QYRy15azU5/J0bsda +izyqySFpOl7ohI5t/hF3pFN372psllduajVYWgiNNvJoUMPP+kZNpOY+UXoJaEIe +8Gpo11hKszpCwrDr5O+a9METdB3FmJvtkIFjLNugtDkjd0L6zlBWzqdNFeJ7fvec +QeDpZmTkkFVOjWCPyqQD +-----END CERTIFICATE----- diff --git a/tests/ssl/server.key b/tests/ssl/server.key new file mode 100644 index 0000000..76b20a5 --- /dev/null +++ b/tests/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDWBLG1Uto4ncPU +q525H4op3LNu4MuztBPeMqAwkOlM8v9XCoP54/Fkg0UebOmzHmJHNGx5dUBdnU92 +N/b/yMH5UnSxJj0Yd0Zht2iCjVobIdv4p6zxrUtDXydDtp+0GOVZ2pRi6raljcfE +/V7ny3P6XCGxWxeyNTahu42Kt7pJN9LmIAvXCwR3GUOQJH2v8L4YxHdXxjWcGR1F +cv7oq691+4sbJS2J5BMkhIa2/5IRjeZXHUrXbMDmr6AVUlLoniTOB+SdWxSSdHxC +/FsfpTPTahn6gFdwYJPlwtxSkeDv6mRoGSYNyhTy8Oquax7rhlycfiUVeSEVKBec +zclcggnJAgMBAAECggEBAM6dfYrWUAK0nKimfgCI6HP9s+TpdP8qbLvpGCmK3REC +z2wSpNMNMrCc4o+7Cet4+9xOSiMABYHbKymwYe8Su+GdrzaO+hCypeoUjPrsx/7F +s33dMuOnL6/9HwUKPCg8mL8kfHj6rBYsSJ5vFb6l9nPPml+E192d7f45+S3griGb +1StC5Dwc78E6ZaKRyHyygI0WqZiL8eUpS1w6HOPp4cJK5QgnY544mKkcwd4kUi8B +yeOBNcmGk9QckwaCngu6Ip3U5ivgKbPBZqTZcVGaVS+ggOkk3R5RAjqyVNl09ENs +ztE5eV2W+jmz481hOUSe/XxYp2RTkl1eA9m3TpkG4fkCgYEA7TTg8XtSpQvvRrxM +8wd+CvKoFrx0wn/2wU/l1+fvxe7o8RBq9cjVypnu4BaC0V8WV1/xb6EdjhlQEyfS ++pcDx0GyNErPJfoDLynfopm/jy1hnQW6tgp8xJBdyYgUlDCmimzeA8BrN0ULZR6D +r9vzQB6HvTuOy1DJb4Un7PJkEh8CgYEA5vmAnsr37A/qIsbIiMMwaSNpRIWC4jyt +OJTiAwY95GFV6o5a6FZ8xahZtQ1FeQ8G4r1pf4wzGLOnfpupU1tWWSVnEBufb6jy +SGMqHOVOhCZpI/E2WNqUk3GR6Ba4QNumBKsLDiPW6GxQ6quZjxlRFfTF7P1rpbDh +x4QQWrvodxcCgYEAy9ttEru8vBF0syLzMs4WmcwPf3K5Gcslwt8qlhJDs6TuVvaY +JeFTM0p1y+osxUlmBvNyqFAb+VpxwfSw0iHk4mLohx5fxrCF+guPoctmoOMMiAk7 +fGWo8rlrkN69aNoi1sZXS3wb6KUS9PVzkTiDZnCWkZ/UyZEFfS0/sdhi/lsCgYBn +NAaPbTt3w+inH1ENIsHnyIXJsyo3Mktn48ZU+Z4ABKniAzeFZtebbcyfhE2NePRn +raCM+DUAjY2CmcT/1OjxLjAt11nXB5MyWvS/Mopxq8QA5k+VRh1rACzkmfo8KKi2 +n0JyT/s/oN5K7N/RO8uqVtN1QAqwXyeTAWRZVmrZgwKBgQCeTRdbXRs1S/TKWedS +9hIApeDSxwcuvH3vJJKPbuWme0YDQ82PBZRoMJbjsgp3y+WUlHjL+AoxlIeE4tLy +KN4iFbdnhzKN7kF3oKHq3mfXRogHPxKarEu7DxAqUGk57bELulvXh1AD/OQP7rjx +MjN23ZKh4Owlem34x5yEyri0bw== +-----END PRIVATE KEY----- diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index 903914a..323f8b8 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -1,5 +1,6 @@ import os import io +import re import logging import subprocess @@ -7,7 +8,11 @@ import yaml import dpkt import pytest -from signature import BrowserSignature, TLSClientHelloSignature +from signature import ( + BrowserSignature, + TLSClientHelloSignature, + HTTP2Signature +) @pytest.fixture @@ -93,14 +98,11 @@ class TestSignatureModule: """ Test the TLS Client Hello parsing code. """ - sig = BrowserSignature( - tls_client_hello=TLSClientHelloSignature.from_bytes( - self.CLIENT_HELLO - ) - ) - - sig2 = BrowserSignature.from_dict( - browser_signatures["chrome_98.0.4758.102_win10"]["signature"] + sig = TLSClientHelloSignature.from_bytes(self.CLIENT_HELLO) + sig2 = TLSClientHelloSignature.from_dict( + browser_signatures["chrome_98.0.4758.102_win10"] \ + ["signature"] \ + ["tls_client_hello"] ) equals, reason = sig.equals(sig2, reason=True) @@ -199,7 +201,22 @@ class TestImpersonation: p.terminate() p.wait(timeout=10) - def _run_curl(self, curl_binary, env_vars, url): + @pytest.fixture + def nghttpd(self): + """Initiailize an HTTP/2 server""" + logging.debug(f"Running nghttpd on :8443") + + p = subprocess.Popen([ + "nghttpd", "-v", + "8443", "ssl/server.key", "ssl/server.crt" + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + yield p + + p.terminate() + p.wait(timeout=10) + + def _run_curl(self, curl_binary, env_vars, extra_args, url): env = os.environ.copy() if env_vars: env |= env_vars @@ -209,13 +226,16 @@ class TestImpersonation: logging.debug("Environment variables: {}".format( " ".join([f"{k}={v}" for k, v in env_vars.items()]))) - curl = subprocess.Popen([ + args = [ curl_binary, "-o", "/dev/null", - "--local-port", f"{self.LOCAL_PORTS[0]}-{self.LOCAL_PORTS[1]}", - url - ], env=env) + "--local-port", f"{self.LOCAL_PORTS[0]}-{self.LOCAL_PORTS[1]}" + ] + if extra_args: + args += extra_args + args.append(url) + curl = subprocess.Popen(args, env=env) return curl.wait(timeout=10) def _extract_client_hello(self, pcap: bytes) -> bytes: @@ -249,6 +269,41 @@ class TestImpersonation: return None + def _parse_nghttpd2_output(self, output): + """Parse the output of nghttpd2. + + nghttpd2 in verbose mode writes out the HTTP/2 + headers that the client had sent. + """ + lines = output.decode("utf-8").splitlines() + stream_id = None + for line in lines: + m = re.search(r"recv HEADERS frame.*stream_id=(\d+)", line) + if m: + stream_id = m.group(1) + break + + assert stream_id is not None, \ + "Failed to find HEADERS frame in nghttpd2 output" + + pseudo_headers = [] + headers = [] + for line in lines: + m = re.search(rf"recv \(stream_id={stream_id}\) (.*)", line) + if m: + header = m.group(1) + # If the headers starts with ":" it is a pseudo-header, + # i.e. ":authority". In this case keep only the header name and + # discard the value + if header.startswith(":"): + m = re.match(r"(:\w+):", header) + if m: + pseudo_headers.append(m.group(1)) + else: + headers.append(header) + + return pseudo_headers, headers + @pytest.mark.parametrize( "curl_binary, env_vars, expected_signature", CURL_BINARIES_AND_SIGNATURES @@ -267,7 +322,10 @@ class TestImpersonation: extracts the Client Hello packet from the capture and compares its signature with the expected one defined in the YAML database. """ - ret = self._run_curl(curl_binary, env_vars, self.TEST_URL) + ret = self._run_curl(curl_binary, + env_vars=env_vars, + extra_args=None, + url=self.TEST_URL) assert ret == 0 try: @@ -300,3 +358,48 @@ class TestImpersonation: equals, msg = sig.equals(expected_sig, reason=True) assert equals, msg + + @pytest.mark.parametrize( + "curl_binary, env_vars, expected_signature", + CURL_BINARIES_AND_SIGNATURES + ) + def test_http2_headers(self, + nghttpd, + curl_binary, + env_vars, + browser_signatures, + expected_signature): + ret = self._run_curl(curl_binary, + env_vars=env_vars, + extra_args=["-k"], + url="https://localhost:8443") + assert ret == 0 + try: + output, stderr = nghttpd.communicate(timeout=2) + + # If nghttpd finished running before timeout, it's likely it failed + # with an error. + assert nghttpd.returncode == 0, \ + (f"nghttpd failed with error code {nghttpd.returncode}, " + f"stderr: {stderr}") + except subprocess.TimeoutExpired: + nghttpd.kill() + output, stderr = nghttpd.communicate(timeout=3) + + assert len(output) > 0 + pseudo_headers, headers = self._parse_nghttpd2_output(output) + + logging.debug( + f"Received {len(pseudo_headers)} HTTP/2 pseudo-headers " + f"and {len(headers)} HTTP/2 headers" + ) + + sig = HTTP2Signature(pseudo_headers, headers) + expected_sig = HTTP2Signature.from_dict( + browser_signatures[expected_signature] \ + ["signature"] \ + ["http2"] + ) + + equals, msg = sig.equals(expected_sig, reason=True) + assert equals, msg