mirror of
https://github.com/lwthiker/curl-impersonate.git
synced 2025-08-09 05:09:36 +00:00
Test curl-impersonate's HTTP headers
Add tests to verify that the HTTP headers and HTTP/2 pseudo-headers generated by curl-impersonate match the expected ones from the browser. The test uses a local nghttpd HTTP/2 server instance with a self-signed certificate.
This commit is contained in:
@@ -3,7 +3,7 @@ FROM python:3.10.1-slim-buster
|
|||||||
WORKDIR /tests
|
WORKDIR /tests
|
||||||
|
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install -y tcpdump libbrotli1 libnss3 gcc libcurl4-openssl-dev
|
apt-get install -y tcpdump libbrotli1 libnss3 gcc libcurl4-openssl-dev nghttp2-server
|
||||||
|
|
||||||
COPY requirements.txt requirements.txt
|
COPY requirements.txt requirements.txt
|
||||||
|
|
||||||
|
@@ -15,10 +15,10 @@ docker run --rm curl-impersonate-tests
|
|||||||
This simply runs `pytest` in the container. You can pass additional flags to `pytest` such as `--log-cli-level DEBUG`.
|
This simply runs `pytest` in the container. You can pass additional flags to `pytest` such as `--log-cli-level DEBUG`.
|
||||||
|
|
||||||
## How the tests work
|
## How the tests work
|
||||||
For each supported browser, a packet capture is started while `curl-impersonate` is run with the relevant wrapper script. The Client Hello message is extracted from the capture, and compared against the known signature of the browser.
|
For each supported browser, the following tests are performed:
|
||||||
|
* A packet capture is started while `curl-impersonate` is run with the relevant wrapper script. The Client Hello message is extracted from the capture and compared against the known signature of the browser.
|
||||||
|
* `curl-impersonate` is run, connecting to a local `nghttpd` server (a simple HTTP/2 server). The HTTP/2 pseudo-headers and headers are extracted from the output log of `nghttpd` and compared to the known headers of the browser.
|
||||||
|
|
||||||
## What's missing
|
## What's missing
|
||||||
The following tests are still missing:
|
The following tests are still missing:
|
||||||
* Test that `curl-impersonate` sends the HTTP headers in the same order as the browser.
|
|
||||||
* Test that `curl-impersonate` sends the HTTP/2 pseudo-headers in the same order as the browser.
|
|
||||||
* Test that `curl-impersonate` sends the same HTTP/2 SETTINGS as the browser.
|
* Test that `curl-impersonate` sends the same HTTP/2 SETTINGS as the browser.
|
||||||
|
@@ -12,6 +12,7 @@
|
|||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <getopt.h>
|
#include <getopt.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
#include <curl/curl.h>
|
#include <curl/curl.h>
|
||||||
|
|
||||||
@@ -20,6 +21,7 @@ struct opts {
|
|||||||
char *outfile;
|
char *outfile;
|
||||||
uint16_t local_port_start;
|
uint16_t local_port_start;
|
||||||
uint16_t local_port_end;
|
uint16_t local_port_end;
|
||||||
|
bool insecure;
|
||||||
char *url;
|
char *url;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -61,13 +63,15 @@ int parse_opts(int argc, char **argv, struct opts *opts)
|
|||||||
|
|
||||||
memset(opts, 0, sizeof(*opts));
|
memset(opts, 0, sizeof(*opts));
|
||||||
|
|
||||||
|
opts->insecure = false;
|
||||||
|
|
||||||
while (1) {
|
while (1) {
|
||||||
int option_index = 0;
|
int option_index = 0;
|
||||||
static struct option long_options[] = {
|
static struct option long_options[] = {
|
||||||
{"local-port", required_argument, NULL, 'l'}
|
{"local-port", required_argument, NULL, 'l'}
|
||||||
};
|
};
|
||||||
|
|
||||||
c = getopt_long(argc, argv, "o:", long_options, &option_index);
|
c = getopt_long(argc, argv, "o:k", long_options, &option_index);
|
||||||
if (c == -1) {
|
if (c == -1) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -84,6 +88,9 @@ int parse_opts(int argc, char **argv, struct opts *opts)
|
|||||||
case 'o':
|
case 'o':
|
||||||
opts->outfile = optarg;
|
opts->outfile = optarg;
|
||||||
break;
|
break;
|
||||||
|
case 'k':
|
||||||
|
opts->insecure = true;
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,6 +174,19 @@ int main(int argc, char *argv[])
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (opts.insecure) {
|
||||||
|
c = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
|
||||||
|
if (c) {
|
||||||
|
fprintf(stderr, "curl_easy_setopt(CURLOPT_SSL_VERIFYPEER) failed\n");
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
c = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
|
||||||
|
if (c) {
|
||||||
|
fprintf(stderr, "curl_easy_setopt(CURLOPT_SSL_VERIFYHOST) failed\n");
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
c = curl_easy_setopt(curl, CURLOPT_URL, opts.url);
|
c = curl_easy_setopt(curl, CURLOPT_URL, opts.url);
|
||||||
if (c) {
|
if (c) {
|
||||||
fprintf(stderr, "curl_easy_setopt(CURLOPT_URL) failed\n");
|
fprintf(stderr, "curl_easy_setopt(CURLOPT_URL) failed\n");
|
||||||
|
@@ -848,19 +848,129 @@ class TLSClientHelloSignature():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class BrowserSignature():
|
class HTTP2Signature:
|
||||||
|
"""
|
||||||
|
The HTTP/2 signature of a browser.
|
||||||
|
|
||||||
|
In HTTP/2 multiple parameters can be used to fingerprint the browser.
|
||||||
|
Currently this class contains the following parameters:
|
||||||
|
* The order of the HTTP/2 pseudo-headers.
|
||||||
|
* The "regular" HTTP headers sent by the browser upon first connection to a
|
||||||
|
website.
|
||||||
|
"""
|
||||||
|
def __init__(self,
|
||||||
|
pseudo_headers: List[str],
|
||||||
|
headers: List[str]):
|
||||||
|
self.pseudo_headers = pseudo_headers
|
||||||
|
self.headers = headers
|
||||||
|
|
||||||
|
def _equals(self, other: 'HTTP2Signature', reason: bool = False):
|
||||||
|
if set(self.pseudo_headers) != set(other.pseudo_headers):
|
||||||
|
symdiff = list(set(self.pseudo_headers).symmetric_difference(
|
||||||
|
other.pseudo_headers
|
||||||
|
))
|
||||||
|
msg = (f"HTTP/2 pseudo-headers differ: "
|
||||||
|
f"Symmetric difference {symdiff}")
|
||||||
|
return False, msg
|
||||||
|
|
||||||
|
if self.pseudo_headers != other.pseudo_headers:
|
||||||
|
msg = (f"HTTP/2 pseudo-headers differ in order: "
|
||||||
|
f"{self.pseudo_headers} != {other.pseudo_headers}")
|
||||||
|
return False, msg
|
||||||
|
|
||||||
|
if self.headers != other.headers:
|
||||||
|
msg = (f"HTTP/2 headers differ: "
|
||||||
|
f"{self.headers} != {other.headers}")
|
||||||
|
return False, msg
|
||||||
|
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
def equals(self, other: 'HTTP2Signature', reason: bool = False):
|
||||||
|
"""Checks whether two browsers have the same HTTP/2 signature.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
other : HTTP2Signature
|
||||||
|
The signature of the other browser.
|
||||||
|
reason : bool
|
||||||
|
If True, returns an additional string describing the reason of the
|
||||||
|
difference in case of a difference, and None otherwise.
|
||||||
|
"""
|
||||||
|
equal, msg = self._equals(other)
|
||||||
|
if reason:
|
||||||
|
return equal, msg
|
||||||
|
else:
|
||||||
|
return equal
|
||||||
|
|
||||||
|
def to_dict(self):
|
||||||
|
"""Serialize to a dict object."""
|
||||||
|
return {
|
||||||
|
"pseudo_headers": self.pseudo_headers,
|
||||||
|
"headers": self.headers
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, d):
|
||||||
|
"""Unserialize a HTTP2Signature from a dict.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
d : dict
|
||||||
|
HTTP/2 signature encoded to a Python dict.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
sig : HTTP2Signature
|
||||||
|
Signature constructed based on the dict representation.
|
||||||
|
"""
|
||||||
|
return HTTP2Signature(**d)
|
||||||
|
|
||||||
|
|
||||||
|
class BrowserSignature:
|
||||||
"""
|
"""
|
||||||
Represents the network signature of a specific browser based on multiple
|
Represents the network signature of a specific browser based on multiple
|
||||||
network parameters.
|
network parameters.
|
||||||
|
|
||||||
Currently includes only the signature of the Client Hello message, but
|
Attributes
|
||||||
designed to include other parameters (HTTP headers, HTTP2 settings, etc.)
|
----------
|
||||||
|
tls_client_hello : TLSClientHelloSignature
|
||||||
|
The signature of the browser's TLS Client Hello message.
|
||||||
|
Can be None, in which case it is ignored.
|
||||||
|
http2 : HTTP2Signature
|
||||||
|
The HTTP/2 signature of the browser.
|
||||||
|
Can be None, in which case it is ignored.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, tls_client_hello: TLSClientHelloSignature):
|
def __init__(self,
|
||||||
|
tls_client_hello: TLSClientHelloSignature = None,
|
||||||
|
http2: HTTP2Signature = None):
|
||||||
self.tls_client_hello = tls_client_hello
|
self.tls_client_hello = tls_client_hello
|
||||||
|
self.http2 = http2
|
||||||
|
|
||||||
def equals(self, other: 'BrowserSignature', reason=False):
|
def _equals(self, other: 'BrowserSignature'):
|
||||||
|
# If one is None, so must be the other
|
||||||
|
if (self.tls_client_hello is None) != (other.tls_client_hello is None):
|
||||||
|
return False, "TLS signature present in one but not the other"
|
||||||
|
|
||||||
|
if self.tls_client_hello is not None:
|
||||||
|
equal, msg = self.tls_client_hello.equals(
|
||||||
|
other.tls_client_hello, reason=True
|
||||||
|
)
|
||||||
|
if not equal:
|
||||||
|
return equal, msg
|
||||||
|
|
||||||
|
# If one is None, so must be the other
|
||||||
|
if (self.http2 is None) != (other.http2 is None):
|
||||||
|
return False, "HTTP2 signature present in one but not the other"
|
||||||
|
|
||||||
|
if self.http2 is not None:
|
||||||
|
equal, msg = self.http2.equals(other.http2, reason=True)
|
||||||
|
if not equal:
|
||||||
|
return equal, msg
|
||||||
|
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
def equals(self, other: 'BrowserSignature', reason: bool = False):
|
||||||
"""Checks whether two browsers have the same network signatures.
|
"""Checks whether two browsers have the same network signatures.
|
||||||
|
|
||||||
Parameters
|
Parameters
|
||||||
@@ -871,21 +981,34 @@ class BrowserSignature():
|
|||||||
If True, returns an additional string describing the reason of the
|
If True, returns an additional string describing the reason of the
|
||||||
difference in case of a difference, and None otherwise.
|
difference in case of a difference, and None otherwise.
|
||||||
"""
|
"""
|
||||||
return self.tls_client_hello.equals(other.tls_client_hello, reason)
|
equal, msg = self._equals(other)
|
||||||
|
if reason:
|
||||||
|
return equal, msg
|
||||||
|
else:
|
||||||
|
return equal
|
||||||
|
|
||||||
def to_dict(self):
|
def to_dict(self):
|
||||||
"""Serialize to a dict object."""
|
"""Serialize to a dict object."""
|
||||||
return {
|
d = {}
|
||||||
"tls_client_hello": self.tls_client_hello.to_dict()
|
if self.tls_client_hello is not None:
|
||||||
}
|
d["tls_client_hello"] = self.tls_client_hello.to_dict()
|
||||||
|
if self.http2 is not None:
|
||||||
|
d["http2"] = self.http2.to_dict()
|
||||||
|
return d
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_dict(cls, d):
|
def from_dict(cls, d):
|
||||||
"""Unserialize a BrowserSignature from a dict."""
|
"""Unserialize a BrowserSignature from a dict."""
|
||||||
tls_client_hello = None
|
|
||||||
if d.get("tls_client_hello"):
|
if d.get("tls_client_hello"):
|
||||||
tls_client_hello=TLSClientHelloSignature.from_dict(
|
tls_client_hello=TLSClientHelloSignature.from_dict(
|
||||||
d["tls_client_hello"]
|
d["tls_client_hello"]
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
tls_client_hello = None
|
||||||
|
|
||||||
return BrowserSignature(tls_client_hello=tls_client_hello)
|
if d.get("http2"):
|
||||||
|
http2 = HTTP2Signature.from_dict(d["http2"])
|
||||||
|
else:
|
||||||
|
http2 = None
|
||||||
|
|
||||||
|
return BrowserSignature(tls_client_hello=tls_client_hello, http2=http2)
|
||||||
|
@@ -1,9 +1,11 @@
|
|||||||
# Browser signatures database
|
# Browser signatures database
|
||||||
#
|
#
|
||||||
# Each signature refers to the browser's behavior upon browsing to a site
|
# Each signature refers to the browser's behavior upon browsing to a site
|
||||||
# not cached or visited before. Each signature contains the various parameters
|
# not cached or visited before.
|
||||||
# in the TLS Client Hello message, and is designed to accomodate other
|
# Each signature contains:
|
||||||
# parameters as well (such as HTTP headers, HTTP/2 settings).
|
# * The parameters in the TLS client hello message.
|
||||||
|
# * The HTTP/2 HEADERS frame sent by the browser.
|
||||||
|
# * (planned) The HTTP/2 SETTINGS frame sent by the browser.
|
||||||
---
|
---
|
||||||
name: chrome_98.0.4758.102_win10
|
name: chrome_98.0.4758.102_win10
|
||||||
browser:
|
browser:
|
||||||
@@ -81,6 +83,25 @@ signature:
|
|||||||
length: 1
|
length: 1
|
||||||
data: !!binary AA==
|
data: !!binary AA==
|
||||||
- type: padding
|
- type: padding
|
||||||
|
http2:
|
||||||
|
pseudo_headers:
|
||||||
|
- ':method'
|
||||||
|
- ':authority'
|
||||||
|
- ':scheme'
|
||||||
|
- ':path'
|
||||||
|
headers:
|
||||||
|
- 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"'
|
||||||
|
- 'sec-ch-ua-mobile: ?0'
|
||||||
|
- 'sec-ch-ua-platform: "Windows"'
|
||||||
|
- 'upgrade-insecure-requests: 1'
|
||||||
|
- 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36'
|
||||||
|
- 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
|
||||||
|
- 'sec-fetch-site: none'
|
||||||
|
- 'sec-fetch-mode: navigate'
|
||||||
|
- 'sec-fetch-user: ?1'
|
||||||
|
- 'sec-fetch-dest: document'
|
||||||
|
- 'accept-encoding: gzip, deflate, br'
|
||||||
|
- 'accept-language: en-US,en;q=0.9'
|
||||||
---
|
---
|
||||||
name: edge_98.0.1108.62_win10
|
name: edge_98.0.1108.62_win10
|
||||||
browser:
|
browser:
|
||||||
@@ -158,6 +179,25 @@ signature:
|
|||||||
length: 1
|
length: 1
|
||||||
data: !!binary AA==
|
data: !!binary AA==
|
||||||
- type: padding
|
- type: padding
|
||||||
|
http2:
|
||||||
|
pseudo_headers:
|
||||||
|
- ':method'
|
||||||
|
- ':authority'
|
||||||
|
- ':scheme'
|
||||||
|
- ':path'
|
||||||
|
headers:
|
||||||
|
- 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="98", "Microsoft Edge";v="98"'
|
||||||
|
- 'sec-ch-ua-mobile: ?0'
|
||||||
|
- 'sec-ch-ua-platform: "Windows"'
|
||||||
|
- 'upgrade-insecure-requests: 1'
|
||||||
|
- 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.62'
|
||||||
|
- 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
|
||||||
|
- 'sec-fetch-site: none'
|
||||||
|
- 'sec-fetch-mode: navigate'
|
||||||
|
- 'sec-fetch-user: ?1'
|
||||||
|
- 'sec-fetch-dest: document'
|
||||||
|
- 'accept-encoding: gzip, deflate, br'
|
||||||
|
- 'accept-language: en-US,en;q=0.9'
|
||||||
---
|
---
|
||||||
name: firefox_91.6.0esr_win10
|
name: firefox_91.6.0esr_win10
|
||||||
browser:
|
browser:
|
||||||
@@ -229,6 +269,23 @@ signature:
|
|||||||
length: 2
|
length: 2
|
||||||
record_size_limit: 16385
|
record_size_limit: 16385
|
||||||
- type: padding
|
- type: padding
|
||||||
|
http2:
|
||||||
|
pseudo_headers:
|
||||||
|
- ':method'
|
||||||
|
- ':path'
|
||||||
|
- ':authority'
|
||||||
|
- ':scheme'
|
||||||
|
headers:
|
||||||
|
- 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0'
|
||||||
|
- 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||||
|
- 'accept-language: en-US,en;q=0.5'
|
||||||
|
- 'accept-encoding: gzip, deflate, br'
|
||||||
|
- 'upgrade-insecure-requests: 1'
|
||||||
|
- 'sec-fetch-dest: document'
|
||||||
|
- 'sec-fetch-mode: navigate'
|
||||||
|
- 'sec-fetch-site: none'
|
||||||
|
- 'sec-fetch-user: ?1'
|
||||||
|
- 'te: trailers'
|
||||||
---
|
---
|
||||||
name: firefox_95.0.2_win10
|
name: firefox_95.0.2_win10
|
||||||
browser:
|
browser:
|
||||||
@@ -300,6 +357,23 @@ signature:
|
|||||||
length: 2
|
length: 2
|
||||||
record_size_limit: 16385
|
record_size_limit: 16385
|
||||||
- type: padding
|
- type: padding
|
||||||
|
http2:
|
||||||
|
pseudo_headers:
|
||||||
|
- ':method'
|
||||||
|
- ':path'
|
||||||
|
- ':authority'
|
||||||
|
- ':scheme'
|
||||||
|
headers:
|
||||||
|
- 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0'
|
||||||
|
- 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
|
||||||
|
- 'accept-language: en-US,en;q=0.5'
|
||||||
|
- 'accept-encoding: gzip, deflate, br'
|
||||||
|
- 'upgrade-insecure-requests: 1'
|
||||||
|
- 'sec-fetch-dest: document'
|
||||||
|
- 'sec-fetch-mode: navigate'
|
||||||
|
- 'sec-fetch-site: none'
|
||||||
|
- 'sec-fetch-user: ?1'
|
||||||
|
- 'te: trailers'
|
||||||
---
|
---
|
||||||
name: safari_15.3_macos11.6.4
|
name: safari_15.3_macos11.6.4
|
||||||
browser:
|
browser:
|
||||||
@@ -372,3 +446,14 @@ signature:
|
|||||||
length: 1
|
length: 1
|
||||||
data: !!binary AA==
|
data: !!binary AA==
|
||||||
- type: padding
|
- type: padding
|
||||||
|
http2:
|
||||||
|
pseudo_headers:
|
||||||
|
- ':method'
|
||||||
|
- ':scheme'
|
||||||
|
- ':path'
|
||||||
|
- ':authority'
|
||||||
|
headers:
|
||||||
|
- 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15'
|
||||||
|
- 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
|
||||||
|
- 'accept-language: en-us'
|
||||||
|
- 'accept-encoding: gzip, deflate, br'
|
||||||
|
8
tests/ssl/dhparam.pem
Normal file
8
tests/ssl/dhparam.pem
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
-----BEGIN DH PARAMETERS-----
|
||||||
|
MIIBCAKCAQEAv9gTtz/gXIU5qQgb14Ki7i9l0UbRJX0bssggCQcKSr+SrKaWWE9N
|
||||||
|
FrJzCHFHIk2vY2/XOHs2XWj3BN7xn1y0JwEQQAp6R8h8iXSZzQAzovpt14qrYbgk
|
||||||
|
JEgx9TYDyj/tq4L0c4Wkwyjz9gKDFc6HppjYXJBMw7iW+cZ8q08easZ+2aoDNikT
|
||||||
|
UoX0x3hrzJwd/BIHvhNnEa87vP+9cohUZGMGLFKykykXrraTprRxZ6bQrb42RMmI
|
||||||
|
9X+vjxtBIh1l1Y6UtsnjhkTgFvJA6r3RZ/w3SVmWyFsoIOjrnoLza4sYQpdMpnTV
|
||||||
|
dyY0tu1ur7sCAWIuZsqZGUyYzUBARKgk0wIBAg==
|
||||||
|
-----END DH PARAMETERS-----
|
21
tests/ssl/server.crt
Normal file
21
tests/ssl/server.crt
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIIDazCCAlOgAwIBAgIUNBtR47HxG8+YmIuj424xy6xxJ5EwDQYJKoZIhvcNAQEL
|
||||||
|
BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
|
||||||
|
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjAzMDQxNDM1NTNaFw0yMzAz
|
||||||
|
MDQxNDM1NTNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
|
||||||
|
HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
|
||||||
|
AQUAA4IBDwAwggEKAoIBAQDWBLG1Uto4ncPUq525H4op3LNu4MuztBPeMqAwkOlM
|
||||||
|
8v9XCoP54/Fkg0UebOmzHmJHNGx5dUBdnU92N/b/yMH5UnSxJj0Yd0Zht2iCjVob
|
||||||
|
Idv4p6zxrUtDXydDtp+0GOVZ2pRi6raljcfE/V7ny3P6XCGxWxeyNTahu42Kt7pJ
|
||||||
|
N9LmIAvXCwR3GUOQJH2v8L4YxHdXxjWcGR1Fcv7oq691+4sbJS2J5BMkhIa2/5IR
|
||||||
|
jeZXHUrXbMDmr6AVUlLoniTOB+SdWxSSdHxC/FsfpTPTahn6gFdwYJPlwtxSkeDv
|
||||||
|
6mRoGSYNyhTy8Oquax7rhlycfiUVeSEVKBeczclcggnJAgMBAAGjUzBRMB0GA1Ud
|
||||||
|
DgQWBBQndv4aJrTkFWRaszsbFvnMcXotHzAfBgNVHSMEGDAWgBQndv4aJrTkFWRa
|
||||||
|
szsbFvnMcXotHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQDD
|
||||||
|
RfFcGK1HiqMN6vSEJ0xbjsQWc7SjufVoBVP2mAtCVcCvBDbHIFNEI2KN9XGSvVX7
|
||||||
|
U19oOBimmq/Qjy7HCiGtAw1jgaYrvHYxowFCx8Wpof/uqIjE79Wp7lAhuIE9XHF9
|
||||||
|
B/j38CrkzK0XcaAQZfVGS/WsHvCxu6O0L+9CpvFSyTz2er/QYRy15azU5/J0bsda
|
||||||
|
izyqySFpOl7ohI5t/hF3pFN372psllduajVYWgiNNvJoUMPP+kZNpOY+UXoJaEIe
|
||||||
|
8Gpo11hKszpCwrDr5O+a9METdB3FmJvtkIFjLNugtDkjd0L6zlBWzqdNFeJ7fvec
|
||||||
|
QeDpZmTkkFVOjWCPyqQD
|
||||||
|
-----END CERTIFICATE-----
|
28
tests/ssl/server.key
Normal file
28
tests/ssl/server.key
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
-----BEGIN PRIVATE KEY-----
|
||||||
|
MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDWBLG1Uto4ncPU
|
||||||
|
q525H4op3LNu4MuztBPeMqAwkOlM8v9XCoP54/Fkg0UebOmzHmJHNGx5dUBdnU92
|
||||||
|
N/b/yMH5UnSxJj0Yd0Zht2iCjVobIdv4p6zxrUtDXydDtp+0GOVZ2pRi6raljcfE
|
||||||
|
/V7ny3P6XCGxWxeyNTahu42Kt7pJN9LmIAvXCwR3GUOQJH2v8L4YxHdXxjWcGR1F
|
||||||
|
cv7oq691+4sbJS2J5BMkhIa2/5IRjeZXHUrXbMDmr6AVUlLoniTOB+SdWxSSdHxC
|
||||||
|
/FsfpTPTahn6gFdwYJPlwtxSkeDv6mRoGSYNyhTy8Oquax7rhlycfiUVeSEVKBec
|
||||||
|
zclcggnJAgMBAAECggEBAM6dfYrWUAK0nKimfgCI6HP9s+TpdP8qbLvpGCmK3REC
|
||||||
|
z2wSpNMNMrCc4o+7Cet4+9xOSiMABYHbKymwYe8Su+GdrzaO+hCypeoUjPrsx/7F
|
||||||
|
s33dMuOnL6/9HwUKPCg8mL8kfHj6rBYsSJ5vFb6l9nPPml+E192d7f45+S3griGb
|
||||||
|
1StC5Dwc78E6ZaKRyHyygI0WqZiL8eUpS1w6HOPp4cJK5QgnY544mKkcwd4kUi8B
|
||||||
|
yeOBNcmGk9QckwaCngu6Ip3U5ivgKbPBZqTZcVGaVS+ggOkk3R5RAjqyVNl09ENs
|
||||||
|
ztE5eV2W+jmz481hOUSe/XxYp2RTkl1eA9m3TpkG4fkCgYEA7TTg8XtSpQvvRrxM
|
||||||
|
8wd+CvKoFrx0wn/2wU/l1+fvxe7o8RBq9cjVypnu4BaC0V8WV1/xb6EdjhlQEyfS
|
||||||
|
+pcDx0GyNErPJfoDLynfopm/jy1hnQW6tgp8xJBdyYgUlDCmimzeA8BrN0ULZR6D
|
||||||
|
r9vzQB6HvTuOy1DJb4Un7PJkEh8CgYEA5vmAnsr37A/qIsbIiMMwaSNpRIWC4jyt
|
||||||
|
OJTiAwY95GFV6o5a6FZ8xahZtQ1FeQ8G4r1pf4wzGLOnfpupU1tWWSVnEBufb6jy
|
||||||
|
SGMqHOVOhCZpI/E2WNqUk3GR6Ba4QNumBKsLDiPW6GxQ6quZjxlRFfTF7P1rpbDh
|
||||||
|
x4QQWrvodxcCgYEAy9ttEru8vBF0syLzMs4WmcwPf3K5Gcslwt8qlhJDs6TuVvaY
|
||||||
|
JeFTM0p1y+osxUlmBvNyqFAb+VpxwfSw0iHk4mLohx5fxrCF+guPoctmoOMMiAk7
|
||||||
|
fGWo8rlrkN69aNoi1sZXS3wb6KUS9PVzkTiDZnCWkZ/UyZEFfS0/sdhi/lsCgYBn
|
||||||
|
NAaPbTt3w+inH1ENIsHnyIXJsyo3Mktn48ZU+Z4ABKniAzeFZtebbcyfhE2NePRn
|
||||||
|
raCM+DUAjY2CmcT/1OjxLjAt11nXB5MyWvS/Mopxq8QA5k+VRh1rACzkmfo8KKi2
|
||||||
|
n0JyT/s/oN5K7N/RO8uqVtN1QAqwXyeTAWRZVmrZgwKBgQCeTRdbXRs1S/TKWedS
|
||||||
|
9hIApeDSxwcuvH3vJJKPbuWme0YDQ82PBZRoMJbjsgp3y+WUlHjL+AoxlIeE4tLy
|
||||||
|
KN4iFbdnhzKN7kF3oKHq3mfXRogHPxKarEu7DxAqUGk57bELulvXh1AD/OQP7rjx
|
||||||
|
MjN23ZKh4Owlem34x5yEyri0bw==
|
||||||
|
-----END PRIVATE KEY-----
|
@@ -1,5 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
import io
|
import io
|
||||||
|
import re
|
||||||
import logging
|
import logging
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
@@ -7,7 +8,11 @@ import yaml
|
|||||||
import dpkt
|
import dpkt
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from signature import BrowserSignature, TLSClientHelloSignature
|
from signature import (
|
||||||
|
BrowserSignature,
|
||||||
|
TLSClientHelloSignature,
|
||||||
|
HTTP2Signature
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -93,14 +98,11 @@ class TestSignatureModule:
|
|||||||
"""
|
"""
|
||||||
Test the TLS Client Hello parsing code.
|
Test the TLS Client Hello parsing code.
|
||||||
"""
|
"""
|
||||||
sig = BrowserSignature(
|
sig = TLSClientHelloSignature.from_bytes(self.CLIENT_HELLO)
|
||||||
tls_client_hello=TLSClientHelloSignature.from_bytes(
|
sig2 = TLSClientHelloSignature.from_dict(
|
||||||
self.CLIENT_HELLO
|
browser_signatures["chrome_98.0.4758.102_win10"] \
|
||||||
)
|
["signature"] \
|
||||||
)
|
["tls_client_hello"]
|
||||||
|
|
||||||
sig2 = BrowserSignature.from_dict(
|
|
||||||
browser_signatures["chrome_98.0.4758.102_win10"]["signature"]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
equals, reason = sig.equals(sig2, reason=True)
|
equals, reason = sig.equals(sig2, reason=True)
|
||||||
@@ -199,7 +201,22 @@ class TestImpersonation:
|
|||||||
p.terminate()
|
p.terminate()
|
||||||
p.wait(timeout=10)
|
p.wait(timeout=10)
|
||||||
|
|
||||||
def _run_curl(self, curl_binary, env_vars, url):
|
@pytest.fixture
|
||||||
|
def nghttpd(self):
|
||||||
|
"""Initiailize an HTTP/2 server"""
|
||||||
|
logging.debug(f"Running nghttpd on :8443")
|
||||||
|
|
||||||
|
p = subprocess.Popen([
|
||||||
|
"nghttpd", "-v",
|
||||||
|
"8443", "ssl/server.key", "ssl/server.crt"
|
||||||
|
], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
|
||||||
|
yield p
|
||||||
|
|
||||||
|
p.terminate()
|
||||||
|
p.wait(timeout=10)
|
||||||
|
|
||||||
|
def _run_curl(self, curl_binary, env_vars, extra_args, url):
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
if env_vars:
|
if env_vars:
|
||||||
env |= env_vars
|
env |= env_vars
|
||||||
@@ -209,13 +226,16 @@ class TestImpersonation:
|
|||||||
logging.debug("Environment variables: {}".format(
|
logging.debug("Environment variables: {}".format(
|
||||||
" ".join([f"{k}={v}" for k, v in env_vars.items()])))
|
" ".join([f"{k}={v}" for k, v in env_vars.items()])))
|
||||||
|
|
||||||
curl = subprocess.Popen([
|
args = [
|
||||||
curl_binary,
|
curl_binary,
|
||||||
"-o", "/dev/null",
|
"-o", "/dev/null",
|
||||||
"--local-port", f"{self.LOCAL_PORTS[0]}-{self.LOCAL_PORTS[1]}",
|
"--local-port", f"{self.LOCAL_PORTS[0]}-{self.LOCAL_PORTS[1]}"
|
||||||
url
|
]
|
||||||
], env=env)
|
if extra_args:
|
||||||
|
args += extra_args
|
||||||
|
args.append(url)
|
||||||
|
|
||||||
|
curl = subprocess.Popen(args, env=env)
|
||||||
return curl.wait(timeout=10)
|
return curl.wait(timeout=10)
|
||||||
|
|
||||||
def _extract_client_hello(self, pcap: bytes) -> bytes:
|
def _extract_client_hello(self, pcap: bytes) -> bytes:
|
||||||
@@ -249,6 +269,41 @@ class TestImpersonation:
|
|||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def _parse_nghttpd2_output(self, output):
|
||||||
|
"""Parse the output of nghttpd2.
|
||||||
|
|
||||||
|
nghttpd2 in verbose mode writes out the HTTP/2
|
||||||
|
headers that the client had sent.
|
||||||
|
"""
|
||||||
|
lines = output.decode("utf-8").splitlines()
|
||||||
|
stream_id = None
|
||||||
|
for line in lines:
|
||||||
|
m = re.search(r"recv HEADERS frame.*stream_id=(\d+)", line)
|
||||||
|
if m:
|
||||||
|
stream_id = m.group(1)
|
||||||
|
break
|
||||||
|
|
||||||
|
assert stream_id is not None, \
|
||||||
|
"Failed to find HEADERS frame in nghttpd2 output"
|
||||||
|
|
||||||
|
pseudo_headers = []
|
||||||
|
headers = []
|
||||||
|
for line in lines:
|
||||||
|
m = re.search(rf"recv \(stream_id={stream_id}\) (.*)", line)
|
||||||
|
if m:
|
||||||
|
header = m.group(1)
|
||||||
|
# If the headers starts with ":" it is a pseudo-header,
|
||||||
|
# i.e. ":authority". In this case keep only the header name and
|
||||||
|
# discard the value
|
||||||
|
if header.startswith(":"):
|
||||||
|
m = re.match(r"(:\w+):", header)
|
||||||
|
if m:
|
||||||
|
pseudo_headers.append(m.group(1))
|
||||||
|
else:
|
||||||
|
headers.append(header)
|
||||||
|
|
||||||
|
return pseudo_headers, headers
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"curl_binary, env_vars, expected_signature",
|
"curl_binary, env_vars, expected_signature",
|
||||||
CURL_BINARIES_AND_SIGNATURES
|
CURL_BINARIES_AND_SIGNATURES
|
||||||
@@ -267,7 +322,10 @@ class TestImpersonation:
|
|||||||
extracts the Client Hello packet from the capture and compares its
|
extracts the Client Hello packet from the capture and compares its
|
||||||
signature with the expected one defined in the YAML database.
|
signature with the expected one defined in the YAML database.
|
||||||
"""
|
"""
|
||||||
ret = self._run_curl(curl_binary, env_vars, self.TEST_URL)
|
ret = self._run_curl(curl_binary,
|
||||||
|
env_vars=env_vars,
|
||||||
|
extra_args=None,
|
||||||
|
url=self.TEST_URL)
|
||||||
assert ret == 0
|
assert ret == 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -300,3 +358,48 @@ class TestImpersonation:
|
|||||||
|
|
||||||
equals, msg = sig.equals(expected_sig, reason=True)
|
equals, msg = sig.equals(expected_sig, reason=True)
|
||||||
assert equals, msg
|
assert equals, msg
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"curl_binary, env_vars, expected_signature",
|
||||||
|
CURL_BINARIES_AND_SIGNATURES
|
||||||
|
)
|
||||||
|
def test_http2_headers(self,
|
||||||
|
nghttpd,
|
||||||
|
curl_binary,
|
||||||
|
env_vars,
|
||||||
|
browser_signatures,
|
||||||
|
expected_signature):
|
||||||
|
ret = self._run_curl(curl_binary,
|
||||||
|
env_vars=env_vars,
|
||||||
|
extra_args=["-k"],
|
||||||
|
url="https://localhost:8443")
|
||||||
|
assert ret == 0
|
||||||
|
try:
|
||||||
|
output, stderr = nghttpd.communicate(timeout=2)
|
||||||
|
|
||||||
|
# If nghttpd finished running before timeout, it's likely it failed
|
||||||
|
# with an error.
|
||||||
|
assert nghttpd.returncode == 0, \
|
||||||
|
(f"nghttpd failed with error code {nghttpd.returncode}, "
|
||||||
|
f"stderr: {stderr}")
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
nghttpd.kill()
|
||||||
|
output, stderr = nghttpd.communicate(timeout=3)
|
||||||
|
|
||||||
|
assert len(output) > 0
|
||||||
|
pseudo_headers, headers = self._parse_nghttpd2_output(output)
|
||||||
|
|
||||||
|
logging.debug(
|
||||||
|
f"Received {len(pseudo_headers)} HTTP/2 pseudo-headers "
|
||||||
|
f"and {len(headers)} HTTP/2 headers"
|
||||||
|
)
|
||||||
|
|
||||||
|
sig = HTTP2Signature(pseudo_headers, headers)
|
||||||
|
expected_sig = HTTP2Signature.from_dict(
|
||||||
|
browser_signatures[expected_signature] \
|
||||||
|
["signature"] \
|
||||||
|
["http2"]
|
||||||
|
)
|
||||||
|
|
||||||
|
equals, msg = sig.equals(expected_sig, reason=True)
|
||||||
|
assert equals, msg
|
||||||
|
Reference in New Issue
Block a user