hoelee/curl-impersonate
1
0
Fork 0
mirror of https://github.com/lwthiker/curl-impersonate.git synced 2026-09-04 16:10:29 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into docs-deps-notes

Browse Source
This commit is contained in:
lwthiker
2022-08-03 10:02:03 +03:00
committed by GitHub
parent 14796a1005 10aa2059ed
commit 45b2248641
14 changed files with 572 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
.github/workflows/publish-docker-image.yml vendored
View File

@@ -21,8 +21,8 @@ jobs:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Extract metadata (tags, labels) for Docker (chrome)
id: meta_chrome
- name: Extract metadata (tags, labels) for Docker (chrome, alpine)
id: meta_chrome_alpine
uses: docker/metadata-action@v3
with:
images: lwthiker/curl-impersonate
@@ -38,11 +38,11 @@ jobs:
push: true
context: chrome/
file: chrome/Dockerfile.alpine
tags: ${{ steps.meta_chrome.outputs.tags }}
labels: ${{ steps.meta_chrome.outputs.labels }}
tags: ${{ steps.meta_chrome_alpine.outputs.tags }}
labels: ${{ steps.meta_chrome_alpine.outputs.labels }}
- name: Extract metadata (tags, labels) for Docker (firefox)
id: meta_firefox
- name: Extract metadata (tags, labels) for Docker (firefox, alpine)
id: meta_ff_alpine
uses: docker/metadata-action@v3
with:
images: lwthiker/curl-impersonate
@@ -58,5 +58,41 @@ jobs:
push: true
context: firefox/
file: firefox/Dockerfile.alpine
tags: ${{ steps.meta_firefox.outputs.tags }}
labels: ${{ steps.meta_firefox.outputs.labels }}
tags: ${{ steps.meta_ff_alpine.outputs.tags }}
labels: ${{ steps.meta_ff_alpine.outputs.labels }}
- name: Extract metadata (tags, labels) for Docker (chrome, debian)
id: meta_chrome_debian
uses: docker/metadata-action@v3
with:
images: lwthiker/curl-impersonate
tags: |
type=semver,pattern={{version}},suffix=-chrome-slim-buster
type=semver,pattern={{major}}.{{minor}},suffix=-chrome-slim-buster
- name: Build and push the Chrome version of curl-impersonate
uses: docker/build-push-action@v2
with:
push: true
context: chrome/
file: chrome/Dockerfile
tags: ${{ steps.meta_chrome_debian.outputs.tags }}
labels: ${{ steps.meta_chrome_debian.outputs.labels }}
- name: Extract metadata (tags, labels) for Docker (firefox, debian)
id: meta_ff_debian
uses: docker/metadata-action@v3
with:
images: lwthiker/curl-impersonate
tags: |
type=semver,pattern={{version}},suffix=-ff-slim-buster
type=semver,pattern={{major}}.{{minor}},suffix=-ff-slim-buster
- name: Build and push the Firefox version of curl-impersonate
uses: docker/build-push-action@v2
with:
push: true
context: firefox/
file: firefox/Dockerfile
tags: ${{ steps.meta_ff_debian.outputs.tags }}
labels: ${{ steps.meta_ff_debian.outputs.labels }}

25
Dockerfile.template
View File

@@ -8,7 +8,7 @@
{{#debian}}
# Python is needed for building libnss.
# Use it as a common base.
FROM python:3.10.1-slim-buster
FROM python:3.10.1-slim-buster as builder
{{/debian}}
{{#alpine}}
FROM alpine:3.15.0 as builder
@@ -225,10 +225,10 @@ COPY curl_chrome* curl_edge* curl_safari* out/
RUN sed -i 's@/usr/bin/env bash@/usr/bin/env ash@' out/curl_*
{{/alpine}}
RUN chmod +x out/curl_*
{{#alpine}}
# When using alpine, create a final, minimal image with the compiled binaries
# Create a final, minimal image with the compiled binaries
# only.
{{#alpine}}
FROM alpine:3.15.0
{{#firefox}}
# curl tries to load the CA certificates for libnss.
@@ -236,9 +236,24 @@ FROM alpine:3.15.0
# which is supplied by 'nss' on alpine.
RUN apk add --no-cache nss
{{/firefox}}
{{/alpine}}
{{#debian}}
FROM debian:buster-slim
RUN apt-get update && apt-get install -y ca-certificates
{{#firefox}}
# curl tries to load the CA certificates for libnss.
# It loads them from /usr/lib/libnssckbi.so and /usr/lib/libnsspem.so,
# which are supplied by 'libnss3' and 'nss-plugin-pem' on debian.
RUN apt-get install -y libnss3 nss-plugin-pem
{{/firefox}}
{{/debian}}
# Copy curl-impersonate from the builder image
COPY --from=builder /build/install /usr/local
{{#debian}}
# Update the loader's cache
RUN ldconfig
# Copy to /build/out as well for backward compatibility with previous versions.
COPY --from=builder /build/out /build/out
{{/debian}}
# Wrapper scripts
COPY --from=builder /build/out/curl_* /usr/local/bin/
{{/alpine}}

6
INSTALL.md
View File

@@ -164,9 +164,9 @@ The Docker build is a bit more reproducible and serves as the reference implemen
```
docker build -t curl-impersonate-chrome chrome/
```
The resulting image contains a `/build/out` directory with the following:
The resulting binaries and libraries are in the `/usr/local` directory, which contains:
* `curl-impersonate-chrome`, `curl-impersonate` - The curl binary that can impersonate Chrome/Edge/Safari. It is compiled statically against libcurl, BoringSSL, and libnghttp2 so that it won't conflict with any existing libraries on your system. You can use it from the container or copy it out. Tested to work on Ubuntu 20.04.
* `curl_chrome98`, `curl_chrome99`, `...` - Wrapper scripts that launch `curl-impersonate` with all the needed flags.
* `curl_chrome99`, `curl_chrome100`, `...` - Wrapper scripts that launch `curl-impersonate` with all the needed flags.
* `libcurl-impersonate-chrome.so`, `libcurl-impersonate.so` - libcurl compiled with impersonation support. See [libcurl-impersonate](README.md#libcurl-impersonate) for more details.
You can use them inside the docker, copy them out using `docker cp` or use them in a multi-stage docker build.
@@ -176,7 +176,7 @@ Build with:
```
docker build -t curl-impersonate-ff firefox/
```
The resulting image contains a `/build/out` directory with the following:
The resulting binaries and libraries are in the `/usr/local` directory, which contains:
* `curl-impersonate-ff`, `curl-impersonate` - The curl binary that can impersonate Firefox. It is compiled statically against libcurl, nss, and libnghttp2 so that it won't conflict with any existing libraries on your system. You can use it from the container or copy it out. Tested to work on Ubuntu 20.04.
* `curl_ff91esr`, `curl_ff95`, `...` - Wrapper scripts that launch `curl-impersonate` with all the needed flags.
* `libcurl-impersonate-ff.so`, `libcurl-impersonate.so` - libcurl compiled with impersonation support. See [libcurl-impersonate](README.md#libcurl-impersonate) for more details.

26
README.md
View File

@@ -83,13 +83,13 @@ Also make sure to read [Notes on Dependencies](#notes-on-dependencies).
See [INSTALL.md](INSTALL.md).
### Docker images
Docker images based on Alpine Linux with `curl-impersonate` compiled and ready to use are available on [Docker Hub](https://hub.docker.com/r/lwthiker/curl-impersonate). The images contain the binary and all the wrapper scripts. Use like the following:
Docker images based on Alpine Linux and Debian with `curl-impersonate` compiled and ready to use are available on [Docker Hub](https://hub.docker.com/r/lwthiker/curl-impersonate). The images contain the binary and all the wrapper scripts. Use like the following:
```bash
# Firefox version
# Firefox version, Alpine Linux
docker pull lwthiker/curl-impersonate:0.5-ff
docker run --rm lwthiker/curl-impersonate:0.5-ff curl_ff100 https://www.wikipedia.org
# Chrome version
# Chrome version, Alpine Linux
docker pull lwthiker/curl-impersonate:0.5-chrome
docker run --rm lwthiker/curl-impersonate:0.5-chrome curl_chrome101 https://www.wikipedia.org
```
@@ -104,18 +104,23 @@ AUR packages are available to Archlinux users:
`libcurl-impersonate.so` is libcurl compiled with the same changes as the command line `curl-impersonate`.
It has an additional API function:
```c
CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target);
CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target,
int default_headers);
```
You can call it with the target names, e.g. `chrome101`, and it will internally set all the options and headers that are otherwise set by the wrapper scripts. Specifically it sets:
You can call it with the target names, e.g. `chrome101`, and it will internally set all the options and headers that are otherwise set by the wrapper scripts.
If `default_headers` is set to 0, the built-in list of HTTP headers will not be set, and the user is expected to provide them instead using the regular [`CURLOPT_HTTPHEADER`](https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html) libcurl option.
Calling the above function sets the following libcurl options:
* `CURLOPT_HTTP_VERSION`
* `CURLOPT_SSLVERSION`, `CURLOPT_SSL_CIPHER_LIST`, `CURLOPT_SSL_EC_CURVES`, `CURLOPT_SSL_ENABLE_NPN`, `CURLOPT_SSL_ENABLE_ALPN`
* `CURLOPT_HTTPBASEHEADER`, `CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER` (non-standard HTTP options created for this project).
* `CURLOPT_HTTPBASEHEADER`, if `default_headers` is non-zero (this is a non-standard HTTP option created for this project).
* `CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER` (non-standard HTTP/2 option created for this project).
* `CURLOPT_SSL_ENABLE_ALPS`, `CURLOPT_SSL_SIG_HASH_ALGS`, `CURLOPT_SSL_CERT_COMPRESSION`, `CURLOPT_SSL_ENABLE_TICKET` (non-standard TLS options created for this project).
Note that if you call `curl_easy_setopt()` later with one of the above it will override the options set by `curl_easy_impersonate()`.
### Using CURL_IMPERSONATE env var
*Experimental*: If your application uses `libcurl` already, you can replace the existing library at runtime with `LD_PRELOAD` (Linux only). You can then set the `CURL_IMPERSONATE` env var. For example:
If your application uses `libcurl` already, you can replace the existing library at runtime with `LD_PRELOAD` (Linux only). You can then set the `CURL_IMPERSONATE` env var. For example:
```bash
LD_PRELOAD=/path/to/libcurl-impersonate.so CURL_IMPERSONATE=chrome101 my_app
```
@@ -125,7 +130,12 @@ The `CURL_IMPERSONATE` env var has two effects:
This means that all the options needed for impersonation will be automatically set for any curl handle.
Note that the above will NOT WORK for `curl` itself because the curl tool overrides the TLS settings. Use the wrapper scripts instead.
If you need precise control over the HTTP headers, set `CURL_IMPERSONATE_HEADERS=no` to disable the built-in list of HTTP headers, then set them yourself with `curl_easy_setopt()`. For example:
```bash
LD_PRELOAD=/path/to/libcurl-impersonate.so CURL_IMPERSONATE=chrome101 CURL_IMPERSONATE_HEADERS=no my_app
```
Note that the `LD_PRELOAD` method will NOT WORK for `curl` itself because the curl tool overrides the TLS settings. Use the wrapper scripts instead.
### Notes on dependencies

15
chrome/Dockerfile
View File

@@ -7,7 +7,7 @@
# Python is needed for building libnss.
# Use it as a common base.
FROM python:3.10.1-slim-buster
FROM python:3.10.1-slim-buster as builder
WORKDIR /build
@@ -134,3 +134,16 @@ RUN ! (ldd ./out/curl-impersonate | grep -q -e nghttp2 -e brotli -e ssl -e crypt
# Wrapper scripts
COPY curl_chrome* curl_edge* curl_safari* out/
RUN chmod +x out/curl_*
# Create a final, minimal image with the compiled binaries
# only.
FROM debian:buster-slim
RUN apt-get update && apt-get install -y ca-certificates
# Copy curl-impersonate from the builder image
COPY --from=builder /build/install /usr/local
# Update the loader's cache
RUN ldconfig
# Copy to /build/out as well for backward compatibility with previous versions.
COPY --from=builder /build/out /build/out
# Wrapper scripts
COPY --from=builder /build/out/curl_* /usr/local/bin/

3
chrome/Dockerfile.alpine
View File

@@ -134,10 +134,9 @@ COPY curl_chrome* curl_edge* curl_safari* out/
RUN sed -i 's@/usr/bin/env bash@/usr/bin/env ash@' out/curl_*
RUN chmod +x out/curl_*
# When using alpine, create a final, minimal image with the compiled binaries
# Create a final, minimal image with the compiled binaries
# only.
FROM alpine:3.15.0
# Copy curl-impersonate from the builder image
COPY --from=builder /build/install /usr/local
# Wrapper scripts

112
chrome/patches/curl-impersonate.patch
View File

@@ -125,10 +125,10 @@ index b00648e79..8f8f19799 100644
} CURLoption;
diff --git a/include/curl/easy.h b/include/curl/easy.h
index 9c7e63ada..d93353c69 100644
index 9c7e63ada..a3c54c4af 100644
--- a/include/curl/easy.h
+++ b/include/curl/easy.h
@@ -43,6 +43,15 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...);
@@ -43,6 +43,16 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...);
CURL_EXTERN CURLcode curl_easy_perform(CURL *curl);
CURL_EXTERN void curl_easy_cleanup(CURL *curl);
@@ -139,7 +139,8 @@ index 9c7e63ada..d93353c69 100644
+ * created as a separate API function and not just as another option to
+ * curl_easy_setopt().
+ */
+CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target);
+CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target,
+ int default_headers);
+
/*
* NAME curl_easy_getinfo()
@@ -237,7 +238,7 @@ index 9bd8e324b..bfd5e90e2 100644
inet_pton.c \
krb5.c \
diff --git a/lib/easy.c b/lib/easy.c
index 704a59df6..c3ee9ac97 100644
index 704a59df6..9cdbdf808 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -81,6 +81,8 @@
@@ -249,7 +250,7 @@ index 704a59df6..c3ee9ac97 100644
/* The last 3 #include files should be in this order */
#include "curl_printf.h"
@@ -332,6 +334,119 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name,
@@ -332,6 +334,122 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name,
return rc;
}
@@ -258,7 +259,8 @@ index 704a59df6..c3ee9ac97 100644
+ * Call curl_easy_setopt() with all the needed options as defined in the
+ * 'impersonations' array.
+ * */
+CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target)
+CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target,
+ int default_headers)
+{
+ int i;
+ int ret;
@@ -333,21 +335,23 @@ index 704a59df6..c3ee9ac97 100644
+ return ret;
+ }
+
+ /* Build a linked list out of the static array of headers. */
+ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) {
+ if(opts->http_headers[i]) {
+ headers = curl_slist_append(headers, opts->http_headers[i]);
+ if(!headers) {
+ return CURLE_OUT_OF_MEMORY;
+ if(default_headers) {
+ /* Build a linked list out of the static array of headers. */
+ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) {
+ if(opts->http_headers[i]) {
+ headers = curl_slist_append(headers, opts->http_headers[i]);
+ if(!headers) {
+ return CURLE_OUT_OF_MEMORY;
+ }
+ }
+ }
+ }
+
+ if(headers) {
+ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers);
+ curl_slist_free_all(headers);
+ if(ret)
+ return ret;
+ if(headers) {
+ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers);
+ curl_slist_free_all(headers);
+ if(ret)
+ return ret;
+ }
+ }
+
+ if(opts->http2_pseudo_headers_order) {
@@ -369,15 +373,16 @@ index 704a59df6..c3ee9ac97 100644
/*
* curl_easy_init() is the external interface to alloc, setup and init an
* easy handle that is returned. If anything goes wrong, NULL is returned.
@@ -340,6 +455,7 @@ struct Curl_easy *curl_easy_init(void)
@@ -340,6 +458,8 @@ struct Curl_easy *curl_easy_init(void)
{
CURLcode result;
struct Curl_easy *data;
+ char *target;
+ char *env_target;
+ char *env_headers;
/* Make sure we inited the global SSL stuff */
global_init_lock();
@@ -362,6 +478,22 @@ struct Curl_easy *curl_easy_init(void)
@@ -362,6 +482,29 @@ struct Curl_easy *curl_easy_init(void)
return NULL;
}
@@ -387,10 +392,17 @@ index 704a59df6..c3ee9ac97 100644
+ * This is a bit hacky but allows seamless integration of libcurl-impersonate
+ * without code modifications to the app.
+ */
+ target = curl_getenv("CURL_IMPERSONATE");
+ if(target) {
+ result = curl_easy_impersonate(data, target);
+ free(target);
+ env_target = curl_getenv("CURL_IMPERSONATE");
+ if(env_target) {
+ env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS");
+ if(env_headers) {
+ result = curl_easy_impersonate(data, env_target,
+ !Curl_strcasecompare(env_headers, "no"));
+ free(env_headers);
+ } else {
+ result = curl_easy_impersonate(data, env_target, true);
+ }
+ free(env_target);
+ if(result) {
+ Curl_close(&data);
+ return NULL;
@@ -400,7 +412,7 @@ index 704a59df6..c3ee9ac97 100644
return data;
}
@@ -936,6 +1068,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
@@ -936,6 +1079,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
outcurl->state.referer_alloc = TRUE;
}
@@ -414,24 +426,36 @@ index 704a59df6..c3ee9ac97 100644
/* Reinitialize an SSL engine for the new handle
* note: the engine name has already been copied by dupset */
if(outcurl->set.str[STRING_SSL_ENGINE]) {
@@ -1025,6 +1164,8 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
@@ -1025,6 +1175,9 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
*/
void curl_easy_reset(struct Curl_easy *data)
{
+ char *target;
+ char *env_target;
+ char *env_headers;
+
Curl_free_request_state(data);
/* zero out UserDefined data: */
@@ -1049,6 +1190,12 @@ void curl_easy_reset(struct Curl_easy *data)
@@ -1049,6 +1202,23 @@ void curl_easy_reset(struct Curl_easy *data)
#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH)
Curl_http_auth_cleanup_digest(data);
#endif
+
+ target = curl_getenv("CURL_IMPERSONATE");
+ if(target) {
+ curl_easy_impersonate(data, target);
+ free(target);
+ /*
+ * curl-impersonate: Hook into curl_easy_reset() to set the required options
+ * from an environment variable, just like in curl_easy_init().
+ */
+ env_target = curl_getenv("CURL_IMPERSONATE");
+ if(env_target) {
+ env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS");
+ if(env_headers) {
+ curl_easy_impersonate(data, env_target,
+ !Curl_strcasecompare(env_headers, "no"));
+ free(env_headers);
+ } else {
+ curl_easy_impersonate(data, env_target, true);
+ }
+ free(env_target);
+ }
}
@@ -654,7 +678,7 @@ index 9453cf55b..01f8918ea 100644
}
nva[i] = authority;
diff --git a/lib/http.c b/lib/http.c
index 258722a60..b8990121d 100644
index 258722a60..9a06e281a 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -85,6 +85,7 @@
@@ -709,7 +733,7 @@ index 258722a60..b8990121d 100644
#endif
/* loop through one or two lists */
@@ -2069,6 +2079,92 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn,
@@ -2069,6 +2079,108 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn,
*reqp = httpreq;
}
@@ -729,6 +753,7 @@ index 258722a60..b8990121d 100644
+ struct curl_slist *head;
+ struct curl_slist *dup = NULL;
+ struct curl_slist *new_list = NULL;
+ char *uagent;
+
+ if (!data->state.base_headers)
+ return CURLE_OK;
@@ -765,6 +790,21 @@ index 258722a60..b8990121d 100644
+ }
+ }
+
+ /* If the user agent was set with CURLOPT_USERAGENT, but not with
+ * CURLOPT_HTTPHEADER, take it from there instead. */
+ if(!found &&
+ strncasecompare(head->data, "User-Agent", prefix_len) &&
+ data->set.str[STRING_USERAGENT] &&
+ *data->set.str[STRING_USERAGENT]) {
+ uagent = aprintf("User-Agent: %s", data->set.str[STRING_USERAGENT]);
+ if(!uagent) {
+ ret = CURLE_OUT_OF_MEMORY;
+ goto fail;
+ }
+ new_list = Curl_slist_append_nodup(new_list, uagent);
+ found = TRUE;
+ }
+
+ if (!found) {
+ new_list = curl_slist_append(new_list, head->data);
+ }
@@ -802,7 +842,7 @@ index 258722a60..b8990121d 100644
CURLcode Curl_http_useragent(struct Curl_easy *data)
{
/* The User-Agent string might have been allocated in url.c already, because
@@ -3088,6 +3184,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
@@ -3088,6 +3200,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
http = data->req.p.http;
DEBUGASSERT(http);

2
configure.ac
View File

@@ -1,4 +1,4 @@
AC_INIT([curl-impersonate], [0.5.0], [lwt@lwthiker.com])
AC_INIT([curl-impersonate], [0.5.2], [lwt@lwthiker.com])
AC_CANONICAL_BUILD
AC_CANONICAL_HOST

19
firefox/Dockerfile
View File

@@ -7,7 +7,7 @@
# Python is needed for building libnss.
# Use it as a common base.
FROM python:3.10.1-slim-buster
FROM python:3.10.1-slim-buster as builder
WORKDIR /build
@@ -130,3 +130,20 @@ RUN ! (ldd ./out/curl-impersonate | grep -q -e nghttp2 -e brotli -e ssl -e crypt
# Wrapper scripts
COPY curl_ff* out/
RUN chmod +x out/curl_*
# Create a final, minimal image with the compiled binaries
# only.
FROM debian:buster-slim
RUN apt-get update && apt-get install -y ca-certificates
# curl tries to load the CA certificates for libnss.
# It loads them from /usr/lib/libnssckbi.so and /usr/lib/libnsspem.so,
# which are supplied by 'libnss3' and 'nss-plugin-pem' on debian.
RUN apt-get install -y libnss3 nss-plugin-pem
# Copy curl-impersonate from the builder image
COPY --from=builder /build/install /usr/local
# Update the loader's cache
RUN ldconfig
# Copy to /build/out as well for backward compatibility with previous versions.
COPY --from=builder /build/out /build/out
# Wrapper scripts
COPY --from=builder /build/out/curl_* /usr/local/bin/

3
firefox/Dockerfile.alpine
View File

@@ -126,14 +126,13 @@ COPY curl_ff* out/
RUN sed -i 's@/usr/bin/env bash@/usr/bin/env ash@' out/curl_*
RUN chmod +x out/curl_*
# When using alpine, create a final, minimal image with the compiled binaries
# Create a final, minimal image with the compiled binaries
# only.
FROM alpine:3.15.0
# curl tries to load the CA certificates for libnss.
# It loads them from /usr/lib/libnssckbi.so,
# which is supplied by 'nss' on alpine.
RUN apk add --no-cache nss
# Copy curl-impersonate from the builder image
COPY --from=builder /build/install /usr/local
# Wrapper scripts

112
firefox/patches/curl-impersonate.patch
View File

@@ -97,10 +97,10 @@ index b00648e79..eefa36f2e 100644
} CURLoption;
diff --git a/include/curl/easy.h b/include/curl/easy.h
index 9c7e63ada..d93353c69 100644
index 9c7e63ada..a3c54c4af 100644
--- a/include/curl/easy.h
+++ b/include/curl/easy.h
@@ -43,6 +43,15 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...);
@@ -43,6 +43,16 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...);
CURL_EXTERN CURLcode curl_easy_perform(CURL *curl);
CURL_EXTERN void curl_easy_cleanup(CURL *curl);
@@ -111,7 +111,8 @@ index 9c7e63ada..d93353c69 100644
+ * created as a separate API function and not just as another option to
+ * curl_easy_setopt().
+ */
+CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target);
+CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target,
+ int default_headers);
+
/*
* NAME curl_easy_getinfo()
@@ -209,7 +210,7 @@ index 9bd8e324b..bfd5e90e2 100644
inet_pton.c \
krb5.c \
diff --git a/lib/easy.c b/lib/easy.c
index 704a59df6..03d710757 100644
index 704a59df6..349d03933 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -81,6 +81,8 @@
@@ -221,7 +222,7 @@ index 704a59df6..03d710757 100644
/* The last 3 #include files should be in this order */
#include "curl_printf.h"
@@ -332,6 +334,73 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name,
@@ -332,6 +334,76 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name,
return rc;
}
@@ -230,7 +231,8 @@ index 704a59df6..03d710757 100644
+ * Call curl_easy_setopt() with all the needed options as defined in the
+ * 'impersonations' array.
+ * */
+CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target)
+CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target,
+ int default_headers)
+{
+ int i;
+ int ret;
@@ -267,21 +269,23 @@ index 704a59df6..03d710757 100644
+ return ret;
+ }
+
+ /* Build a linked list out of the static array of headers. */
+ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) {
+ if(opts->http_headers[i]) {
+ headers = curl_slist_append(headers, opts->http_headers[i]);
+ if(!headers) {
+ return CURLE_OUT_OF_MEMORY;
+ if(default_headers) {
+ /* Build a linked list out of the static array of headers. */
+ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) {
+ if(opts->http_headers[i]) {
+ headers = curl_slist_append(headers, opts->http_headers[i]);
+ if(!headers) {
+ return CURLE_OUT_OF_MEMORY;
+ }
+ }
+ }
+ }
+
+ if(headers) {
+ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers);
+ curl_slist_free_all(headers);
+ if(ret)
+ return ret;
+ if(headers) {
+ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers);
+ curl_slist_free_all(headers);
+ if(ret)
+ return ret;
+ }
+ }
+
+ /* Always enable all supported compressions. */
@@ -295,15 +299,16 @@ index 704a59df6..03d710757 100644
/*
* curl_easy_init() is the external interface to alloc, setup and init an
* easy handle that is returned. If anything goes wrong, NULL is returned.
@@ -340,6 +409,7 @@ struct Curl_easy *curl_easy_init(void)
@@ -340,6 +412,8 @@ struct Curl_easy *curl_easy_init(void)
{
CURLcode result;
struct Curl_easy *data;
+ char *target;
+ char *env_target;
+ char *env_headers;
/* Make sure we inited the global SSL stuff */
global_init_lock();
@@ -362,6 +432,22 @@ struct Curl_easy *curl_easy_init(void)
@@ -362,6 +436,29 @@ struct Curl_easy *curl_easy_init(void)
return NULL;
}
@@ -313,10 +318,17 @@ index 704a59df6..03d710757 100644
+ * This is a bit hacky but allows seamless integration of libcurl-impersonate
+ * without code modifications to the app.
+ */
+ target = curl_getenv("CURL_IMPERSONATE");
+ if(target) {
+ result = curl_easy_impersonate(data, target);
+ free(target);
+ env_target = curl_getenv("CURL_IMPERSONATE");
+ if(env_target) {
+ env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS");
+ if(env_headers) {
+ result = curl_easy_impersonate(data, env_target,
+ !Curl_strcasecompare(env_headers, "no"));
+ free(env_headers);
+ } else {
+ result = curl_easy_impersonate(data, env_target, true);
+ }
+ free(env_target);
+ if(result) {
+ Curl_close(&data);
+ return NULL;
@@ -326,7 +338,7 @@ index 704a59df6..03d710757 100644
return data;
}
@@ -936,6 +1022,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
@@ -936,6 +1033,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
outcurl->state.referer_alloc = TRUE;
}
@@ -340,24 +352,36 @@ index 704a59df6..03d710757 100644
/* Reinitialize an SSL engine for the new handle
* note: the engine name has already been copied by dupset */
if(outcurl->set.str[STRING_SSL_ENGINE]) {
@@ -1025,6 +1118,8 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
@@ -1025,6 +1129,9 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
*/
void curl_easy_reset(struct Curl_easy *data)
{
+ char *target;
+ char *env_target;
+ char *env_headers;
+
Curl_free_request_state(data);
/* zero out UserDefined data: */
@@ -1049,6 +1144,12 @@ void curl_easy_reset(struct Curl_easy *data)
@@ -1049,6 +1156,23 @@ void curl_easy_reset(struct Curl_easy *data)
#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH)
Curl_http_auth_cleanup_digest(data);
#endif
+
+ target = curl_getenv("CURL_IMPERSONATE");
+ if(target) {
+ curl_easy_impersonate(data, target);
+ free(target);
+ /*
+ * curl-impersonate: Hook into curl_easy_reset() to set the required options
+ * from an environment variable, just like in curl_easy_init().
+ */
+ env_target = curl_getenv("CURL_IMPERSONATE");
+ if(env_target) {
+ env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS");
+ if(env_headers) {
+ curl_easy_impersonate(data, env_target,
+ !Curl_strcasecompare(env_headers, "no"));
+ free(env_headers);
+ } else {
+ curl_easy_impersonate(data, env_target, true);
+ }
+ free(env_target);
+ }
}
@@ -399,7 +423,7 @@ index 9453cf55b..ba9433cb9 100644
}
diff --git a/lib/http.c b/lib/http.c
index 258722a60..b8990121d 100644
index 258722a60..9a06e281a 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -85,6 +85,7 @@
@@ -454,7 +478,7 @@ index 258722a60..b8990121d 100644
#endif
/* loop through one or two lists */
@@ -2069,6 +2079,92 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn,
@@ -2069,6 +2079,108 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn,
*reqp = httpreq;
}
@@ -474,6 +498,7 @@ index 258722a60..b8990121d 100644
+ struct curl_slist *head;
+ struct curl_slist *dup = NULL;
+ struct curl_slist *new_list = NULL;
+ char *uagent;
+
+ if (!data->state.base_headers)
+ return CURLE_OK;
@@ -510,6 +535,21 @@ index 258722a60..b8990121d 100644
+ }
+ }
+
+ /* If the user agent was set with CURLOPT_USERAGENT, but not with
+ * CURLOPT_HTTPHEADER, take it from there instead. */
+ if(!found &&
+ strncasecompare(head->data, "User-Agent", prefix_len) &&
+ data->set.str[STRING_USERAGENT] &&
+ *data->set.str[STRING_USERAGENT]) {
+ uagent = aprintf("User-Agent: %s", data->set.str[STRING_USERAGENT]);
+ if(!uagent) {
+ ret = CURLE_OUT_OF_MEMORY;
+ goto fail;
+ }
+ new_list = Curl_slist_append_nodup(new_list, uagent);
+ found = TRUE;
+ }
+
+ if (!found) {
+ new_list = curl_slist_append(new_list, head->data);
+ }
@@ -547,7 +587,7 @@ index 258722a60..b8990121d 100644
CURLcode Curl_http_useragent(struct Curl_easy *data)
{
/* The User-Agent string might have been allocated in url.c already, because
@@ -3088,6 +3184,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
@@ -3088,6 +3200,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
http = data->req.p.http;
DEBUGASSERT(http);

18
tests/Dockerfile
View File

@@ -16,21 +16,19 @@ COPY requirements.txt requirements.txt
RUN pip install --upgrade pip && \
pip install -r requirements.txt
RUN mkdir /tests/firefox /tests/chrome
# Copy the built binaries from both containers
COPY --from=ff /build/out/curl-impersonate-ff /tests/install/bin/
COPY --from=ff /build/out/curl_* /tests/install/bin/
COPY --from=ff /build/out/libcurl-impersonate* /tests/install/lib/
COPY --from=chrome /build/out/curl-impersonate-chrome /tests/install/bin/
COPY --from=chrome /build/out/curl_* /tests/install/bin/
COPY --from=chrome /build/out/libcurl-impersonate* /tests/install/lib/
COPY --from=ff /usr/local/ /usr/local/
COPY --from=chrome /usr/local/ /usr/local/
# Needed to update the loader's cache
RUN ldconfig
COPY . .
# Compile 'minicurl' which is used for testing libcurl-impersonate.
# 'minicurl' is compiled against the "regular" libcurl.
# libcurl-impersonate will replace it at runtime via LD_PRELOAD.
RUN gcc -Wall -Werror -o /tests/install/bin/minicurl minicurl.c `curl-config --libs`
RUN gcc -Wall -Werror -o minicurl minicurl.c `curl-config --libs`
RUN install minicurl /usr/local/bin
ENTRYPOINT ["pytest", "--install-dir", "/tests/install"]
ENTRYPOINT ["pytest"]

61
tests/minicurl.c
View File

@@ -18,6 +18,7 @@
/* Support up to 16 URLs */
#define MAX_URLS 16
/* Command line options. */
struct opts {
char *outfile;
@@ -25,6 +26,8 @@ struct opts {
uint16_t local_port_end;
bool insecure;
char *urls[MAX_URLS];
char *user_agent;
struct curl_slist *headers;
};
int parse_ports_range(char *str, uint16_t *start, uint16_t *end)
@@ -63,6 +66,7 @@ int parse_opts(int argc, char **argv, struct opts *opts)
int c;
int r;
int i;
struct curl_slist *tmp;
memset(opts, 0, sizeof(*opts));
@@ -71,15 +75,21 @@ int parse_opts(int argc, char **argv, struct opts *opts)
while (1) {
int option_index = 0;
static struct option long_options[] = {
{"local-port", required_argument, NULL, 'l'}
{"header", required_argument, NULL, 'H'},
{"local-port", required_argument, NULL, 'l'},
{"user-agent", required_argument, NULL, 'A'},
{0, 0, NULL, 0}
};
c = getopt_long(argc, argv, "o:k", long_options, &option_index);
c = getopt_long(argc, argv, "o:kH:A:", long_options, &option_index);
if (c == -1) {
break;
}
switch (c) {
case 'A':
opts->user_agent = optarg;
break;
case 'l':
r = parse_ports_range(optarg,
&opts->local_port_start,
@@ -94,16 +104,29 @@ int parse_opts(int argc, char **argv, struct opts *opts)
case 'k':
opts->insecure = true;
break;
case 'H':
tmp = curl_slist_append(opts->headers, optarg);
if (!tmp) {
fprintf(stderr, "curl_slist_append() failed\n");
if (opts->headers) {
curl_slist_free_all(opts->headers);
}
return 1;
}
opts->headers = tmp;
break;
case '?':
break;
}
}
/* No URL supplied. */
i = 0;
if (optind >= argc) {
return 1;
}
/* The rest of the options are URLs */
i = 0;
while (optind < argc) {
opts->urls[i++] = argv[optind++];
}
@@ -111,6 +134,13 @@ int parse_opts(int argc, char **argv, struct opts *opts)
return 0;
}
void clean_opts(struct opts *opts)
{
if (opts->headers) {
curl_slist_free_all(opts->headers);
}
}
/* Set all options except for the URL. */
int set_opts(CURL *curl, struct opts *opts, FILE *file)
{
@@ -160,6 +190,22 @@ int set_opts(CURL *curl, struct opts *opts, FILE *file)
}
}
if (opts->user_agent) {
c = curl_easy_setopt(curl, CURLOPT_USERAGENT, opts->user_agent);
if (c) {
fprintf(stderr, "curl_easy_setopt(CURLOPT_USERAGENT) failed\n");
return 1;
}
}
if (opts->headers) {
c = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, opts->headers);
if (c) {
fprintf(stderr, "curl_easy_setopt(CURLOPT_HTTPHEADER) failed\n");
return 1;
}
}
return 0;
}
@@ -180,7 +226,8 @@ int main(int argc, char *argv[])
file = fopen(opts.outfile, "w");
if (!file) {
fprintf(stderr, "Failed opening %s for writing\n", opts.outfile);
exit(1);
c = 1;
goto out_clean_opts;
}
} else {
file = stdout;
@@ -212,7 +259,9 @@ int main(int argc, char *argv[])
c = curl_easy_perform(curl);
if (c) {
fprintf(stderr, "curl_easy_perform() failed\n");
fprintf(stderr,
"curl_easy_perform() failed: %d (%s)\n",
c, curl_easy_strerror(c));
goto out;
}
@@ -231,5 +280,7 @@ out_close:
if (file) {
fclose(file);
}
out_clean_opts:
clean_opts(&opts);
return c;
}

236
tests/test_impersonate.py
View File

@@ -8,6 +8,7 @@ import logging
import pathlib
import subprocess
import tempfile
import itertools
from typing import List
import yaml
@@ -628,3 +629,238 @@ class TestImpersonation:
"<html>" in body or
"<!doctype html>" in body
)
@pytest.mark.parametrize(
"curl_binary, env_vars, ld_preload",
[
(
"minicurl",
{
"CURL_IMPERSONATE": "chrome101",
"CURL_IMPERSONATE_HEADERS": "no"
},
"libcurl-impersonate-chrome"
),
(
"minicurl",
{
"CURL_IMPERSONATE": "ff102",
"CURL_IMPERSONATE_HEADERS": "no"
},
"libcurl-impersonate-ff",
)
]
)
async def test_no_builtin_headers(self,
pytestconfig,
nghttpd,
curl_binary,
env_vars,
ld_preload):
"""
Ensure the built-in headers of libcurl-impersonate are not added when
the CURL_IMPERSONATE_HEADERS environment variable is set to "no".
"""
curl_binary = os.path.join(
pytestconfig.getoption("install_dir"), "bin", curl_binary
)
if not sys.platform.startswith("linux"):
pytest.skip()
self._set_ld_preload(env_vars, os.path.join(
pytestconfig.getoption("install_dir"), "lib", ld_preload
))
# Use some custom headers with a specific order.
# We will test that the headers are sent in the exact given order, as
# it is important for users to be able to control the exact headers
# content and order.
headers = [
"X-Hello: World",
"Accept: application/json",
"X-Goodbye: World",
"Accept-Encoding: deflate, gzip, br"
"X-Foo: Bar",
"User-Agent: curl-impersonate"
]
header_args = list(itertools.chain(*[
["-H", header]
for header in headers
]))
ret = self._run_curl(curl_binary,
env_vars=env_vars,
extra_args=["-k"] + header_args,
urls=["https://localhost:8443"])
assert ret == 0
output = await self._read_proc_output(nghttpd, timeout=2)
assert len(output) > 0
_, output_headers = self._parse_nghttpd2_output(output)
for i, header in enumerate(output_headers):
assert header.lower() == headers[i].lower()
@pytest.mark.parametrize(
"curl_binary, env_vars, ld_preload",
[
(
"minicurl",
{
"CURL_IMPERSONATE": "chrome101"
},
"libcurl-impersonate-chrome",
),
(
"minicurl",
{
"CURL_IMPERSONATE": "chrome101",
"CURL_IMPERSONATE_HEADERS": "no"
},
"libcurl-impersonate-chrome",
),
(
"minicurl",
{
"CURL_IMPERSONATE": "ff102"
},
"libcurl-impersonate-ff",
),
(
"minicurl",
{
"CURL_IMPERSONATE": "ff102",
"CURL_IMPERSONATE_HEADERS": "no"
},
"libcurl-impersonate-ff",
)
]
)
async def test_user_agent(
self,
pytestconfig,
nghttpd,
curl_binary,
env_vars,
ld_preload
):
"""
Ensure that any user-agent set with CURLOPT_HTTPHEADER will override
the one set by libcurl-impersonate.
"""
curl_binary = os.path.join(
pytestconfig.getoption("install_dir"), "bin", curl_binary
)
if not sys.platform.startswith("linux"):
pytest.skip()
self._set_ld_preload(env_vars, os.path.join(
pytestconfig.getoption("install_dir"), "lib", ld_preload
))
user_agent = "My-User-Agent"
ret = self._run_curl(curl_binary,
env_vars=env_vars,
extra_args=[
"-k",
"-H",
f"User-Agent: {user_agent}"
],
urls=["https://localhost:8443"])
assert ret == 0
output = await self._read_proc_output(nghttpd, timeout=2)
assert len(output) > 0
_, headers = self._parse_nghttpd2_output(output)
assert any([
header.lower().startswith("user-agent:") for header in headers
])
for header in headers:
if header.lower().startswith("user-agent:"):
assert header[len("user-agent:"):].strip() == user_agent
@pytest.mark.parametrize(
"curl_binary, env_vars, ld_preload",
[
(
"minicurl",
{
"CURL_IMPERSONATE": "chrome101"
},
"libcurl-impersonate-chrome",
),
(
"minicurl",
{
"CURL_IMPERSONATE": "chrome101",
"CURL_IMPERSONATE_HEADERS": "no"
},
"libcurl-impersonate-chrome",
),
(
"minicurl",
{
"CURL_IMPERSONATE": "ff102"
},
"libcurl-impersonate-ff",
),
(
"minicurl",
{
"CURL_IMPERSONATE": "ff102",
"CURL_IMPERSONATE_HEADERS": "no"
},
"libcurl-impersonate-ff",
)
]
)
async def test_user_agent_curlopt_useragent(
self,
pytestconfig,
nghttpd,
curl_binary,
env_vars,
ld_preload
):
"""
Ensure that any user-agent set with CURLOPT_USERAGENT will override
the one set by libcurl-impersonate. See:
https://github.com/lwthiker/curl-impersonate/issues/51
"""
curl_binary = os.path.join(
pytestconfig.getoption("install_dir"), "bin", curl_binary
)
if not sys.platform.startswith("linux"):
pytest.skip()
self._set_ld_preload(env_vars, os.path.join(
pytestconfig.getoption("install_dir"), "lib", ld_preload
))
user_agent = "My-User-Agent"
ret = self._run_curl(curl_binary,
env_vars=env_vars,
extra_args=["-k", "-A", user_agent],
urls=["https://localhost:8443"])
assert ret == 0
output = await self._read_proc_output(nghttpd, timeout=2)
assert len(output) > 0
_, headers = self._parse_nghttpd2_output(output)
assert any([
header.lower().startswith("user-agent:") for header in headers
])
for header in headers:
if header.lower().startswith("user-agent:"):
assert header[len("user-agent:"):].strip() == user_agent