From 0c79eb38069b41047ddd5bb92809a56c489638a9 Mon Sep 17 00:00:00 2001 From: hoelee Date: Mon, 3 Aug 2026 10:53:59 +0800 Subject: [PATCH] Initial version --- .traefik.yml | 28 +++++ LICENSE | 201 ++++++++++++++++++++++++++++++++++++ README.md | 254 ++++++++++++++++++++++++++++++++++++++++++++++ botfilter.go | 165 ++++++++++++++++++++++++++++++ botfilter_test.go | 183 +++++++++++++++++++++++++++++++++ cache.go | 223 ++++++++++++++++++++++++++++++++++++++++ config.go | 168 ++++++++++++++++++++++++++++++ go.mod | 3 + logger.go | 17 ++++ matcher.go | 141 +++++++++++++++++++++++++ util.go | 127 +++++++++++++++++++++++ 11 files changed, 1510 insertions(+) create mode 100644 .traefik.yml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 botfilter.go create mode 100644 botfilter_test.go create mode 100644 cache.go create mode 100644 config.go create mode 100644 go.mod create mode 100644 logger.go create mode 100644 matcher.go create mode 100644 util.go diff --git a/.traefik.yml b/.traefik.yml new file mode 100644 index 0000000..3c7bc10 --- /dev/null +++ b/.traefik.yml @@ -0,0 +1,28 @@ +displayName: Bot Filter +type: middleware +import: github.com/hoelee/traefik-botfilter +summary: Rejects common scans and temporarily bans suspicious clients using bounded per-IP scoring. +testData: + statusCode: 403 + requireUserAgent: true + requireAccept: true + requireHost: true + browserValidation: true + whitelistCIDRs: + - 192.168.0.0/16 + - 10.0.0.0/8 + temporaryBanMinutes: 15 + blockedUserAgents: + - curl + - wget + - python + blockedPaths: + - /.env + - /.git + - /wp-login.php + - /xmlrpc.php + - /phpmyadmin + blockedExtensions: + - .env + - .bak + - .zip diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6c3a688 --- /dev/null +++ b/README.md @@ -0,0 +1,254 @@ +# Traefik Bot Filter + +`traefik-botfilter` is a dependency-free Traefik middleware plugin for public +sites that are receiving scanners or unsophisticated HTTP floods. It blocks +known scan paths before they reach the upstream and keeps a bounded, local +per-client score cache for temporary bans. + +It is deliberately a **request filter**, not a bot-management service. A +distributed attacker that sends valid browser-like requests from many IPs +needs an upstream CDN/WAF or firewall as well. + +## What it does + +- Immediately rejects and temporarily bans configured scanner paths, file + extensions, and User-Agent tokens. +- Optionally requires `User-Agent`, `Accept`, and `Host`. A failed required + check is immediately cached as a temporary ban, so repeated requests do not + reach Kiwix. +- Adds scores in a configurable sliding window. Defaults implement the stated + model: empty UA `+40`, missing Accept `+20`, bad path `+50`, blocked UA + `+80`, first request directly to `/content/` `+15`, and an upstream `404` + `+40`. +- Bans when the score reaches `scoreThreshold` (default `100`). A blocked scan + path always bans immediately, independently of the score threshold. +- Applies conservative browser plausibility checks. It detects internally + inconsistent browser User-Agents, but does not claim to distinguish every + automation client from a real browser. That requires a JavaScript challenge + at a CDN/WAF layer. +- Bounds memory with `maxTrackedIPs` (default `50,000`) and + `maxScoreEventsPerIP` (default `16`), with lock-safe access and no cleanup + goroutine. +- Never trusts `X-Forwarded-For` unless `clientIPHeader` is explicitly set + **and** the TCP peer is in `trustedProxyCIDRs`. + +## Important fixes before enabling it + +Your Kiwix service currently publishes `6001:8080`. Requests to that port go +straight to Kiwix and bypass this middleware, rate limiting, and Traefik +access controls. Remove this line unless you need a separate protected private +listener: + +```yaml +kiwix: + # ports: + # - "6001:8080" +``` + +Also narrow the current error middleware. `400-599` makes every ordinary +Kiwix 404 and upstream 502 call `host.docker.internal:44440`, which adds work +and obscures the original failure. Keep it only for rate-limit responses: + +```yaml +cp-ratelimit-errorpages: + errors: + status: + - "429-429" + service: srv-error + query: "/{status}.html" +``` + +The access log supplied with this request contains 57,141 requests with an +empty User-Agent (`"-"`), including thousands of `404` and `502` responses. +With `requireUserAgent: true`, those requests are rejected at Traefik before +they consume Kiwix CPU. + +## Configuration + +Add the following to the dynamic file provider configuration. The middleware +must be attached to each public Kiwix router. + +```yaml +http: + middlewares: + botfilter: + plugin: + botfilter: + statusCode: 403 + + requireUserAgent: true + requireAccept: true + requireHost: true + browserValidation: true + + whitelistCIDRs: + - 192.168.0.0/16 + - 10.0.0.0/8 + + temporaryBanMinutes: 15 + scoreThreshold: 100 + scoreWindowMinutes: 15 + maxTrackedIPs: 50000 + maxScoreEventsPerIP: 16 + + blockedUserAgents: + - curl + - wget + - python + - Go-http-client + - masscan + - sqlmap + - zgrab + - nikto + + blockedPaths: + - /.env + - /.git + - /wp-login.php + - /xmlrpc.php + - /phpmyadmin + + blockedExtensions: + - .env + - .bak + - .zip + + # This weak signal contributes only 15 points. Direct links to a + # Kiwix article remain possible; they are not banned by themselves. + randomArticlePatterns: + - /content/ + + # Leave both unset when Traefik accepts traffic directly. If a CDN + # or load balancer is in front, configure its exact source CIDRs and + # ensure it overwrites this header. + # clientIPHeader: X-Forwarded-For + # trustedProxyCIDRs: + # - 203.0.113.0/24 + + # Do not turn this on during an attack unless short diagnostics are + # needed: per-request disk logging can itself become expensive. + logBlockedRequests: false +``` + +Apply it before `cp-ratelimit` so rejected requests do not consume the +rate-limiter's work. Keep the narrowed errors middleware as the outer wrapper +for the rate limiter: + +```yaml +http: + routers: + rtr-default-wiki: + # ... existing rule/service fields ... + middlewares: + - cp-ratelimit-errorpages + - botfilter + - cp-ratelimit +``` + +Use the same middleware list on `rtr-wiki` and `rtr-yes` if they expose Kiwix. + +## Install in Traefik + +### Production: remote plugin + +The module name in `.traefik.yml` is intentionally the one requested here. +Create the public repository `github.com/hoelee/traefik-botfilter`, push this +directory, and create the immutable Git tag `v0.1.0`. Then put this in the +**static** Traefik configuration (`traefik.yml`), not `dynamic.yml`: + +```yaml +experimental: + plugins: + botfilter: + moduleName: github.com/hoelee/traefik-botfilter + version: v0.1.0 +``` + +Restart Traefik after changing static plugin configuration. Do not retag an +existing version; publish `v0.1.1` for later changes. + +### Test locally first + +Traefik local plugins need the Go module at the exact module path below +`/plugins-local/src`. On the Synology host, copy or clone this repository to: + +```text +/volume1/docker/traefik-plugins/src/github.com/hoelee/traefik-botfilter +``` + +Add this readonly mount to the Traefik service: + +```yaml +volumes: + - /volume1/docker/traefik-plugins:/plugins-local:ro +``` + +And use this static configuration instead of the remote `plugins` block: + +```yaml +experimental: + localPlugins: + botfilter: + moduleName: github.com/hoelee/traefik-botfilter +``` + +Restart Traefik and confirm its startup log says that the `botfilter` plugin +loaded before exposing the public router. + +## Option reference + +| Option | Default | Meaning | +| --- | ---: | --- | +| `statusCode` | `403` | HTTP status for rejected requests (`400`–`599`). | +| `temporaryBanMinutes` | `15` | In-memory ban duration. | +| `scoreThreshold` | `100` | Score at which a client is banned. | +| `scoreWindowMinutes` | `15` | Sliding window for score events. | +| `maxTrackedIPs` | `50000` | Hard maximum size of the per-IP state map. | +| `maxScoreEventsPerIP` | `16` | Bound on score events retained per client. | +| `requireUserAgent` | `false` | Immediately ban missing User-Agent requests. | +| `requireAccept` | `false` | Immediately ban missing Accept requests. | +| `requireHost` | `false` | Immediately ban missing Host requests. | +| `browserValidation` | `false` | Score implausible Mozilla-family header combinations. | +| `whitelistCIDRs` | none | Clients that bypass all checks and cache updates. | +| `clientIPHeader` | empty | Optional header used only from `trustedProxyCIDRs`. | +| `trustedProxyCIDRs` | none | TCP peer ranges allowed to supply the client-IP header. | +| `randomArticlePatterns` | `/content/` | First-request path prefixes that add `randomArticleScore`. | +| `logBlockedRequests` | `false` | Opt-in rejected-request logging. | + +All listed score fields are configurable: `emptyUserAgentScore`, +`missingAcceptScore`, `blockedUserAgentScore`, `badPathScore`, +`randomArticleScore`, `notFoundScore`, and `fakeBrowserScore`. Set a score to +`0` to disable that one signal; scanner-path and required-header rejections +still ban immediately. + +## Operational limits and recommended defences + +This plugin protects Kiwix from the malformed-header/scanner pattern in the +log, but it cannot stop a botnet that rotates IPs and perfectly imitates +browser headers. For that case: + +1. Put the hostname behind a CDN/WAF with bot challenge and request-rate rules. +2. Firewall the NAS so public clients cannot reach Kiwix's port `6001` or any + Traefik entry point other than the intended public port. +3. Do not expose the NAS origin address in DNS or other services; otherwise + attackers can bypass the CDN. +4. Keep Traefik access logs sampled or rotate them quickly during an incident. + Per-request synchronous disk logging becomes material at flood volume. +5. `deploy.resources` is commonly ignored by non-Swarm Docker Compose. Verify + resource limits with `docker inspect` on the NAS rather than assuming the + `deploy` block limits CPU. + +The cache is intentionally local to a Traefik process and is reset on +container restart. That is appropriate for a low-overhead edge filter; use a +CDN/WAF or shared store if bans must survive restarts or be shared by multiple +Traefik replicas. + +## Development + +```text +go test ./... +go vet ./... +``` + +The implementation only uses the Go standard library, which reduces plugin +startup and supply-chain risk. diff --git a/botfilter.go b/botfilter.go new file mode 100644 index 0000000..3a214b0 --- /dev/null +++ b/botfilter.go @@ -0,0 +1,165 @@ +// Package botfilter is a Traefik middleware plugin that blocks known scans +// and applies a bounded, in-memory per-IP suspicion score. +package botfilter + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "strconv" + "time" +) + +type botFilter struct { + next http.Handler + config *compiledConfig + cache *clientCache + logger filterLogger +} + +// New constructs a Traefik middleware. Its signature is the interface used +// by Traefik's Go plugin runtime. +func New(_ context.Context, next http.Handler, config *Config, name string) (http.Handler, error) { + if next == nil { + return nil, fmt.Errorf("botfilter: next handler is nil") + } + compiled, err := compileConfig(config) + if err != nil { + return nil, err + } + return &botFilter{ + next: next, + config: compiled, + cache: newClientCache(compiled), + logger: filterLogger{name: name, enabled: compiled.LogBlockedRequests}, + }, nil +} + +func (b *botFilter) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + now := time.Now() + addr := clientIP(r, b.config) + ip := addressString(addr) + if addressInPrefixes(addr, b.config.whitelist) { + b.next.ServeHTTP(rw, r) + return + } + if existing := b.cache.isBanned(ip, now); existing.banned { + b.reject(rw, ip, "temporary ban", existing) + return + } + + match := inspectRequest(r, b.config) + paths := requestPaths(r) + decision := b.cache.observeRequest(ip, requestObservation{ + at: now, + paths: paths, + points: match.points, + forceBan: match.forceBan, + }) + if decision.banned { + reason := match.reason + if reason == "" { + reason = "suspicion score threshold" + } + b.reject(rw, ip, reason, decision) + return + } + + recorder := &statusRecorder{ResponseWriter: rw} + b.next.ServeHTTP(recorder, r) + if result := b.cache.observeResponse(ip, recorder.statusCode(), b.config.NotFoundScore, time.Now()); result.banned { + b.logger.blocked(ip, "404 score threshold", result.score) + } +} + +func (b *botFilter) reject(rw http.ResponseWriter, ip, reason string, decision cacheResult) { + b.logger.blocked(ip, reason, decision.score) + rw.Header().Set("Cache-Control", "no-store") + rw.Header().Set("Content-Type", "text/plain; charset=utf-8") + if decision.banUntil.After(time.Now()) { + seconds := int(time.Until(decision.banUntil).Seconds()) + if seconds < 1 { + seconds = 1 + } + rw.Header().Set("Retry-After", strconv.Itoa(seconds)) + } + rw.WriteHeader(b.config.StatusCode) + _, _ = io.WriteString(rw, "request denied\n") +} + +func addressString(addr netip.Addr) string { + if !addr.IsValid() { + return "" + } + return addr.String() +} + +// statusRecorder keeps downstream response capabilities available while +// recording the final status used for 404 scoring. +type statusRecorder struct { + http.ResponseWriter + status int +} + +func (r *statusRecorder) WriteHeader(status int) { + if r.status != 0 { + return + } + r.status = status + r.ResponseWriter.WriteHeader(status) +} + +func (r *statusRecorder) Write(data []byte) (int, error) { + if r.status == 0 { + r.status = http.StatusOK + } + return r.ResponseWriter.Write(data) +} + +func (r *statusRecorder) statusCode() int { + if r.status == 0 { + return http.StatusOK + } + return r.status +} + +func (r *statusRecorder) Unwrap() http.ResponseWriter { return r.ResponseWriter } + +func (r *statusRecorder) Flush() { + if r.status == 0 { + r.status = http.StatusOK + } + if flusher, ok := r.ResponseWriter.(http.Flusher); ok { + flusher.Flush() + } +} + +func (r *statusRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hijacker, ok := r.ResponseWriter.(http.Hijacker) + if !ok { + return nil, nil, http.ErrNotSupported + } + return hijacker.Hijack() +} + +func (r *statusRecorder) Push(target string, options *http.PushOptions) error { + pusher, ok := r.ResponseWriter.(http.Pusher) + if !ok { + return http.ErrNotSupported + } + return pusher.Push(target, options) +} + +func (r *statusRecorder) ReadFrom(source io.Reader) (int64, error) { + if r.status == 0 { + r.status = http.StatusOK + } + if readerFrom, ok := r.ResponseWriter.(io.ReaderFrom); ok { + return readerFrom.ReadFrom(source) + } + return io.Copy(struct{ io.Writer }{Writer: r.ResponseWriter}, source) +} diff --git a/botfilter_test.go b/botfilter_test.go new file mode 100644 index 0000000..1e885bc --- /dev/null +++ b/botfilter_test.go @@ -0,0 +1,183 @@ +package botfilter + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +func TestRequiredHeaderCreatesTemporaryBan(t *testing.T) { + cfg := CreateConfig() + cfg.RequireUserAgent = true + cfg.RequireAccept = true + cfg.RequireHost = true + nextCalls := 0 + handler, err := New(context.Background(), http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + nextCalls++ + rw.WriteHeader(http.StatusOK) + }), cfg, "test") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + first := httptest.NewRequest(http.MethodGet, "http://wiki.example/content/article", nil) + first.RemoteAddr = "203.0.113.11:54321" + first.Header.Set("Accept", "text/html") + first.Header.Del("User-Agent") + firstResponse := httptest.NewRecorder() + handler.ServeHTTP(firstResponse, first) + if firstResponse.Code != http.StatusForbidden { + t.Fatalf("first response status = %d, want %d", firstResponse.Code, http.StatusForbidden) + } + if firstResponse.Header().Get("Retry-After") == "" { + t.Fatal("first response did not include Retry-After") + } + + second := httptest.NewRequest(http.MethodGet, "http://wiki.example/", nil) + second.RemoteAddr = "203.0.113.11:54321" + second.Header.Set("Accept", "text/html") + second.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0 AppleWebKit/537.36 Safari/537.36") + secondResponse := httptest.NewRecorder() + handler.ServeHTTP(secondResponse, second) + if secondResponse.Code != http.StatusForbidden { + t.Fatalf("second response status = %d, want cached ban", secondResponse.Code) + } + if nextCalls != 0 { + t.Fatalf("next handler calls = %d, want 0", nextCalls) + } +} + +func TestWhitelistBypassesFilter(t *testing.T) { + cfg := CreateConfig() + cfg.RequireUserAgent = true + cfg.WhitelistCIDRs = []string{"192.168.0.0/16"} + handler, err := New(context.Background(), http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + rw.WriteHeader(http.StatusNoContent) + }), cfg, "test") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + request := httptest.NewRequest(http.MethodGet, "http://wiki.example/", nil) + request.RemoteAddr = "192.168.30.25:1234" + request.Header.Del("User-Agent") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusNoContent { + t.Fatalf("response status = %d, want whitelist to reach next handler", response.Code) + } +} + +func TestEncodedScanPathBansBeforeUpstream(t *testing.T) { + cfg := CreateConfig() + cfg.RequireUserAgent = false + cfg.RequireAccept = false + nextCalls := 0 + handler, err := New(context.Background(), http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + nextCalls++ + rw.WriteHeader(http.StatusOK) + }), cfg, "test") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + request := httptest.NewRequest(http.MethodGet, "http://wiki.example/foo/..%2F.env", nil) + request.RemoteAddr = "198.51.100.22:1234" + request.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0 AppleWebKit/537.36 Safari/537.36") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusForbidden { + t.Fatalf("response status = %d, want %d", response.Code, http.StatusForbidden) + } + if nextCalls != 0 { + t.Fatalf("next handler calls = %d, want 0", nextCalls) + } +} + +func Test404ScoreBansOnSubsequentRequest(t *testing.T) { + cfg := CreateConfig() + cfg.RequireUserAgent = false + cfg.RequireAccept = false + cfg.RandomArticlePatterns = nil + cfg.EmptyUserAgentScore = 0 + cfg.MissingAcceptScore = 0 + cfg.ScoreThreshold = 40 + nextCalls := 0 + handler, err := New(context.Background(), http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + nextCalls++ + rw.WriteHeader(http.StatusNotFound) + }), cfg, "test") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + first := httptest.NewRequest(http.MethodGet, "http://wiki.example/not-found", nil) + first.RemoteAddr = "198.51.100.23:1234" + first.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0 AppleWebKit/537.36 Safari/537.36") + first.Header.Set("Accept", "text/html") + firstResponse := httptest.NewRecorder() + handler.ServeHTTP(firstResponse, first) + if firstResponse.Code != http.StatusNotFound { + t.Fatalf("first response status = %d, want 404", firstResponse.Code) + } + + second := httptest.NewRequest(http.MethodGet, "http://wiki.example/another-miss", nil) + second.RemoteAddr = "198.51.100.23:1234" + second.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0 AppleWebKit/537.36 Safari/537.36") + second.Header.Set("Accept", "text/html") + secondResponse := httptest.NewRecorder() + handler.ServeHTTP(secondResponse, second) + if secondResponse.Code != http.StatusForbidden { + t.Fatalf("second response status = %d, want cached ban after two 404s", secondResponse.Code) + } + if nextCalls != 1 { + t.Fatalf("next handler calls = %d, want 1", nextCalls) + } +} + +func TestTrustedProxyHeaderSeparatesClients(t *testing.T) { + cfg := CreateConfig() + cfg.RequireUserAgent = true + cfg.ClientIPHeader = "X-Forwarded-For" + cfg.TrustedProxyCIDRs = []string{"127.0.0.0/8"} + nextCalls := 0 + handler, err := New(context.Background(), http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + nextCalls++ + rw.WriteHeader(http.StatusNoContent) + }), cfg, "test") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + bad := httptest.NewRequest(http.MethodGet, "http://wiki.example/", nil) + bad.RemoteAddr = "127.0.0.1:1234" + bad.Header.Set("X-Forwarded-For", "198.51.100.61") + bad.Header.Del("User-Agent") + badResponse := httptest.NewRecorder() + handler.ServeHTTP(badResponse, bad) + if badResponse.Code != http.StatusForbidden { + t.Fatalf("bad client status = %d, want 403", badResponse.Code) + } + + good := httptest.NewRequest(http.MethodGet, "http://wiki.example/", nil) + good.RemoteAddr = "127.0.0.1:1234" + good.Header.Set("X-Forwarded-For", "198.51.100.62") + good.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0 AppleWebKit/537.36 Safari/537.36") + goodResponse := httptest.NewRecorder() + handler.ServeHTTP(goodResponse, good) + if goodResponse.Code != http.StatusNoContent { + t.Fatalf("good client status = %d, want 204", goodResponse.Code) + } + if nextCalls != 1 { + t.Fatalf("next handler calls = %d, want 1", nextCalls) + } +} + +func TestCompileConfigRejectsInvalidCIDR(t *testing.T) { + cfg := CreateConfig() + cfg.WhitelistCIDRs = []string{"not-a-cidr"} + if _, err := compileConfig(cfg); err == nil { + t.Fatal("compileConfig() error = nil, want invalid CIDR error") + } +} diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..a5bee7d --- /dev/null +++ b/cache.go @@ -0,0 +1,223 @@ +package botfilter + +import ( + "sync" + "time" +) + +type scoreEvent struct { + at time.Time + points int +} + +type clientState struct { + banUntil time.Time + lastSeen time.Time + requests int + sawHome bool + sawStatic bool + sawFavicon bool + events []scoreEvent +} + +type requestObservation struct { + at time.Time + paths []string + points int + forceBan bool +} + +type cacheResult struct { + banned bool + banUntil time.Time + score int +} + +// clientCache deliberately has no background goroutine. A middleware +// instance can be discarded on a Traefik reload without needing goroutine +// cleanup, and opportunistic cleanup keeps its memory use bounded. +type clientCache struct { + mu sync.Mutex + entries map[string]*clientState + maxEntries int + maxEvents int + scoreWindow time.Duration + banDuration time.Duration + threshold int + randomPaths []string + randomScore int + operations uint64 +} + +func newClientCache(cfg *compiledConfig) *clientCache { + return &clientCache{ + entries: make(map[string]*clientState), + maxEntries: cfg.MaxTrackedIPs, + maxEvents: cfg.MaxScoreEventsPerIP, + scoreWindow: cfg.scoreWindow, + banDuration: cfg.banDuration, + threshold: cfg.ScoreThreshold, + randomPaths: append([]string(nil), cfg.randomPaths...), + randomScore: cfg.RandomArticleScore, + } +} + +func (c *clientCache) isBanned(ip string, now time.Time) cacheResult { + if ip == "" { + return cacheResult{} + } + c.mu.Lock() + defer c.mu.Unlock() + state := c.entries[ip] + if state == nil || !state.banUntil.After(now) { + return cacheResult{} + } + return cacheResult{banned: true, banUntil: state.banUntil} +} + +func (c *clientCache) observeRequest(ip string, observation requestObservation) cacheResult { + if ip == "" { + // An invalid peer address must never share a cache entry with another + // malformed request. The request can still be rejected by its headers. + return cacheResult{banned: observation.forceBan} + } + + c.mu.Lock() + defer c.mu.Unlock() + c.operations++ + c.cleanupLocked(observation.at) + + state := c.entries[ip] + if state != nil && state.banUntil.After(observation.at) { + return cacheResult{banned: true, banUntil: state.banUntil} + } + if state == nil { + c.ensureCapacityLocked(observation.at) + state = &clientState{} + c.entries[ip] = state + } + + c.pruneEventsLocked(state, observation.at) + points := observation.points + c.behaviorScoreLocked(state, observation.paths) + state.requests++ + state.lastSeen = observation.at + if points > 0 { + state.events = append(state.events, scoreEvent{at: observation.at, points: points}) + if len(state.events) > c.maxEvents { + state.events = append([]scoreEvent(nil), state.events[len(state.events)-c.maxEvents:]...) + } + } + score := sumScore(state.events) + if observation.forceBan || score >= c.threshold { + state.banUntil = observation.at.Add(c.banDuration) + return cacheResult{banned: true, banUntil: state.banUntil, score: score} + } + return cacheResult{score: score} +} + +func (c *clientCache) observeResponse(ip string, status int, points int, now time.Time) cacheResult { + if ip == "" || status != 404 || points <= 0 { + return cacheResult{} + } + + c.mu.Lock() + defer c.mu.Unlock() + state := c.entries[ip] + if state == nil || state.banUntil.After(now) { + return cacheResult{} + } + c.pruneEventsLocked(state, now) + state.lastSeen = now + state.events = append(state.events, scoreEvent{at: now, points: points}) + if len(state.events) > c.maxEvents { + state.events = append([]scoreEvent(nil), state.events[len(state.events)-c.maxEvents:]...) + } + score := sumScore(state.events) + if score >= c.threshold { + state.banUntil = now.Add(c.banDuration) + return cacheResult{banned: true, banUntil: state.banUntil, score: score} + } + return cacheResult{score: score} +} + +func (c *clientCache) behaviorScoreLocked(state *clientState, paths []string) int { + // The heuristic is intentionally weak: visiting a content page first is + // normal for a shared link, so it adds only the configured 15 points. + // It becomes useful in combination with malformed headers or 404 scans. + firstRequest := state.requests == 0 + isContent := false + for _, requestPath := range paths { + if requestPath == "/" || requestPath == "/index.html" { + state.sawHome = true + } + if requestPath == "/favicon.ico" { + state.sawFavicon = true + } + if hasStaticAssetExtension(requestPath) { + state.sawStatic = true + } + if pathMatchesAny(requestPath, c.randomPaths) { + isContent = true + } + } + if firstRequest && isContent && !state.sawHome && !state.sawStatic && !state.sawFavicon { + return c.randomScore + } + return 0 +} + +func (c *clientCache) pruneEventsLocked(state *clientState, now time.Time) { + cutoff := now.Add(-c.scoreWindow) + first := 0 + for first < len(state.events) && !state.events[first].at.After(cutoff) { + first++ + } + if first > 0 { + state.events = append([]scoreEvent(nil), state.events[first:]...) + } +} + +func (c *clientCache) cleanupLocked(now time.Time) { + // Full cleanup every 256 requests amortises the map scan while retaining + // recently scored clients for the entire score window. + if c.operations%256 != 0 { + return + } + cutoff := now.Add(-c.scoreWindow) + for ip, state := range c.entries { + if !state.banUntil.After(now) && !state.lastSeen.After(cutoff) { + delete(c.entries, ip) + } + } +} + +func (c *clientCache) ensureCapacityLocked(now time.Time) { + if len(c.entries) < c.maxEntries { + return + } + + // Evict the oldest non-banned entry first. If all entries are banned, the + // oldest ban is evicted; the cap remains a hard upper bound either way. + var oldestIP string + var oldestTime time.Time + for ip, state := range c.entries { + candidate := state.lastSeen + if state.banUntil.After(now) { + candidate = state.banUntil + } + if oldestIP == "" || candidate.Before(oldestTime) { + oldestIP, oldestTime = ip, candidate + } + } + if oldestIP != "" { + delete(c.entries, oldestIP) + } +} + +func sumScore(events []scoreEvent) int { + total := 0 + for _, event := range events { + total += event.points + } + return total +} diff --git a/config.go b/config.go new file mode 100644 index 0000000..bfe9f4f --- /dev/null +++ b/config.go @@ -0,0 +1,168 @@ +package botfilter + +import ( + "fmt" + "net/netip" + "net/textproto" + "strings" + "time" +) + +// Config is the plugin configuration exposed by Traefik's dynamic file +// provider. All durations are expressed as integers because Traefik plugin +// configuration is deliberately kept YAML-friendly. +type Config struct { + StatusCode int `json:"statusCode,omitempty" yaml:"statusCode,omitempty" toml:"statusCode,omitempty"` + RequireUserAgent bool `json:"requireUserAgent,omitempty" yaml:"requireUserAgent,omitempty" toml:"requireUserAgent,omitempty"` + RequireAccept bool `json:"requireAccept,omitempty" yaml:"requireAccept,omitempty" toml:"requireAccept,omitempty"` + RequireHost bool `json:"requireHost,omitempty" yaml:"requireHost,omitempty" toml:"requireHost,omitempty"` + BrowserValidation bool `json:"browserValidation,omitempty" yaml:"browserValidation,omitempty" toml:"browserValidation,omitempty"` + WhitelistCIDRs []string `json:"whitelistCIDRs,omitempty" yaml:"whitelistCIDRs,omitempty" toml:"whitelistCIDRs,omitempty"` + TemporaryBanMinutes int `json:"temporaryBanMinutes,omitempty" yaml:"temporaryBanMinutes,omitempty" toml:"temporaryBanMinutes,omitempty"` + BlockedUserAgents []string `json:"blockedUserAgents,omitempty" yaml:"blockedUserAgents,omitempty" toml:"blockedUserAgents,omitempty"` + BlockedPaths []string `json:"blockedPaths,omitempty" yaml:"blockedPaths,omitempty" toml:"blockedPaths,omitempty"` + BlockedExtensions []string `json:"blockedExtensions,omitempty" yaml:"blockedExtensions,omitempty" toml:"blockedExtensions,omitempty"` + ScoreThreshold int `json:"scoreThreshold,omitempty" yaml:"scoreThreshold,omitempty" toml:"scoreThreshold,omitempty"` + ScoreWindowMinutes int `json:"scoreWindowMinutes,omitempty" yaml:"scoreWindowMinutes,omitempty" toml:"scoreWindowMinutes,omitempty"` + MaxTrackedIPs int `json:"maxTrackedIPs,omitempty" yaml:"maxTrackedIPs,omitempty" toml:"maxTrackedIPs,omitempty"` + MaxScoreEventsPerIP int `json:"maxScoreEventsPerIP,omitempty" yaml:"maxScoreEventsPerIP,omitempty" toml:"maxScoreEventsPerIP,omitempty"` + EmptyUserAgentScore int `json:"emptyUserAgentScore,omitempty" yaml:"emptyUserAgentScore,omitempty" toml:"emptyUserAgentScore,omitempty"` + MissingAcceptScore int `json:"missingAcceptScore,omitempty" yaml:"missingAcceptScore,omitempty" toml:"missingAcceptScore,omitempty"` + BlockedUserAgentScore int `json:"blockedUserAgentScore,omitempty" yaml:"blockedUserAgentScore,omitempty" toml:"blockedUserAgentScore,omitempty"` + BadPathScore int `json:"badPathScore,omitempty" yaml:"badPathScore,omitempty" toml:"badPathScore,omitempty"` + RandomArticleScore int `json:"randomArticleScore,omitempty" yaml:"randomArticleScore,omitempty" toml:"randomArticleScore,omitempty"` + NotFoundScore int `json:"notFoundScore,omitempty" yaml:"notFoundScore,omitempty" toml:"notFoundScore,omitempty"` + FakeBrowserScore int `json:"fakeBrowserScore,omitempty" yaml:"fakeBrowserScore,omitempty" toml:"fakeBrowserScore,omitempty"` + RandomArticlePatterns []string `json:"randomArticlePatterns,omitempty" yaml:"randomArticlePatterns,omitempty" toml:"randomArticlePatterns,omitempty"` + ClientIPHeader string `json:"clientIPHeader,omitempty" yaml:"clientIPHeader,omitempty" toml:"clientIPHeader,omitempty"` + TrustedProxyCIDRs []string `json:"trustedProxyCIDRs,omitempty" yaml:"trustedProxyCIDRs,omitempty" toml:"trustedProxyCIDRs,omitempty"` + LogBlockedRequests bool `json:"logBlockedRequests,omitempty" yaml:"logBlockedRequests,omitempty" toml:"logBlockedRequests,omitempty"` +} + +// CreateConfig creates the default configuration. The defaults protect common +// public HTTP services without requiring a third-party dependency. +func CreateConfig() *Config { + return &Config{ + StatusCode: 403, + TemporaryBanMinutes: 15, + ScoreThreshold: 100, + ScoreWindowMinutes: 15, + MaxTrackedIPs: 50000, + MaxScoreEventsPerIP: 16, + EmptyUserAgentScore: 40, + MissingAcceptScore: 20, + BlockedUserAgentScore: 80, + BadPathScore: 50, + RandomArticleScore: 15, + NotFoundScore: 40, + FakeBrowserScore: 40, + RandomArticlePatterns: []string{"/content/"}, + } +} + +type compiledConfig struct { + Config + banDuration time.Duration + scoreWindow time.Duration + whitelist []netip.Prefix + trustedProxies []netip.Prefix + blockedAgents []string + blockedPaths []string + blockedExts []string + randomPaths []string + clientIPHeader string +} + +func compileConfig(input *Config) (*compiledConfig, error) { + if input == nil { + return nil, fmt.Errorf("botfilter: configuration is nil") + } + + // Copy scalar fields and slices so a later configuration reload cannot + // mutate an already-running middleware instance. + cfg := *input + cfg.WhitelistCIDRs = append([]string(nil), input.WhitelistCIDRs...) + cfg.TrustedProxyCIDRs = append([]string(nil), input.TrustedProxyCIDRs...) + cfg.BlockedUserAgents = append([]string(nil), input.BlockedUserAgents...) + cfg.BlockedPaths = append([]string(nil), input.BlockedPaths...) + cfg.BlockedExtensions = append([]string(nil), input.BlockedExtensions...) + cfg.RandomArticlePatterns = append([]string(nil), input.RandomArticlePatterns...) + + defaults := CreateConfig() + applyDefaults(&cfg, defaults) + + if cfg.StatusCode < 400 || cfg.StatusCode > 599 { + return nil, fmt.Errorf("botfilter: statusCode must be between 400 and 599") + } + if cfg.TemporaryBanMinutes <= 0 { + return nil, fmt.Errorf("botfilter: temporaryBanMinutes must be greater than zero") + } + if cfg.ScoreThreshold <= 0 || cfg.ScoreWindowMinutes <= 0 { + return nil, fmt.Errorf("botfilter: scoreThreshold and scoreWindowMinutes must be greater than zero") + } + if cfg.MaxTrackedIPs <= 0 || cfg.MaxScoreEventsPerIP <= 0 { + return nil, fmt.Errorf("botfilter: maxTrackedIPs and maxScoreEventsPerIP must be greater than zero") + } + + whitelist, err := parseCIDRs(cfg.WhitelistCIDRs, "whitelistCIDRs") + if err != nil { + return nil, err + } + trusted, err := parseCIDRs(cfg.TrustedProxyCIDRs, "trustedProxyCIDRs") + if err != nil { + return nil, err + } + + return &compiledConfig{ + Config: cfg, + banDuration: time.Duration(cfg.TemporaryBanMinutes) * time.Minute, + scoreWindow: time.Duration(cfg.ScoreWindowMinutes) * time.Minute, + whitelist: whitelist, + trustedProxies: trusted, + blockedAgents: normaliseTokens(cfg.BlockedUserAgents), + blockedPaths: normalisePaths(cfg.BlockedPaths), + blockedExts: normaliseExtensions(cfg.BlockedExtensions), + randomPaths: normalisePaths(cfg.RandomArticlePatterns), + clientIPHeader: textproto.CanonicalMIMEHeaderKey(strings.TrimSpace(cfg.ClientIPHeader)), + }, nil +} + +func applyDefaults(cfg, defaults *Config) { + if cfg.StatusCode == 0 { + cfg.StatusCode = defaults.StatusCode + } + if cfg.TemporaryBanMinutes == 0 { + cfg.TemporaryBanMinutes = defaults.TemporaryBanMinutes + } + if cfg.ScoreThreshold == 0 { + cfg.ScoreThreshold = defaults.ScoreThreshold + } + if cfg.ScoreWindowMinutes == 0 { + cfg.ScoreWindowMinutes = defaults.ScoreWindowMinutes + } + if cfg.MaxTrackedIPs == 0 { + cfg.MaxTrackedIPs = defaults.MaxTrackedIPs + } + if cfg.MaxScoreEventsPerIP == 0 { + cfg.MaxScoreEventsPerIP = defaults.MaxScoreEventsPerIP + } + // Score fields deliberately do not receive fallback values here. Traefik + // starts from CreateConfig(), so omitted values retain their defaults, while + // an explicit YAML zero remains a useful way to disable one signal. +} + +func parseCIDRs(values []string, field string) ([]netip.Prefix, error) { + result := make([]netip.Prefix, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + prefix, err := netip.ParsePrefix(value) + if err != nil { + return nil, fmt.Errorf("botfilter: invalid %s entry %q: %w", field, value, err) + } + result = append(result, prefix.Masked()) + } + return result, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..41f73e5 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/hoelee/traefik-botfilter + +go 1.20 diff --git a/logger.go b/logger.go new file mode 100644 index 0000000..df92830 --- /dev/null +++ b/logger.go @@ -0,0 +1,17 @@ +package botfilter + +import "log" + +type filterLogger struct { + name string + enabled bool +} + +func (l filterLogger) blocked(ip, reason string, score int) { + if !l.enabled { + return + } + // This is intentionally opt-in. Logging every rejected request during a + // flood can become a second source of CPU and disk pressure. + log.Printf("botfilter[%s]: blocked client=%s reason=%q score=%d", l.name, ip, reason, score) +} diff --git a/matcher.go b/matcher.go new file mode 100644 index 0000000..793147d --- /dev/null +++ b/matcher.go @@ -0,0 +1,141 @@ +package botfilter + +import ( + "net/http" + "strings" +) + +type matchResult struct { + points int + forceBan bool + reason string +} + +func inspectRequest(r *http.Request, cfg *compiledConfig) matchResult { + paths := requestPaths(r) + userAgent := strings.TrimSpace(r.UserAgent()) + + if cfg.RequireUserAgent && userAgent == "" { + return matchResult{points: cfg.EmptyUserAgentScore, forceBan: true, reason: "missing user-agent"} + } + if cfg.RequireAccept && !hasHeader(r, "Accept") { + return matchResult{points: cfg.MissingAcceptScore, forceBan: true, reason: "missing accept"} + } + if cfg.RequireHost && strings.TrimSpace(r.Host) == "" { + return matchResult{forceBan: true, reason: "missing host"} + } + if containsToken(strings.ToLower(userAgent), cfg.blockedAgents) { + return matchResult{points: cfg.BlockedUserAgentScore, forceBan: true, reason: "blocked user-agent"} + } + if matchesBlockedPath(paths, cfg.blockedPaths) { + return matchResult{points: cfg.BadPathScore, forceBan: true, reason: "blocked path"} + } + if matchesBlockedExtension(paths, cfg.blockedExts) { + return matchResult{points: cfg.BadPathScore, forceBan: true, reason: "blocked extension"} + } + + result := matchResult{} + if userAgent == "" { + result.points += cfg.EmptyUserAgentScore + result.reason = "empty user-agent" + } + if !hasHeader(r, "Accept") { + result.points += cfg.MissingAcceptScore + result.reason = appendReason(result.reason, "missing accept") + } + if cfg.BrowserValidation && implausibleBrowser(r, userAgent) { + result.points += cfg.FakeBrowserScore + result.reason = appendReason(result.reason, "implausible browser headers") + } + return result +} + +func containsToken(value string, tokens []string) bool { + for _, token := range tokens { + if strings.Contains(value, token) { + return true + } + } + return false +} + +func matchesBlockedPath(paths, blockedPaths []string) bool { + for _, requestPath := range paths { + if pathMatchesAny(requestPath, blockedPaths) { + return true + } + } + return false +} + +func matchesBlockedExtension(paths, extensions []string) bool { + for _, requestPath := range paths { + for _, extension := range extensions { + if strings.HasSuffix(requestPath, extension) { + return true + } + } + } + return false +} + +func pathMatchesAny(requestPath string, rules []string) bool { + for _, rule := range rules { + if pathMatches(requestPath, rule) { + return true + } + } + return false +} + +func hasStaticAssetExtension(requestPath string) bool { + for _, extension := range []string{".css", ".js", ".mjs", ".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ico", ".woff", ".woff2"} { + if strings.HasSuffix(requestPath, extension) { + return true + } + } + return false +} + +func implausibleBrowser(r *http.Request, userAgent string) bool { + ua := strings.ToLower(userAgent) + if !strings.Contains(ua, "mozilla/") { + return false + } + + // Do not require optional browser client-hint or fetch-metadata headers: + // older browsers and privacy tools legitimately omit them. Instead reject + // internally inconsistent browser family declarations. + isChromium := strings.Contains(ua, "chrome/") || strings.Contains(ua, "crios/") || strings.Contains(ua, "edg/") || strings.Contains(ua, "opr/") + isFirefox := strings.Contains(ua, "firefox/") + isSafari := strings.Contains(ua, "safari/") && !isChromium + + if !isChromium && !isFirefox && !isSafari { + return true + } + if isChromium && !strings.Contains(ua, "applewebkit/") { + return true + } + if isFirefox && !strings.Contains(ua, "gecko/") { + return true + } + if isSafari && (!strings.Contains(ua, "applewebkit/") || !strings.Contains(ua, "version/")) { + return true + } + + if value := r.Header.Get("Sec-Fetch-Site"); value != "" { + switch value { + case "same-origin", "same-site", "cross-site", "none": + default: + return true + } + } + return false +} + +func appendReason(current, next string) string { + if current == "" { + return next + } + return current + "; " + next +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..7a80ec9 --- /dev/null +++ b/util.go @@ -0,0 +1,127 @@ +package botfilter + +import ( + "net" + "net/http" + "net/netip" + "net/url" + pathpkg "path" + "strings" +) + +func clientIP(r *http.Request, cfg *compiledConfig) netip.Addr { + remote := parseRemoteAddress(r.RemoteAddr) + if cfg.clientIPHeader == "" || !addressInPrefixes(remote, cfg.trustedProxies) { + return remote + } + + // A trusted reverse proxy must overwrite (not append to an untrusted + // client-provided value) this header. The left-most valid value is the + // original client in the conventional X-Forwarded-For representation. + for _, value := range strings.Split(r.Header.Get(cfg.clientIPHeader), ",") { + if addr, err := netip.ParseAddr(strings.TrimSpace(value)); err == nil { + return addr.Unmap() + } + } + return remote +} + +func parseRemoteAddress(value string) netip.Addr { + host, _, err := net.SplitHostPort(strings.TrimSpace(value)) + if err == nil { + value = host + } + addr, err := netip.ParseAddr(strings.Trim(strings.TrimSpace(value), "[]")) + if err != nil { + return netip.Addr{} + } + return addr.Unmap() +} + +func addressInPrefixes(addr netip.Addr, prefixes []netip.Prefix) bool { + if !addr.IsValid() { + return false + } + for _, prefix := range prefixes { + if prefix.Contains(addr) { + return true + } + } + return false +} + +func normaliseTokens(values []string) []string { + result := make([]string, 0, len(values)) + for _, value := range values { + if token := strings.ToLower(strings.TrimSpace(value)); token != "" { + result = append(result, token) + } + } + return result +} + +func normalisePaths(values []string) []string { + result := make([]string, 0, len(values)) + for _, value := range values { + if path := normalisePath(value); path != "" { + result = append(result, path) + } + } + return result +} + +func normaliseExtensions(values []string) []string { + result := make([]string, 0, len(values)) + for _, value := range values { + value = strings.ToLower(strings.TrimSpace(value)) + if value == "" { + continue + } + if !strings.HasPrefix(value, ".") { + value = "." + value + } + result = append(result, value) + } + return result +} + +func normalisePath(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + value = strings.ReplaceAll(value, "\\", "/") + if !strings.HasPrefix(value, "/") { + value = "/" + value + } + return strings.ToLower(pathpkg.Clean(value)) +} + +func requestPaths(r *http.Request) []string { + values := []string{r.URL.Path, r.URL.EscapedPath()} + result := make([]string, 0, len(values)*3) + seen := make(map[string]struct{}) + for _, value := range values { + for i := 0; i < 3 && value != ""; i++ { + normalised := normalisePath(value) + if _, ok := seen[normalised]; !ok { + seen[normalised] = struct{}{} + result = append(result, normalised) + } + decoded, err := url.PathUnescape(value) + if err != nil || decoded == value { + break + } + value = decoded + } + } + return result +} + +func pathMatches(path, rule string) bool { + return path == rule || strings.HasPrefix(path, rule+"/") +} + +func hasHeader(r *http.Request, name string) bool { + return strings.TrimSpace(r.Header.Get(name)) != "" +}