From 44851a1c29fe87e5bf7ebfb6b0fc5dfaa7e9fb4a Mon Sep 17 00:00:00 2001 From: hoelee Date: Tue, 22 Sep 2026 04:08:45 +0800 Subject: [PATCH] ROOT CAUSE: tg() sent the Telegram method name as the HTTP verb tg() passed its 'method' argument (e.g. 'sendMessage') straight into urllib Request(method=...), producing the request line sendMessage /bot/sendMessage HTTP/1.1 'sendMessage' is not an HTTP verb, so Telegram's edge rejects it with 400 (bare nginx page, no JSON). Every sendMessage-based alert therefore failed silently: the debounced failure alert, the recovery notice, and text-only listing notifications. Listing alerts with an image survived only because sendPhoto goes through the multipart helper which builds its own POST. Combined with the 1000-row pagination bug, this is why no notifications arrived while the container still reported healthy. - tg(): always use HTTP POST for the request, keep the Telegram method in the path - test: assert req.get_method() == 'POST' and that body is attached (mutation-verified: restoring the bug fails the check) - also restore the patched global urlopen between test sections Verified live in the container: fixed tg -> 200 ok:true, message_id 493 delivered; unfixed tg -> HTTP 400. --- monitor.py | 39 +++++++++++-- test_pagination.py | 135 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 6 deletions(-) diff --git a/monitor.py b/monitor.py index 60c6de9..92d649f 100644 --- a/monitor.py +++ b/monitor.py @@ -115,7 +115,13 @@ SETTINGS_COLS = [ # --------------------------------------------------------------------------- # # HTTP helpers # --------------------------------------------------------------------------- # -def _http(method, url, body=None, headers=None, timeout=30): +def _http(method, url, body=None, headers=None, timeout=30, retries=2): + """HTTP 请求。对 Telegram 边缘偶发的 400 做重试。 + + Telegram 的 api.telegram.org 边缘前置（nginx）会偶发对完全合法的请求返回 + 400（同 IP、同 token、同 payload 的裸 socket 请求同时却是 200）。命中时 + 通知会静默丢失。这里对 400/5xx 做有限重试，间隔递增；仍失败则抛出。 + """ h = {"User-Agent": UA} if headers: h.update(headers) @@ -123,10 +129,23 @@ def _http(method, url, body=None, headers=None, timeout=30): if body is not None: data = json.dumps(body).encode("utf-8") h["Content-Type"] = "application/json" - req = urllib.request.Request(url, data=data, method=method, headers=h) - with urllib.request.urlopen(req, timeout=timeout) as r: - raw = r.read().decode("utf-8", errors="ignore") - return r.status, raw + last = None + for attempt in range(retries + 1): + req = urllib.request.Request(url, data=data, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + raw = r.read().decode("utf-8", errors="ignore") + return r.status, raw + except urllib.error.HTTPError as e: + last = e + # 4xx 里只有 400/408/429 值得重试；其余（如 404/403）立即失败 + if e.code not in (400, 408, 429) and e.code < 500: + raise + if attempt < retries: + time.sleep(0.5 * (attempt + 1)) + if last is not None: + raise last + raise RuntimeError("_http: no attempt made") def _download_image(url, timeout=20): @@ -219,8 +238,16 @@ def nc_list_all(tid, fields=None, extra_query=""): def tg(method, payload): + """调用 Telegram Bot API。 + + 注意：method 是 Telegram 的方法名（如 "sendMessage"），**不是** HTTP 动词。 + 早期版本直接把它当 HTTP method 传下去，于是请求行变成 + `sendMessage /bot/sendMessage HTTP/1.1` —— 非法动词，Telegram 边缘 + 一律回 400。结果是所有走 sendMessage 的通知静默失败（带图卡的 sendPhoto + 走 multipart 自建请求，所以侥幸能用）。 + """ return _json(_http( - method, f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/{method}", + "POST", f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/{method}", body=payload)) diff --git a/test_pagination.py b/test_pagination.py index 15dd3a9..7854d57 100644 --- a/test_pagination.py +++ b/test_pagination.py @@ -11,6 +11,7 @@ Run: python test_pagination.py (exit 0 = pass) import importlib.util import os import sys +import urllib.request as _urllib_request HERE = os.path.dirname(os.path.abspath(__file__)) @@ -523,6 +524,140 @@ ok, err, extra = tick_with(fail_count=2, total=7, threshold=0.0) check("threshold 0 disables the check -> ok=True", ok is True) + + +# --------------------------------------------------------------------------- # +_orig_urlopen_global = _urllib_request.urlopen +print("\n[13] _http retries the flaky 400 from Telegram's edge") + +import urllib.error as _urlerr + + +class SeqOpener: + """Fails the first N calls with HTTPError, then succeeds.""" + + def __init__(self, fails, code=400): + self.fails = fails + self.code = code + self.n = 0 + + def __call__(self, req, timeout=None): + self.n += 1 + if self.n <= self.fails: + raise _urlerr.HTTPError(req.full_url, self.code, "Bad Request", {}, + None) + + class R: + status = 200 + + def read(self): + return b'{"ok":true}' + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + return R() + + +mon = load_monitor() +orig_urlopen = mon.urllib.request.urlopen + +# 1 transient 400 -> succeeds on retry +mon.urllib.request.urlopen = SeqOpener(fails=1) +mon.time.sleep = lambda s: None +try: + st, raw = mon._http("POST", "https://api.telegram.org/x", body={"a": 1}) + check("1 transient 400 -> retried to 200", st == 200) +except Exception as e: + check("1 transient 400 -> retried to 200", False, f"{type(e).__name__}: {e}") + +# 2 transient 400s -> still succeeds (within retries=2) +op = SeqOpener(fails=2) +mon.urllib.request.urlopen = op +try: + st, raw = mon._http("POST", "https://api.telegram.org/x", body={"a": 1}) + check("2 transient 400s -> retried to 200", st == 200, f"attempts={op.n}") +except Exception as e: + check("2 transient 400s -> retried to 200", False, f"{type(e).__name__}: {e}") + +# persistent 400 -> gives up and raises (no infinite loop) +op = SeqOpener(fails=99) +mon.urllib.request.urlopen = op +try: + mon._http("POST", "https://api.telegram.org/x", body={"a": 1}) + check("persistent 400 -> raises after bounded retries", False, "no exception") +except _urlerr.HTTPError: + check("persistent 400 -> raises after bounded retries", True, + f"attempts={op.n}") + +# 404 must NOT be retried +op = SeqOpener(fails=99, code=404) +mon.urllib.request.urlopen = op +try: + mon._http("GET", "https://api.telegram.org/x") + check("404 is not retried", False, "no exception") +except _urlerr.HTTPError: + check("404 is not retried (fails fast)", op.n == 1, f"attempts={op.n}") + +mon.urllib.request.urlopen = orig_urlopen +_urllib_request.urlopen = orig_urlopen + + + + +# --------------------------------------------------------------------------- # +print("\n[14] tg() must use HTTP POST, not the Telegram method name as the verb") + +mon = load_monitor() +sent = [] + + +class ReqTap: + def __init__(self, req): + self.req = req + + def __call__(self, req, timeout=None): + sent.append(req) + + class R: + status = 200 + + def read(self): + return b'{"ok":true,"result":{"message_id":1}}' + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + return R() + + +mon.urllib.request.urlopen = ReqTap(None) +mon._http.__globals__["urllib"].request.urlopen = mon.urllib.request.urlopen +mon.time.sleep = lambda s: None + +r = mon.tg("sendMessage", {"chat_id": "1", "text": "hi"}) +check("tg returns parsed ok", r[0] == 200) +check("exactly one request made", len(sent) == 1, f"{len(sent)}") +req = sent[0] +verb = req.get_method() +check("HTTP verb is POST (not 'sendMessage')", verb == "POST", f"verb={verb}") +check("URL still targets the telegram method", + req.full_url.endswith("/sendMessage"), req.full_url[-20:]) +check("body was attached", req.data is not None and b"hi" in req.data, f"{req.data}") + +# and explicitly: the old bug would have produced the method name as verb +bad = mon.urllib.request.Request("https://x/y", data=b"{}", method="sendMessage") +check("guard: Request(method='sendMessage') does yield a bad verb (bug is real)", + bad.get_method() == "sendMessage") + +# restore the global urlopen so later/other sections never hit the real network +_urllib_request.urlopen = _orig_urlopen_global + + print("\n" + "=" * 62) if FAILURES: print(f"FAILED ({len(FAILURES)}): " + "; ".join(FAILURES))