ROOT CAUSE: tg() sent the Telegram method name as the HTTP verb

tg() passed its 'method' argument (e.g. 'sendMessage') straight into urllib Request(method=...), producing the request line sendMessage /bot<token>/sendMessage HTTP/1.1 'sendMessage' is not an HTTP verb, so Telegram's edge rejects it with 400 (bare nginx page, no JSON). Every sendMessage-based alert therefore failed silently: the debounced failure alert, the recovery notice, and text-only listing notifications. Listing alerts with an image survived only because sendPhoto goes through the multipart helper which builds its own POST. Combined with the 1000-row pagination bug, this is why no notifications arrived while the container still reported healthy. - tg(): always use HTTP POST for the request, keep the Telegram method in the path - test: assert req.get_method() == 'POST' and that body is attached (mutation-verified: restoring the bug fails the check) - also restore the patched global urlopen between test sections Verified live in the container: fixed tg -> 200 ok:true, message_id 493 delivered; unfixed tg -> HTTP 400.
2026-09-22 04:08:45 +08:00
parent f99a0d878e
commit 44851a1c29
2 changed files with 168 additions and 6 deletions
@@ -115,7 +115,13 @@ SETTINGS_COLS = [
 # --------------------------------------------------------------------------- #
 # HTTP helpers
 # --------------------------------------------------------------------------- #
-def _http(method, url, body=None, headers=None, timeout=30):
+def _http(method, url, body=None, headers=None, timeout=30, retries=2):
+    """HTTP 请求。对 Telegram 边缘偶发的 400 做重试。
+
+    Telegram 的 api.telegram.org 边缘前置（nginx）会偶发对完全合法的请求返回
+    400（同 IP、同 token、同 payload 的裸 socket 请求同时却是 200）。命中时
+    通知会静默丢失。这里对 400/5xx 做有限重试，间隔递增；仍失败则抛出。
+    """
    h = {"User-Agent": UA}
    if headers:
        h.update(headers)
@@ -123,10 +129,23 @@ def _http(method, url, body=None, headers=None, timeout=30):
    if body is not None:
        data = json.dumps(body).encode("utf-8")
        h["Content-Type"] = "application/json"
-    req = urllib.request.Request(url, data=data, method=method, headers=h)
-    with urllib.request.urlopen(req, timeout=timeout) as r:
-        raw = r.read().decode("utf-8", errors="ignore")
-        return r.status, raw
+    last = None
+    for attempt in range(retries + 1):
+        req = urllib.request.Request(url, data=data, method=method, headers=h)
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as r:
+                raw = r.read().decode("utf-8", errors="ignore")
+                return r.status, raw
+        except urllib.error.HTTPError as e:
+            last = e
+            # 4xx 里只有 400/408/429 值得重试；其余（如 404/403）立即失败
+            if e.code not in (400, 408, 429) and e.code < 500:
+                raise
+            if attempt < retries:
+                time.sleep(0.5 * (attempt + 1))
+    if last is not None:
+        raise last
+    raise RuntimeError("_http: no attempt made")


 def _download_image(url, timeout=20):
@@ -219,8 +238,16 @@ def nc_list_all(tid, fields=None, extra_query=""):


 def tg(method, payload):
+    """调用 Telegram Bot API。
+
+    注意：method 是 Telegram 的方法名（如 "sendMessage"），**不是** HTTP 动词。
+    早期版本直接把它当 HTTP method 传下去，于是请求行变成
+    `sendMessage /bot<token>/sendMessage HTTP/1.1` —— 非法动词，Telegram 边缘
+    一律回 400。结果是所有走 sendMessage 的通知静默失败（带图卡的 sendPhoto
+    走 multipart 自建请求，所以侥幸能用）。
+    """
    return _json(_http(
-        method, f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/{method}",
+        "POST", f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/{method}",
        body=payload))


@@ -11,6 +11,7 @@ Run: python test_pagination.py   (exit 0 = pass)
 import importlib.util
 import os
 import sys
+import urllib.request as _urllib_request

 HERE = os.path.dirname(os.path.abspath(__file__))

@@ -523,6 +524,140 @@ ok, err, extra = tick_with(fail_count=2, total=7, threshold=0.0)
 check("threshold 0 disables the check -> ok=True", ok is True)


+
+
+# --------------------------------------------------------------------------- #
+_orig_urlopen_global = _urllib_request.urlopen
+print("\n[13] _http retries the flaky 400 from Telegram's edge")
+
+import urllib.error as _urlerr
+
+
+class SeqOpener:
+    """Fails the first N calls with HTTPError, then succeeds."""
+
+    def __init__(self, fails, code=400):
+        self.fails = fails
+        self.code = code
+        self.n = 0
+
+    def __call__(self, req, timeout=None):
+        self.n += 1
+        if self.n <= self.fails:
+            raise _urlerr.HTTPError(req.full_url, self.code, "Bad Request", {},
+                                    None)
+
+        class R:
+            status = 200
+
+            def read(self):
+                return b'{"ok":true}'
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *a):
+                return False
+        return R()
+
+
+mon = load_monitor()
+orig_urlopen = mon.urllib.request.urlopen
+
+# 1 transient 400 -> succeeds on retry
+mon.urllib.request.urlopen = SeqOpener(fails=1)
+mon.time.sleep = lambda s: None
+try:
+    st, raw = mon._http("POST", "https://api.telegram.org/x", body={"a": 1})
+    check("1 transient 400 -> retried to 200", st == 200)
+except Exception as e:
+    check("1 transient 400 -> retried to 200", False, f"{type(e).__name__}: {e}")
+
+# 2 transient 400s -> still succeeds (within retries=2)
+op = SeqOpener(fails=2)
+mon.urllib.request.urlopen = op
+try:
+    st, raw = mon._http("POST", "https://api.telegram.org/x", body={"a": 1})
+    check("2 transient 400s -> retried to 200", st == 200, f"attempts={op.n}")
+except Exception as e:
+    check("2 transient 400s -> retried to 200", False, f"{type(e).__name__}: {e}")
+
+# persistent 400 -> gives up and raises (no infinite loop)
+op = SeqOpener(fails=99)
+mon.urllib.request.urlopen = op
+try:
+    mon._http("POST", "https://api.telegram.org/x", body={"a": 1})
+    check("persistent 400 -> raises after bounded retries", False, "no exception")
+except _urlerr.HTTPError:
+    check("persistent 400 -> raises after bounded retries", True,
+          f"attempts={op.n}")
+
+# 404 must NOT be retried
+op = SeqOpener(fails=99, code=404)
+mon.urllib.request.urlopen = op
+try:
+    mon._http("GET", "https://api.telegram.org/x")
+    check("404 is not retried", False, "no exception")
+except _urlerr.HTTPError:
+    check("404 is not retried (fails fast)", op.n == 1, f"attempts={op.n}")
+
+mon.urllib.request.urlopen = orig_urlopen
+_urllib_request.urlopen = orig_urlopen
+
+
+
+
+# --------------------------------------------------------------------------- #
+print("\n[14] tg() must use HTTP POST, not the Telegram method name as the verb")
+
+mon = load_monitor()
+sent = []
+
+
+class ReqTap:
+    def __init__(self, req):
+        self.req = req
+
+    def __call__(self, req, timeout=None):
+        sent.append(req)
+
+        class R:
+            status = 200
+
+            def read(self):
+                return b'{"ok":true,"result":{"message_id":1}}'
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *a):
+                return False
+        return R()
+
+
+mon.urllib.request.urlopen = ReqTap(None)
+mon._http.__globals__["urllib"].request.urlopen = mon.urllib.request.urlopen
+mon.time.sleep = lambda s: None
+
+r = mon.tg("sendMessage", {"chat_id": "1", "text": "hi"})
+check("tg returns parsed ok", r[0] == 200)
+check("exactly one request made", len(sent) == 1, f"{len(sent)}")
+req = sent[0]
+verb = req.get_method()
+check("HTTP verb is POST (not 'sendMessage')", verb == "POST", f"verb={verb}")
+check("URL still targets the telegram method",
+      req.full_url.endswith("/sendMessage"), req.full_url[-20:])
+check("body was attached", req.data is not None and b"hi" in req.data, f"{req.data}")
+
+# and explicitly: the old bug would have produced the method name as verb
+bad = mon.urllib.request.Request("https://x/y", data=b"{}", method="sendMessage")
+check("guard: Request(method='sendMessage') does yield a bad verb (bug is real)",
+      bad.get_method() == "sendMessage")
+
+# restore the global urlopen so later/other sections never hit the real network
+_urllib_request.urlopen = _orig_urlopen_global
+
+
 print("\n" + "=" * 62)
 if FAILURES:
    print(f"FAILED ({len(FAILURES)}): " + "; ".join(FAILURES))