From 04bfd2d3a0eef6231ec307011b768abf9a32ae2d Mon Sep 17 00:00:00 2001 From: hoelee Date: Sun, 13 Sep 2026 18:59:57 +0800 Subject: [PATCH] Space watch URL fetches with FETCH_GAP_SECONDS (default 1s) Same tick previously fired every due watch back-to-back (burst of N requests, worst right after a restart when all watches are due at once). Now each watch URL fetch is followed by a pause of FETCH_GAP_SECONDS (float, default 1, env tunable; 0 disables) on both success and failure paths. Also thread kw_fk_col into load_ignored_keywords() inside send_pending_notifications() so the physical-FK lookup (445040a) actually takes effect instead of silently falling back to the watch Link column. --- .env.example | 1 + COMPOSE-SETUP.md | 2 ++ DOCUMENTATION.md | 1 + docker-compose.yml | 1 + monitor.py | 8 +++++++- 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index d79b6dd..c045d57 100644 --- a/.env.example +++ b/.env.example @@ -11,4 +11,5 @@ TELEGRAM_CHAT_ID=5648309582 # Optional tuning: TICK_SECONDS=60 +FETCH_GAP_SECONDS=1 HEALTH_STALE_SECONDS=600 diff --git a/COMPOSE-SETUP.md b/COMPOSE-SETUP.md index b9a5162..d3eab5a 100644 --- a/COMPOSE-SETUP.md +++ b/COMPOSE-SETUP.md @@ -96,6 +96,7 @@ build context is the repo directory). TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN} TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID} TICK_SECONDS: ${TICK_SECONDS:-60} + FETCH_GAP_SECONDS: ${FETCH_GAP_SECONDS:-1} HEALTH_STALE_SECONDS: ${HEALTH_STALE_SECONDS:-600} TZ: Asia/Kuala_Lumpur ``` @@ -108,6 +109,7 @@ build context is the repo directory). | `TELEGRAM_BOT_TOKEN` | *(required)* | @carousellFoundBot token (xref `SECRETS.md`). | | `TELEGRAM_CHAT_ID` | *(required)* | `5648309582` — @MrFullStackDev. | | `TICK_SECONDS` | `60` | Scheduler granularity: heartbeat + watch-list reload interval. | +| `FETCH_GAP_SECONDS` | `1` | Minimum pause (s) between watch URL fetches within one tick — prevents request bursts (default 1; set `0` to disable). | | `HEALTH_STALE_SECONDS` | `600` | Docker healthcheck tolerance: if last tick older than this → unhealthy. | | `TZ` | `Asia/Kuala_Lumpur` | Container clock (mostly cosmetic; timestamps are written in UTC deliberately for NocoDB). | diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 2c19db0..7ed8e19 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -57,6 +57,7 @@ Credentials are documented in `SECRETS.md` there. | `TELEGRAM_BOT_TOKEN` | *(secret)* | @carousellFoundBot | | `TELEGRAM_CHAT_ID` | `5648309582` | alert destination | | `TICK_SECONDS` | `60` | scheduler granularity | +| `FETCH_GAP_SECONDS` | `1` | min pause (s) between watch URL fetches within one tick — anti-burst | | `HEALTH_STALE_SECONDS` | `600` | healthcheck staleness window | **Secrets = env vars (`.env`). Operational knobs = NocoDB Settings table.** diff --git a/docker-compose.yml b/docker-compose.yml index 19bc988..aa940b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,7 @@ services: TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN} TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID} TICK_SECONDS: ${TICK_SECONDS:-60} + FETCH_GAP_SECONDS: ${FETCH_GAP_SECONDS:-1} HEALTH_STALE_SECONDS: ${HEALTH_STALE_SECONDS:-600} TZ: Asia/Kuala_Lumpur volumes: diff --git a/monitor.py b/monitor.py index c2bfe6e..f0df534 100644 --- a/monitor.py +++ b/monitor.py @@ -38,6 +38,8 @@ DATA_DIR = os.environ.get("DATA_DIR", "/data") TICK_SECONDS = int(os.environ.get("TICK_SECONDS", "60")) HEALTH_STALE_SECONDS = int(os.environ.get("HEALTH_STALE_SECONDS", "600")) DEFAULT_INTERVAL_MIN = int(os.environ.get("DEFAULT_INTERVAL_MIN", "5")) +# 同一 tick 内逐条抓取 watch URL 之间的最小间隔秒数（防瞬时并发打爆 Carousell）。 +FETCH_GAP_SECONDS = float(os.environ.get("FETCH_GAP_SECONDS", "1")) UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/120.0 Safari/537.36") @@ -629,7 +631,8 @@ def send_pending_notifications(listings_tid, settings_tid, ignored_sellers_tid, # 每轮重新加载忽略列表，中途增删立即生效 ignored = load_ignored_sellers(ignored_sellers_tid) - ignored_kw_by_url = load_ignored_keywords(ignored_keywords_tid, settings_tid) + ignored_kw_by_url = load_ignored_keywords(ignored_keywords_tid, settings_tid, + kw_fk_col) # 拉 notified=false 的记录 st, j = nc("GET", f"/api/v2/tables/{listings_tid}/records" @@ -703,6 +706,7 @@ def run_tick(listings_tid, settings_tid, ignored_sellers_tid, ignored_keywords_t failures.append(f"{w.get('title')}: {e}") # still advance so a hard-failing watch doesn't hammer every tick last_run[wid] = now + time.sleep(FETCH_GAP_SECONDS) continue first_seed = not w.get("last_checked_at") @@ -719,6 +723,8 @@ def run_tick(listings_tid, settings_tid, ignored_sellers_tid, ignored_keywords_t new_total += len(fresh) update_checked(settings_tid, wid) last_run[wid] = now + # 同一 tick 内逐条抓取 watch URL 之间的最小间隔（默认 1s，env FETCH_GAP_SECONDS 可调） + time.sleep(FETCH_GAP_SECONDS) # 归档完成后，统一发送待通知的记录（解耦：归档成功才通知） send_pending_notifications(listings_tid, settings_tid, ignored_sellers_tid,