From 1cafa21039e825d36c9a924c0d1b7696538d16df Mon Sep 17 00:00:00 2001 From: hoelee Date: Wed, 2 Sep 2026 08:25:12 +0800 Subject: [PATCH] Carousell new-listing monitor: NocoDB archive + Telegram alerts --- .dockerignore | 9 + .env.example | 14 ++ .gitignore | 4 + AGENTS.md | 47 +++++ Dockerfile | 12 ++ README.md | 51 ++++++ SECRETS.md | 9 + docker-compose.yml | 30 ++++ healthcheck.py | 19 +++ monitor.py | 418 +++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 613 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 SECRETS.md create mode 100644 docker-compose.yml create mode 100644 healthcheck.py create mode 100644 monitor.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..aa1e2e1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.gitignore +.env +.env.example +__pycache__ +*.pyc +*.md +SECRETS.md +docker-compose.yml diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d79b6dd --- /dev/null +++ b/.env.example @@ -0,0 +1,14 @@ +# Copy to .env and fill in. All values are required. + +# NocoDB (container DNS when on bridge_hoelee; LAN IP from a desktop): +NOCODB_URL=http://nocodb:10380 +NOCODB_BASE_ID=poqw1zjw3hnsk37 +NOCODB_TOKEN= + +# Telegram alerts (@HoeleeAgentBot): +TELEGRAM_BOT_TOKEN= +TELEGRAM_CHAT_ID=5648309582 + +# Optional tuning: +TICK_SECONDS=60 +HEALTH_STALE_SECONDS=600 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a16c91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +__pycache__/ +*.pyc +.DS_Store diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..574302b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,47 @@ +# AGENTS.md + +Project: Carousell new-listing monitor (Python stdlib, Docker, NocoDB, Telegram). + +## What it does + +`monitor.py` polls Carousell search URLs (sort_by=3 = recent), extracts listings +from the server-rendered `', + html, re.S) + if not blobs: + raise RuntimeError("no application/json state found (blocked/ratelimited?)") + state = json.loads(max(blobs, key=len)) + cards = state["SearchListing"]["listingCards"] + out = [] + for c in cards: + try: + lid = int(c["listingID"]) + ts = None + for item in c.get("aboveFold", []): + if item.get("component") == "time_created": + ts = item["timestampContent"]["seconds"]["low"] + break + bf = c.get("belowFold", []) + title = next((i["stringContent"] for i in bf + if i.get("component") == "header_1"), "") + price_raw = next((i["stringContent"] for i in bf + if i.get("component") == "header_2"), "") + paras = [i.get("stringContent", "") for i in bf + if i.get("component") == "paragraph"] + cond = paras[1].strip() if len(paras) > 1 else "" + if cond not in CONDITIONS: + cond = "" + thumb = c.get("thumbnailURL", "") + seller = (c.get("seller") or {}).get("username", "") + out.append({ + "listing_id": lid, + "title": title, + "price": price_raw, + "condition": cond, + "thumbnail": thumb, + "seller": seller, + "ts": ts, + }) + except (KeyError, TypeError, ValueError): + continue + return out + + +def parse_price(s): + if not s: + return None + t = re.sub(r"[^0-9.]", "", s) + if not t: + return None + try: + return round(float(t), 2) + except ValueError: + return None + + +def mimetype_for(url): + p = url.lower() + if ".png" in p: + return "image/png" + if ".webp" in p: + return "image/webp" + if ".gif" in p: + return "image/gif" + return "image/jpeg" + + +def iso_now(): + # UTC: NocoDB parses naive datetimes as UTC. + return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()) + + +def iso_from_epoch(epoch): + if not epoch: + return None + return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(epoch)) + + +# --------------------------------------------------------------------------- # +# NocoDB row IO +# --------------------------------------------------------------------------- # +def load_seen(listings_tid): + seen = set() + limit, offset = 1000, 0 + while True: + st, j = nc("GET", f"/api/v2/tables/{listings_tid}/records" + f"?limit={limit}&offset={offset}") + if st != 200: + raise RuntimeError(f"load seen failed: {j}") + lst = j.get("list", []) + for r in lst: + if r.get("product_url"): + seen.add(r["product_url"]) + if len(lst) < limit: + break + offset += limit + return seen + + +def load_watches(settings_tid): + st, j = nc("GET", f"/api/v2/tables/{settings_tid}/records?limit=1000") + if st != 200: + raise RuntimeError(f"load watches failed: {j}") + watches = [] + for r in j.get("list", []): + if r.get("enabled"): + watches.append(r) + return watches + + +def insert_listings(listings_tid, rows): + if not rows: + return + st, out = nc("POST", f"/api/v2/tables/{listings_tid}/records", rows) + if st != 200: + raise RuntimeError(f"insert listings failed: {out}") + + +def update_checked(settings_tid, watch_id): + nc("PATCH", f"/api/v2/tables/{settings_tid}/records", + [{"Id": watch_id, "last_checked_at": iso_now()}]) + + +def build_row(l, watch): + row = { + "product_url": PRODUCT_URL_TMPL.format(listing_id=l["listing_id"]), + "title": l["title"], + "image_url": l["thumbnail"], + "image": json.dumps([{"path": l["thumbnail"], + "mimetype": mimetype_for(l["thumbnail"]), + "title": f"{l['listing_id']}.jpg"}]), + "seller_name": l["seller"], + "seller_url": SELLER_URL_TMPL.format(username=l["seller"]), + "search_title": watch.get("title", ""), + "search_url": watch.get("url", ""), + "first_seen_at": iso_now(), + } + price = parse_price(l["price"]) + if price is not None: + row["price"] = price + if l["condition"]: + row["condition"] = l["condition"] + listed = iso_from_epoch(l["ts"]) + if listed: + row["listed_at"] = listed + return row + + +# --------------------------------------------------------------------------- # +# Telegram +# --------------------------------------------------------------------------- # +def send_telegram(text): + if not TELEGRAM_BOT_TOKEN or not TELEGRAM_CHAT_ID: + return + try: + tg("sendMessage", {"chat_id": TELEGRAM_CHAT_ID, "text": text}) + except urllib.error.HTTPError as e: + sys.stderr.write(f"telegram send failed: {e.code} {e.read()[:200]}\n") + + +def plural(n, word): + return f"{n} {word}{'' if n == 1 else 's'}" + + +# --------------------------------------------------------------------------- # +# Health +# --------------------------------------------------------------------------- # +def write_health(ok, error, extra=None): + os.makedirs(DATA_DIR, exist_ok=True) + h = {"last_run_epoch": int(time.time()), "ok": ok, "error": error or ""} + if extra: + h.update(extra) + tmp = os.path.join(DATA_DIR, "health.json.tmp") + with open(tmp, "w", encoding="utf-8") as f: + json.dump(h, f) + os.replace(tmp, os.path.join(DATA_DIR, "health.json")) + + +# --------------------------------------------------------------------------- # +# Main loop +# --------------------------------------------------------------------------- # +def run_tick(listings_tid, settings_tid, seen, last_run): + failures = [] + new_total = 0 + watches = load_watches(settings_tid) + now = time.time() + + for w in watches: + wid = w.get("Id") + interval_min = w.get("check_interval_minutes") or DEFAULT_INTERVAL_MIN + interval_sec = max(int(interval_min), 1) * 60 + if wid in last_run and (now - last_run[wid]) < interval_sec: + continue + + try: + listings = fetch_listings(w.get("url", "")) + except Exception as e: + failures.append(f"{w.get('title')}: {e}") + # still advance so a hard-failing watch doesn't hammer every tick + last_run[wid] = now + continue + + first_seed = not w.get("last_checked_at") + fresh = [l for l in listings + if PRODUCT_URL_TMPL.format(listing_id=l["listing_id"]) not in seen] + + rows = [build_row(l, w) for l in fresh] + if rows: + insert_listings(listings_tid, rows) + for l in fresh: + seen.add(PRODUCT_URL_TMPL.format(listing_id=l["listing_id"])) + + if fresh and not first_seed and w.get("notify", True): + send_telegram(f"{w.get('title')}: {plural(len(fresh), 'new listing')}") + + new_total += len(fresh) + update_checked(settings_tid, wid) + last_run[wid] = now + + ok = len(failures) == 0 + return ok, ("; ".join(failures) if failures else ""), { + "watch_count": len(watches), "new_this_tick": new_total} + + +def main(): + if not NOCODB_TOKEN: + sys.stderr.write("NOCODB_TOKEN not set\n") + write_health(False, "NOCODB_TOKEN not set") + sys.exit(2) + + listings_tid, settings_tid = bootstrap() + seen = load_seen(listings_tid) + last_run = {} + + sys.stderr.write(f"ready: listings={listings_tid} settings={settings_tid} " + f"seen={len(seen)}\n") + + while True: + try: + ok, err, extra = run_tick(listings_tid, settings_tid, seen, last_run) + except Exception as e: + ok, err, extra = False, f"tick error: {e}", {} + write_health(ok, err, extra) + time.sleep(TICK_SECONDS) + + +if __name__ == "__main__": + main()