From 17b7c4c55fdb071c3442469d3b4431698c886a7a Mon Sep 17 00:00:00 2001 From: hoelee Date: Wed, 2 Sep 2026 08:25:12 +0800 Subject: [PATCH] Carousell new-listing monitor: NocoDB archive + Telegram alerts --- .dockerignore | 9 + .env.example | 14 ++ .gitignore | 4 + AGENTS.md | 47 +++++ Dockerfile | 12 ++ README.md | 51 ++++++ docker-compose.yml | 30 ++++ healthcheck.py | 19 +++ monitor.py | 418 +++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 604 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 healthcheck.py create mode 100644 monitor.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..aa1e2e1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.gitignore +.env +.env.example +__pycache__ +*.pyc +*.md +SECRETS.md +docker-compose.yml diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d79b6dd --- /dev/null +++ b/.env.example @@ -0,0 +1,14 @@ +# Copy to .env and fill in. All values are required. + +# NocoDB (container DNS when on bridge_hoelee; LAN IP from a desktop): +NOCODB_URL=http://nocodb:10380 +NOCODB_BASE_ID=poqw1zjw3hnsk37 +NOCODB_TOKEN= + +# Telegram alerts (@HoeleeAgentBot): +TELEGRAM_BOT_TOKEN= +TELEGRAM_CHAT_ID=5648309582 + +# Optional tuning: +TICK_SECONDS=60 +HEALTH_STALE_SECONDS=600 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a16c91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +__pycache__/ +*.pyc +.DS_Store diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..574302b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,47 @@ +# AGENTS.md + +Project: Carousell new-listing monitor (Python stdlib, Docker, NocoDB, Telegram). + +## What it does + +`monitor.py` polls Carousell search URLs (sort_by=3 = recent), extracts listings +from the server-rendered `', + html, re.S) + if not blobs: + raise RuntimeError("no application/json state found (blocked/ratelimited?)") + state = json.loads(max(blobs, key=len)) + cards = state["SearchListing"]["listingCards"] + out = [] + for c in cards: + try: + lid = int(c["listingID"]) + ts = None + for item in c.get("aboveFold", []): + if item.get("component") == "time_created": + ts = item["timestampContent"]["seconds"]["low"] + break + bf = c.get("belowFold", []) + title = next((i["stringContent"] for i in bf + if i.get("component") == "header_1"), "") + price_raw = next((i["stringContent"] for i in bf + if i.get("component") == "header_2"), "") + paras = [i.get("stringContent", "") for i in bf + if i.get("component") == "paragraph"] + cond = paras[1].strip() if len(paras) > 1 else "" + if cond not in CONDITIONS: + cond = "" + thumb = c.get("thumbnailURL", "") + seller = (c.get("seller") or {}).get("username", "") + out.append({ + "listing_id": lid, + "title": title, + "price": price_raw, + "condition": cond, + "thumbnail": thumb, + "seller": seller, + "ts": ts, + }) + except (KeyError, TypeError, ValueError): + continue + return out + + +def parse_price(s): + if not s: + return None + t = re.sub(r"[^0-9.]", "", s) + if not t: + return None + try: + return round(float(t), 2) + except ValueError: + return None + + +def mimetype_for(url): + p = url.lower() + if ".png" in p: + return "image/png" + if ".webp" in p: + return "image/webp" + if ".gif" in p: + return "image/gif" + return "image/jpeg" + + +def iso_now(): + # UTC: NocoDB parses naive datetimes as UTC. + return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()) + + +def iso_from_epoch(epoch): + if not epoch: + return None + return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(epoch)) + + +# --------------------------------------------------------------------------- # +# NocoDB row IO +# --------------------------------------------------------------------------- # +def load_seen(listings_tid): + seen = set() + limit, offset = 1000, 0 + while True: + st, j = nc("GET", f"/api/v2/tables/{listings_tid}/records" + f"?limit={limit}&offset={offset}") + if st != 200: + raise RuntimeError(f"load seen failed: {j}") + lst = j.get("list", []) + for r in lst: + if r.get("product_url"): + seen.add(r["product_url"]) + if len(lst) < limit: + break + offset += limit + return seen + + +def load_watches(settings_tid): + st, j = nc("GET", f"/api/v2/tables/{settings_tid}/records?limit=1000") + if st != 200: + raise RuntimeError(f"load watches failed: {j}") + watches = [] + for r in j.get("list", []): + if r.get("enabled"): + watches.append(r) + return watches + + +def insert_listings(listings_tid, rows): + if not rows: + return + st, out = nc("POST", f"/api/v2/tables/{listings_tid}/records", rows) + if st != 200: + raise RuntimeError(f"insert listings failed: {out}") + + +def update_checked(settings_tid, watch_id): + nc("PATCH", f"/api/v2/tables/{settings_tid}/records", + [{"Id": watch_id, "last_checked_at": iso_now()}]) + + +def build_row(l, watch): + row = { + "product_url": PRODUCT_URL_TMPL.format(listing_id=l["listing_id"]), + "title": l["title"], + "image_url": l["thumbnail"], + "image": json.dumps([{"path": l["thumbnail"], + "mimetype": mimetype_for(l["thumbnail"]), + "title": f"{l['listing_id']}.jpg"}]), + "seller_name": l["seller"], + "seller_url": SELLER_URL_TMPL.format(username=l["seller"]), + "search_title": watch.get("title", ""), + "search_url": watch.get("url", ""), + "first_seen_at": iso_now(), + } + price = parse_price(l["price"]) + if price is not None: + row["price"] = price + if l["condition"]: + row["condition"] = l["condition"] + listed = iso_from_epoch(l["ts"]) + if listed: + row["listed_at"] = listed + return row + + +# --------------------------------------------------------------------------- # +# Telegram +# --------------------------------------------------------------------------- # +def send_telegram(text): + if not TELEGRAM_BOT_TOKEN or not TELEGRAM_CHAT_ID: + return + try: + tg("sendMessage", {"chat_id": TELEGRAM_CHAT_ID, "text": text}) + except urllib.error.HTTPError as e: + sys.stderr.write(f"telegram send failed: {e.code} {e.read()[:200]}\n") + + +def plural(n, word): + return f"{n} {word}{'' if n == 1 else 's'}" + + +# --------------------------------------------------------------------------- # +# Health +# --------------------------------------------------------------------------- # +def write_health(ok, error, extra=None): + os.makedirs(DATA_DIR, exist_ok=True) + h = {"last_run_epoch": int(time.time()), "ok": ok, "error": error or ""} + if extra: + h.update(extra) + tmp = os.path.join(DATA_DIR, "health.json.tmp") + with open(tmp, "w", encoding="utf-8") as f: + json.dump(h, f) + os.replace(tmp, os.path.join(DATA_DIR, "health.json")) + + +# --------------------------------------------------------------------------- # +# Main loop +# --------------------------------------------------------------------------- # +def run_tick(listings_tid, settings_tid, seen, last_run): + failures = [] + new_total = 0 + watches = load_watches(settings_tid) + now = time.time() + + for w in watches: + wid = w.get("Id") + interval_min = w.get("check_interval_minutes") or DEFAULT_INTERVAL_MIN + interval_sec = max(int(interval_min), 1) * 60 + if wid in last_run and (now - last_run[wid]) < interval_sec: + continue + + try: + listings = fetch_listings(w.get("url", "")) + except Exception as e: + failures.append(f"{w.get('title')}: {e}") + # still advance so a hard-failing watch doesn't hammer every tick + last_run[wid] = now + continue + + first_seed = not w.get("last_checked_at") + fresh = [l for l in listings + if PRODUCT_URL_TMPL.format(listing_id=l["listing_id"]) not in seen] + + rows = [build_row(l, w) for l in fresh] + if rows: + insert_listings(listings_tid, rows) + for l in fresh: + seen.add(PRODUCT_URL_TMPL.format(listing_id=l["listing_id"])) + + if fresh and not first_seed and w.get("notify", True): + send_telegram(f"{w.get('title')}: {plural(len(fresh), 'new listing')}") + + new_total += len(fresh) + update_checked(settings_tid, wid) + last_run[wid] = now + + ok = len(failures) == 0 + return ok, ("; ".join(failures) if failures else ""), { + "watch_count": len(watches), "new_this_tick": new_total} + + +def main(): + if not NOCODB_TOKEN: + sys.stderr.write("NOCODB_TOKEN not set\n") + write_health(False, "NOCODB_TOKEN not set") + sys.exit(2) + + listings_tid, settings_tid = bootstrap() + seen = load_seen(listings_tid) + last_run = {} + + sys.stderr.write(f"ready: listings={listings_tid} settings={settings_tid} " + f"seen={len(seen)}\n") + + while True: + try: + ok, err, extra = run_tick(listings_tid, settings_tid, seen, last_run) + except Exception as e: + ok, err, extra = False, f"tick error: {e}", {} + write_health(ok, err, extra) + time.sleep(TICK_SECONDS) + + +if __name__ == "__main__": + main()