diff --git a/.env.example b/.env.example index 09778cd..7c1159e 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,14 @@ HOST_PORT=5057 # Controls what "6:00 AM" means for your schedules TZ=America/Phoenix +# ── Run verification delay ────────────────────────────────────────── +# Minutes to wait after a run is confirmed sent before checking +# MassPrice's own Activity Log to see whether it actually completed. +# MassPrice's processing time varies a lot in practice (observed +# anywhere from ~1 to ~22 minutes) — this is a reasonable middle +# ground, not a guarantee. +VERIFY_DELAY_MINUTES=10 + # ── Session-expiry email alert (optional) ─────────────────────────── # Sent once when the saved MassPrice session expires/is missing — not # on every failed run, just the first one after it goes bad. Same diff --git a/CLAUDE.md b/CLAUDE.md index 2e138a5..a3bbed5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,6 +39,12 @@ tool is confirmed working, so there's exactly one thing triggering runs. `workers = 1` for this reason — a second worker would instantiate a second scheduler and duplicate every run. Fine for a low-traffic single-user tool. + - Also schedules **one-off** `verify-` jobs (APScheduler + `date` trigger, not recurring) `VERIFY_DELAY_MINUTES` after a run is + confirmed sent — see `automation.verify_run()` below. + `rebuild_jobs()` only ever removes `rule-*` jobs, deliberately, so + editing a schedule mid-flight can't cancel a verification that's + already in progress for an earlier run. - **`automation.py`** — all Playwright logic. `list_rules()` scrapes `https://massprice.tcgplayer.com/Rule/List` for the live rule id/name pairs (cached to `rules_cache.json`, refreshed via the @@ -63,6 +69,25 @@ tool is confirmed working, so there's exactly one thing triggering runs. and its request (`POST /Rule/Run`, confirmed by live network capture) is almost certainly anti-forgery-token protected, so automation drives the real page rather than replaying a raw HTTP POST. +- **Run verification** — confirming the modal only means MassPrice + *accepted* the run; it still processes asynchronously on their side + (queued → running → Complete), taking anywhere from ~1 to ~22 minutes + in observed practice. `run_rule()` captures a "watermark" — the + `data-activitylogid` of that rule's current top entry on + `/Activity/List` (a stable, monotonically increasing ID; deliberately + *not* parsing the displayed timestamp string, which would need + fragile date/timezone handling) — **before** submitting, logs the run + as `status: "sent"` with `verification: "pending"`, and the caller in + `scheduler.py` schedules `verify_run()` to run `VERIFY_DELAY_MINUTES` + later. `verify_run()` re-checks `/Activity/List`: a new entry (higher + `data-activitylogid` than the watermark) whose status text contains + "Complete" → `verification: "complete"`; otherwise + `verification: "not_confirmed"` (which can mean "still processing", + not necessarily "failed" — the delay is a reasonable middle ground, + not a guarantee it's had enough time). `_find_latest_activity()` is + shared by the watermark capture and the follow-up check. The Activity + page's "Complete" column surfaces this; "Status" shows whether the + click itself was sent, independent of whether MassPrice finished it. - **Session persistence** — a Playwright `storage_state` (cookies + localStorage) is saved to `/data/auth_state.json` and reused across runs: scheduled runs never touch the login form at all, they just load diff --git a/app.py b/app.py index dc65fdc..5e09b42 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,6 @@ import hmac import os +from datetime import datetime, timedelta, timezone from functools import wraps from flask import Flask, jsonify, redirect, render_template, request, session, url_for @@ -8,7 +9,7 @@ import automation import scheduler import storage -APP_VERSION = "0.6.0" +APP_VERSION = "0.7.0" APP_VERSION_DATE = "2026-09-01" app = Flask(__name__) @@ -127,11 +128,20 @@ def api_session_import(): return jsonify({"ok": True}) +def _parse_ts(value): + try: + return datetime.fromisoformat(value) + except (TypeError, ValueError): + return None + + @app.route("/activity") @login_required def activity(): history = storage.load("run_history.json", []) - return render_template("activity.html", history=history) + cutoff = datetime.now(timezone.utc) - timedelta(hours=24) + recent = [e for e in history if (_parse_ts(e.get("timestamp")) or cutoff) >= cutoff] + return render_template("activity.html", history=recent) @app.route("/api/rules/refresh", methods=["POST"]) diff --git a/automation.py b/automation.py index 07882b6..212141b 100644 --- a/automation.py +++ b/automation.py @@ -32,6 +32,7 @@ authenticates before accepting it. import json import logging import os +import uuid from datetime import datetime, timezone from playwright.sync_api import sync_playwright @@ -163,11 +164,36 @@ def list_rules(): browser.close() +def _find_latest_activity(page, rule_name): + """Returns (activity_id: int, status_text: str) for the most recent + Activity Log entry matching rule_name, or (None, None) if there isn't + one. Each Activity Log row carries a stable, monotonically increasing + `data-activitylogid` — comparing that (not the displayed timestamp + string, which would need fragile date/timezone parsing) is how a + "new completion appeared" check is done reliably.""" + page.goto(f"{BASE_URL}/Activity/List") + page.wait_for_timeout(1000) + rows = page.locator("div.activity[data-activitylogid]").filter( + has=page.locator(f'.activity__name[title="{rule_name}"]') + ) + if rows.count() == 0: + return None, None + row = rows.first + activity_id = int(row.get_attribute("data-activitylogid")) + status_text = row.locator(".activity__runtime").first.inner_text().strip() + return activity_id, status_text + + def run_rule(rule_id, rule_name=None): - """Click "Run Rule" for the given rule id. Logs the outcome to - run_history.json and returns (status, message).""" + """Click "Run Rule" (through its confirmation modal) for the given + rule id. Logs the outcome to run_history.json and returns a dict: + {"status": "sent"|"failure", "message": ..., "entry_id": ..., + "watermark_activity_id": ...}. On "sent", the caller (scheduler.py) + schedules a delayed verify_run() call using entry_id/watermark_activity_id + to confirm the run actually completed on MassPrice's side.""" status = "failure" message = "" + watermark_id = None with sync_playwright() as p: browser = p.chromium.launch(headless=True, args=LAUNCH_ARGS) try: @@ -176,6 +202,20 @@ def run_rule(rule_id, rule_name=None): _ensure_authenticated(page) _save_state(context) + # Capture a watermark before submitting, so the later + # verification check can tell "a new completion appeared" + # apart from "the same old entry is still sitting there". + # Best-effort: if this fails for any reason, proceed with the + # actual run anyway — the run itself matters more than the + # optional follow-up verification. + if rule_name: + try: + watermark_id, _ = _find_latest_activity(page, rule_name) + except Exception: + logger.exception("Failed to capture activity watermark for %s", rule_name) + page.goto(f"{BASE_URL}/Rule/List") + page.wait_for_timeout(500) + run_selector = f'a.run-rule[data-rule-id="{rule_id}"]' row = page.locator("li.rule.widget-rule").filter( has=page.locator(run_selector) @@ -206,31 +246,109 @@ def run_rule(rule_id, rule_name=None): modal.wait_for(state="hidden", timeout=10000) page.wait_for_timeout(1000) # let the confirmed run's request land - status = "success" - message = "Run Rule clicked" + status = "sent" + message = "Run Rule confirmed" except Exception as exc: message = str(exc) logger.exception("run_rule failed for rule %s", rule_id) finally: browser.close() - _log_history(rule_id, rule_name, status, message) - return status, message + entry_id = _log_history(rule_id, rule_name, status, message) + will_verify = status == "sent" and bool(rule_name) + if will_verify: + _set_verification_pending(entry_id) + + return { + "status": status, + "message": message, + "entry_id": entry_id, + "watermark_activity_id": watermark_id, + "will_verify": will_verify, + } + + +def verify_run(entry_id, rule_name, watermark_id): + """Delayed follow-up: check whether the run logged as entry_id + actually completed on MassPrice's side by now. Updates that + run_history.json entry's "verification" field in place.""" + verification = "not_confirmed" + verify_message = "" + with sync_playwright() as p: + browser = p.chromium.launch(headless=True, args=LAUNCH_ARGS) + try: + context = _new_context(browser) + page = context.new_page() + _ensure_authenticated(page) + _save_state(context) + + activity_id, status_text = _find_latest_activity(page, rule_name) + if activity_id is None: + verify_message = "No matching activity found on MassPrice" + elif watermark_id is not None and activity_id <= watermark_id: + verify_message = "No new completion appeared within the allotted time" + elif status_text and "complete" in status_text.lower(): + verification = "complete" + else: + verify_message = f"Latest activity shows \"{status_text}\", not Complete" + except Exception as exc: + verify_message = str(exc) + logger.exception("verify_run failed for entry %s", entry_id) + finally: + browser.close() + + _update_verification(entry_id, verification, verify_message) def _log_history(rule_id, rule_name, status, message): + entry_id = uuid.uuid4().hex history = storage.load("run_history.json", []) history.insert( 0, { + "id": entry_id, "rule_id": rule_id, "rule_name": rule_name, "timestamp": datetime.now(timezone.utc).isoformat(), "status": status, "message": message, + # "pending" only when a verify_run() has actually been + # scheduled for this entry (status == "sent" and rule_name + # was known) — set explicitly by run_rule() below, not here, + # since _log_history() itself doesn't know whether scheduling + # succeeded. + "verification": None, + "verify_message": None, }, ) storage.save("run_history.json", history[:200]) + return entry_id + + +def _set_verification_pending(entry_id): + history = storage.load("run_history.json", []) + for entry in history: + if entry.get("id") == entry_id: + entry["verification"] = "pending" + break + else: + return + storage.save("run_history.json", history) + + +def _update_verification(entry_id, verification, verify_message): + history = storage.load("run_history.json", []) + for entry in history: + if entry.get("id") == entry_id: + entry["verification"] = verification + entry["verify_message"] = verify_message + entry["verified_at"] = datetime.now(timezone.utc).isoformat() + break + else: + # Entry aged out of the 200-cap between being logged and this + # delayed check running — nothing to update, not an error. + return + storage.save("run_history.json", history) # ── Login-session import (paste cookies exported from a real browser) ─── diff --git a/docker-compose.yml b/docker-compose.yml index c046887..19b4bbc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,7 @@ services: - LOGIN_PASSWORD=${LOGIN_PASSWORD} - SECRET_KEY=${SECRET_KEY} - TZ=${TZ:-America/Phoenix} + - VERIFY_DELAY_MINUTES=${VERIFY_DELAY_MINUTES:-10} - GMAIL_USER=${GMAIL_USER:-} - GMAIL_APP_PASSWORD=${GMAIL_APP_PASSWORD:-} - ALERT_EMAIL=${ALERT_EMAIL:-} diff --git a/scheduler.py b/scheduler.py index f996b58..4d1c192 100644 --- a/scheduler.py +++ b/scheduler.py @@ -4,11 +4,17 @@ A single background worker thread owns every Playwright run — both APScheduler-triggered runs and manual "Run Now" clicks are pushed onto one queue.Queue so exactly one browser session is ever active at a time. This must run in exactly one process; see gunicorn.conf.py's workers=1. + +The same worker also handles delayed "verify_run" tasks: after a rule +run is confirmed sent, a one-off APScheduler job (not a recurring cron +job) fires VERIFY_DELAY_MINUTES later and enqueues a check of whether it +actually completed on MassPrice's side — see automation.verify_run(). """ import logging import os import queue import threading +from datetime import datetime, timedelta import pytz from apscheduler.schedulers.background import BackgroundScheduler @@ -29,7 +35,7 @@ _work_queue: "queue.Queue" = queue.Queue() # wiring). Passing it to the scheduler alone is NOT enough — a # CronTrigger built without its own explicit timezone still resolves # get_localzone() itself, regardless of what the scheduler is configured -# with, and crashes the same way. Every CronTrigger(...) below must pass +# with, and crashes the same way. Every trigger built below must pass # timezone=_TZ explicitly. _TZ = pytz.timezone(os.environ.get("TZ", "UTC")) _scheduler = BackgroundScheduler(timezone=_TZ) @@ -37,20 +43,54 @@ _started = False VALID_DAYS = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] +# How long to wait after a run is confirmed sent before checking whether +# it actually completed on MassPrice's side. MassPrice's own processing +# time varies a lot in practice (observed anywhere from ~1 to ~22 +# minutes) — this is a reasonable middle ground, not a guarantee; a +# "not_confirmed" result can mean "still processing", not just "failed". +VERIFY_DELAY_MINUTES = int(os.environ.get("VERIFY_DELAY_MINUTES", "10")) + def _worker_loop(): while True: - rule_id, rule_name = _work_queue.get() + item = _work_queue.get() + kind = item[0] try: - automation.run_rule(rule_id, rule_name) + if kind == "run": + _, rule_id, rule_name = item + result = automation.run_rule(rule_id, rule_name) + if result.get("will_verify"): + schedule_verification( + result["entry_id"], rule_name, result["watermark_activity_id"] + ) + elif kind == "verify": + _, entry_id, rule_name, watermark_id = item + automation.verify_run(entry_id, rule_name, watermark_id) except Exception: - logger.exception("Unhandled error running rule %s", rule_id) + logger.exception("Unhandled error processing %r", item) finally: _work_queue.task_done() def enqueue_run(rule_id, rule_name=None): - _work_queue.put((rule_id, rule_name)) + _work_queue.put(("run", rule_id, rule_name)) + + +def enqueue_verify(entry_id, rule_name, watermark_id): + _work_queue.put(("verify", entry_id, rule_name, watermark_id)) + + +def schedule_verification(entry_id, rule_name, watermark_id, delay_minutes=None): + delay = VERIFY_DELAY_MINUTES if delay_minutes is None else delay_minutes + run_date = datetime.now(_TZ) + timedelta(minutes=delay) + _scheduler.add_job( + enqueue_verify, + trigger="date", + run_date=run_date, + args=[entry_id, rule_name, watermark_id], + id=f"verify-{entry_id}", + misfire_grace_time=600, + ) def _job_id(rule_id, index): @@ -58,9 +98,16 @@ def _job_id(rule_id, index): def rebuild_jobs(): - """Clear and re-add every APScheduler job from schedules.json.""" + """Clear and re-add every recurring cron job from schedules.json. + + Only touches "rule-*" jobs — one-off "verify-*" jobs (see + schedule_verification()) must survive a schedule edit elsewhere, + since they're tracking an already-in-flight run, not a recurring + schedule. + """ for job in _scheduler.get_jobs(): - job.remove() + if job.id.startswith("rule-"): + job.remove() schedules = storage.load("schedules.json", {}) for rule_id, rule in schedules.items(): diff --git a/static/css/style.css b/static/css/style.css index 5ae5a27..8cfc06e 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -9,6 +9,7 @@ --accent-dim: #1e3a5f; --success: #34d399; --failure: #f87171; + --warn: #fbbf24; --radius: 8px; } @@ -111,8 +112,12 @@ footer { .rule-last-run { font-size: 0.82rem; } .status-dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 6px; } -.status-dot.status-success { background: var(--success); } +.status-dot.status-success, +.status-dot.status-sent, +.status-dot.status-complete { background: var(--success); } .status-dot.status-failure { background: var(--failure); } +.status-dot.status-pending { background: var(--text-dim); } +.status-dot.status-not-confirmed { background: var(--warn); } .status-row-failure td { color: #f2a3a3; } .schedule-list { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 6px; } diff --git a/templates/activity.html b/templates/activity.html index cd9c07c..db092f4 100644 --- a/templates/activity.html +++ b/templates/activity.html @@ -2,20 +2,35 @@ {% block title %}Activity · MassPrice Scheduler{% endblock %} {% block content %}

Activity

+

Showing the last 24 hours.

{% if not history %} -

No runs recorded yet.

+

No runs recorded in the last 24 hours.

{% else %} - + {% for entry in history %} - - + + + {% endfor %}
TimeRuleStatusMessage
TimeRuleStatusCompleteMessage
{{ entry.timestamp }} {{ entry.rule_name or entry.rule_id }}{{ entry.status }}{{ entry.message }}{{ entry.status | capitalize }} + {% if entry.verification == 'complete' %} + Complete + {% elif entry.verification == 'pending' %} + Pending + {% elif entry.verification == 'not_confirmed' %} + Not Confirmed + {% else %} + — + {% endif %} + + {{ entry.message }} + {% if entry.verify_message %}
{{ entry.verify_message }}
{% endif %} +