Add two-stage run verification and 24-hour Activity filter
All checks were successful
Deploy / deploy (push) Successful in 57s

Status now reads "Sent" when the confirmation click lands, since that
only means MassPrice accepted the run request, not that it finished —
their own processing is async and takes anywhere from ~1 to ~22
minutes in observed practice. A new "Complete" column tracks that
separately: run_rule() captures a watermark (the target rule's current
top data-activitylogid on /Activity/List, not the displayed timestamp
string, to sidestep fragile date/timezone parsing) before submitting,
and scheduler.py schedules a one-off verify_run() job
VERIFY_DELAY_MINUTES later to check whether a new "Complete" entry
appeared past that watermark.

rebuild_jobs() now only clears recurring "rule-*" jobs, not the one-off
"verify-*" jobs, so editing a schedule can't cancel a verification
that's already in flight for an earlier run.

Also scopes the Activity page to the last 24 hours per request.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01H6TgffxTvYhqxykAbVQg2v
This commit is contained in:
Barely Removable 2026-08-31 21:49:45 -07:00
parent 988db5d5c4
commit 8b58dadc5a
8 changed files with 249 additions and 20 deletions

View file

@ -4,11 +4,17 @@ A single background worker thread owns every Playwright run — both
APScheduler-triggered runs and manual "Run Now" clicks are pushed onto
one queue.Queue so exactly one browser session is ever active at a time.
This must run in exactly one process; see gunicorn.conf.py's workers=1.
The same worker also handles delayed "verify_run" tasks: after a rule
run is confirmed sent, a one-off APScheduler job (not a recurring cron
job) fires VERIFY_DELAY_MINUTES later and enqueues a check of whether it
actually completed on MassPrice's side — see automation.verify_run().
"""
import logging
import os
import queue
import threading
from datetime import datetime, timedelta
import pytz
from apscheduler.schedulers.background import BackgroundScheduler
@ -29,7 +35,7 @@ _work_queue: "queue.Queue" = queue.Queue()
# wiring). Passing it to the scheduler alone is NOT enough — a
# CronTrigger built without its own explicit timezone still resolves
# get_localzone() itself, regardless of what the scheduler is configured
# with, and crashes the same way. Every CronTrigger(...) below must pass
# with, and crashes the same way. Every trigger built below must pass
# timezone=_TZ explicitly.
_TZ = pytz.timezone(os.environ.get("TZ", "UTC"))
_scheduler = BackgroundScheduler(timezone=_TZ)
@ -37,20 +43,54 @@ _started = False
VALID_DAYS = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"]
# How long to wait after a run is confirmed sent before checking whether
# it actually completed on MassPrice's side. MassPrice's own processing
# time varies a lot in practice (observed anywhere from ~1 to ~22
# minutes) — this is a reasonable middle ground, not a guarantee; a
# "not_confirmed" result can mean "still processing", not just "failed".
VERIFY_DELAY_MINUTES = int(os.environ.get("VERIFY_DELAY_MINUTES", "10"))
def _worker_loop():
while True:
rule_id, rule_name = _work_queue.get()
item = _work_queue.get()
kind = item[0]
try:
automation.run_rule(rule_id, rule_name)
if kind == "run":
_, rule_id, rule_name = item
result = automation.run_rule(rule_id, rule_name)
if result.get("will_verify"):
schedule_verification(
result["entry_id"], rule_name, result["watermark_activity_id"]
)
elif kind == "verify":
_, entry_id, rule_name, watermark_id = item
automation.verify_run(entry_id, rule_name, watermark_id)
except Exception:
logger.exception("Unhandled error running rule %s", rule_id)
logger.exception("Unhandled error processing %r", item)
finally:
_work_queue.task_done()
def enqueue_run(rule_id, rule_name=None):
_work_queue.put((rule_id, rule_name))
_work_queue.put(("run", rule_id, rule_name))
def enqueue_verify(entry_id, rule_name, watermark_id):
_work_queue.put(("verify", entry_id, rule_name, watermark_id))
def schedule_verification(entry_id, rule_name, watermark_id, delay_minutes=None):
delay = VERIFY_DELAY_MINUTES if delay_minutes is None else delay_minutes
run_date = datetime.now(_TZ) + timedelta(minutes=delay)
_scheduler.add_job(
enqueue_verify,
trigger="date",
run_date=run_date,
args=[entry_id, rule_name, watermark_id],
id=f"verify-{entry_id}",
misfire_grace_time=600,
)
def _job_id(rule_id, index):
@ -58,9 +98,16 @@ def _job_id(rule_id, index):
def rebuild_jobs():
"""Clear and re-add every APScheduler job from schedules.json."""
"""Clear and re-add every recurring cron job from schedules.json.
Only touches "rule-*" jobs — one-off "verify-*" jobs (see
schedule_verification()) must survive a schedule edit elsewhere,
since they're tracking an already-in-flight run, not a recurring
schedule.
"""
for job in _scheduler.get_jobs():
job.remove()
if job.id.startswith("rule-"):
job.remove()
schedules = storage.load("schedules.json", {})
for rule_id, rule in schedules.items():