From 602bce8ce36bc80e5df6764c5b48af29d11e284e Mon Sep 17 00:00:00 2001 From: Barely Removable Date: Sat, 22 Aug 2026 13:07:09 -0700 Subject: [PATCH] Expire stored source photos after 7 days, keeping grades forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Storing originals for Regrade grew the database ~16MB per grade at the upload cap. Photos now expire on a configurable window (default 7 days, counted from last grade so regrading resets it) and the file is VACUUMed so the space is actually returned — clearing the column alone only moves pages to the freelist, which would have made the whole feature a no-op on disk. Grades, thumbnails and measurements are never pruned; a pruned card's Regrade falls back to the existing re-pick path, now with a banner saying why. --- README.md | 27 +++++++++++++++++++++++ app.py | 35 ++++++++++++++++++++++++++++++ docker-compose.yml | 7 ++++++ static/app.js | 4 ++++ store.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+) diff --git a/README.md b/README.md index e84b8df..54f7f9a 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,33 @@ All your data lives in `grades.db` in this folder — your settings and your entire grading history, thumbnails included. Back up that one file and you've backed up everything. +### What's kept, and for how long + +Grades are kept **forever**: the estimate, every category finding, the +measurements, and the thumbnail. Nothing expires there. + +The **original photos** are a different matter. Each grade stores the +photo(s) that produced it so **Regrade** can re-run without asking for them +again — which is worth most right after a grading-logic change, and worth +progressively less as time passes. They're also what makes the database +grow: up to ~16MB per grade at the upload cap, so a few hundred grades +would otherwise run into gigabytes. + +So stored photos expire after **7 days**, and the database is compacted to +actually give the space back. Past that window a card's Regrade button asks +you to pick the photo again instead of re-running instantly — and once you +do, that card starts the clock over. + +Change the window with `CARD_GRADER_IMAGE_RETENTION_DAYS` (set it to `0` to +keep photos forever, if disk isn't a concern): + +```bash +CARD_GRADER_IMAGE_RETENTION_DAYS=30 python3 app.py +``` + +The countdown runs from when a card was last graded, not when it was first +graded — so regrading a card keeps its photos for another full window. + ### Optional packages Two packages unlock real functionality. Each is checked for at runtime, so diff --git a/app.py b/app.py index a6f30ef..e8c81a6 100644 --- a/app.py +++ b/app.py @@ -21,6 +21,7 @@ import io import json import os import sys +import threading import time import urllib.parse from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -43,6 +44,11 @@ MAX_UPLOAD_BYTES = 12 * 1024 * 1024 # ~4/3 that as base64, plus JSON overhead. Kept in step with nginx's # client_max_body_size in nginx/card-grader.conf. MAX_BODY_BYTES = 20 * 1024 * 1024 +# How often to expire stored source photos (see store.prune_source_images). +# Six-hourly rather than daily so a long-running container doesn't hold a +# week's worth of extra images for up to another day past the window, and +# so the first sweep after a restart isn't the only one that ever runs. +PRUNE_INTERVAL_SECONDS = 6 * 60 * 60 # More angles help — front, back, corner close-ups — but past a handful the # extra photos cost tokens without adding evidence. MAX_GRADE_IMAGES = 6 @@ -490,6 +496,26 @@ class Server(ThreadingHTTPServer): daemon_threads = True +def _prune_forever(): + """Expire stored source photos on a timer, for the life of the process. + + A daemon thread rather than a cron entry so the retention window holds + on any host this runs on, without a second thing to install and keep in + step. Never lets an exception end the loop: a failed sweep should cost + one cycle, not disable pruning until the next restart. + """ + while True: + try: + pruned = store.prune_source_images() + if pruned: + print("[prune] dropped stored photos from {} grade(s) older " + "than {} days".format(pruned, store.SOURCE_IMAGE_RETENTION_DAYS), + flush=True) + except Exception as exc: + print("[prune] failed: {}".format(exc), flush=True) + time.sleep(PRUNE_INTERVAL_SECONDS) + + def main(): store.init() try: @@ -507,8 +533,17 @@ def main(): if ip: print(" On your phone: http://{}:{} (same Wi-Fi)".format(ip, PORT)) print(" Database: {}".format(store.DB_PATH)) + if store.SOURCE_IMAGE_RETENTION_DAYS > 0: + print(" Photos kept: {} days (grades kept forever)".format( + store.SOURCE_IMAGE_RETENTION_DAYS)) + else: + print(" Photos kept: forever (pruning disabled)") print("\n Ctrl-C to stop.\n") + # Sweep once at boot so a container that restarts often still expires + # things, then hand off to the timer. + threading.Thread(target=_prune_forever, daemon=True).start() + try: server.serve_forever() except KeyboardInterrupt: diff --git a/docker-compose.yml b/docker-compose.yml index e441e2c..e94a807 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,3 +35,10 @@ services: # This app is reverse-proxied at hippofam.com/cards, not the domain # root — see app.py's BASE_PATH handling. - CARD_GRADER_BASE_PATH=/cards + # How long a grade keeps the original photo(s) that produced it, so + # Regrade can re-run without asking for them again. Grades, their + # thumbnails and every measurement are kept FOREVER regardless — only + # the photos expire, since they're what makes the database grow (up + # to ~16MB per grade at the upload cap). Past the window, Regrade + # falls back to asking for the photo. 0 disables pruning. + - CARD_GRADER_IMAGE_RETENTION_DAYS=7 diff --git a/static/app.js b/static/app.js index 905595c..be45d1c 100644 --- a/static/app.js +++ b/static/app.js @@ -483,6 +483,10 @@ function startRegrade(id, hasImages) { if (hasImages) { runRegrade(id, {}); } else { + // Say why the picker is opening. Otherwise a Regrade that silently + // asks for a photo looks broken — especially on a card that regraded + // instantly last week, before its stored photo aged out. + banner("This card's photo is no longer stored — pick it again to regrade."); regradeTargetId = id; $('#regrade-file').click(); } diff --git a/store.py b/store.py index 1ffc727..4a4285a 100644 --- a/store.py +++ b/store.py @@ -58,6 +58,20 @@ DEFAULT_SETTINGS = { "vision_effort": "low", } +# How long a grade keeps the original photo(s) that produced it. They exist +# so Regrade can re-run without asking for the photo again, which is worth +# most right after a grading-logic change — a value that decays fast. What +# doesn't decay is the grade record itself, so only the images are dropped +# here; the history row, its thumbnail and every measurement stay forever. +# A pruned card's Regrade falls back to asking for the photo, exactly as a +# card graded before images were stored does. +# Counted from created_at, which update_grade_result bumps — so a card you +# regraded yesterday keeps its photos for another week, rather than being +# pruned on the age of its first grading. +# 0 or negative disables pruning entirely. +SOURCE_IMAGE_RETENTION_DAYS = int( + os.environ.get("CARD_GRADER_IMAGE_RETENTION_DAYS", "7")) + def connect(): # timeout: how long to wait for a writer's lock before giving up. The @@ -256,6 +270,46 @@ def update_grade_result(grade_id, grade, thumbnail=None, source_images=None): return get_grade(grade_id) +def prune_source_images(days=None): + """Drop stored photos older than the retention window. Returns the count. + + Only source_images_json is cleared — the grade, its thumbnail and its + measurements are untouched, so history stays complete and only the + expensive part expires. + + VACUUM afterwards because clearing a column returns its pages to + SQLite's freelist without shrinking the file: without it the database + would keep every byte this is meant to reclaim, and the whole feature + would silently do nothing to disk usage. It rewrites the file, so it's + run only when something was actually pruned, and on its own connection + since VACUUM cannot execute inside a transaction. + """ + days = SOURCE_IMAGE_RETENTION_DAYS if days is None else days + if days <= 0: + return 0 + cutoff = time.strftime("%Y-%m-%dT%H:%M:%S", + time.localtime(time.time() - days * 86400)) + conn = connect() + try: + cur = conn.execute( + "UPDATE grades SET source_images_json = NULL " + "WHERE source_images_json IS NOT NULL AND created_at < ?", + (cutoff,), + ) + pruned = cur.rowcount or 0 + conn.commit() + finally: + conn.close() + + if pruned: + vac = sqlite3.connect(DB_PATH, timeout=60.0, isolation_level=None) + try: + vac.execute("VACUUM") + finally: + vac.close() + return pruned + + def get_grade_images(grade_id): """The original (bytes, filename) pairs for Regrade, or None if this grade never had them stored (a card graded before this feature existed,