ninja_hippo/n64-tracker
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial commit: N64 Tracker source from server (excludes Coverart images)

Browse source
This commit is contained in:
mattie726 2026-08-25 22:54:08 -07:00 • committed by Barely Removable
commit ccf87df36d
14 changed files with 10105 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

29
.claude/settings.local.json Normal file
View file

@ -0,0 +1,29 @@
{
"permissions": {
"allow": [
"Bash(python3 -c \"\nimport subprocess\n# Try using built-in tools to extract text from PDF\nresult = subprocess.run\\([''mdls'', ''/Users/matt/Desktop/VideoGameList/N64.pdf''], capture_output=True, text=True\\)\nprint\\(result.stdout[:500]\\)\n\")",
"Bash(pip3 install:*)",
"Bash(python3:*)",
"WebFetch(domain:gamesdb.launchbox-app.com)",
"Bash(docker:*)",
"Bash(curl:*)",
"Bash(open http://localhost:8080)",
"Bash(awk:*)",
"Bash(shuf)",
"Bash(/Users/matt/Desktop/VideoGameList/build_coverart_mapping.py:*)",
"Bash(/Users/matt/Desktop/VideoGameList/fix_coverart_mapping.py:*)",
"WebFetch(domain:www.pricecharting.com)",
"Bash(/tmp/scrape_prices.py:*)",
"Bash(/tmp/match_prices.py:*)",
"Bash(/tmp/fuzzy_prices.py:*)",
"Bash(echo:*)",
"WebFetch(domain:www.rarityguide.com)",
"Bash(pip install:*)",
"Bash(/tmp/scrape_rarity.py:*)",
"Bash(/tmp/match_rarity.py:*)",
"WebFetch(domain:www.dkoldies.com)",
"Bash(rsync:*)",
"Bash(ssh:*)"
]
}
}

5
.gitignore vendored Normal file
View file

@ -0,0 +1,5 @@
.env
data/
__pycache__/
*.pyc
Coverart/

21
Dockerfile Normal file
View file

@ -0,0 +1,21 @@
FROM python:3.11-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app.py .
COPY static/ static/
# Persistent data dir (games.json lives here via volume)
RUN mkdir -p /app/data
COPY games.json /app/data/games.json
# Coverart/ is mounted at runtime via docker-compose volume
RUN mkdir -p /app/coverart
EXPOSE 5000
CMD ["gunicorn", "--bind", "0.0.0.0:5000", "--workers", "2", "--timeout", "120", "app:app"]

BIN
N64.pdf Normal file
View file

Binary file not shown.

464
app.py Normal file
View file

@ -0,0 +1,464 @@
from flask import (Flask, jsonify, request, send_from_directory,
send_file, abort, session, redirect, url_for)
import json
import os
import requests
from bs4 import BeautifulSoup
import re
import hashlib
import secrets
import threading
from PIL import Image
# ── Config ────────────────────────────────────────────────────────────────
BASE_PREFIX = '/n64' # subpath for nginx reverse proxy
DATA_FILE = '/app/data/games.json'
SETTINGS_FILE= '/app/data/settings.json'
ART_BASE = '/app/coverart'
THUMB_DIR = '/app/data/thumbs' # persisted on the game-data volume
THUMB_WIDTH = 300 # px — enough for the card grid
app = Flask(__name__, static_folder='static')
app.secret_key = os.environ.get('SECRET_KEY', secrets.token_hex(32))
# ── Thumbnail generator ────────────────────────────────────────────────────
def _make_thumb(src_path, thumb_path):
"""Convert a large PNG to a small WebP thumbnail."""
try:
with Image.open(src_path) as img:
# Maintain aspect ratio, limit width to THUMB_WIDTH
w, h = img.size
ratio = THUMB_WIDTH / w
new_h = int(h * ratio)
img = img.convert('RGB')
img = img.resize((THUMB_WIDTH, new_h), Image.LANCZOS)
img.save(thumb_path, 'WEBP', quality=82, method=4)
except Exception as e:
print(f'Thumb error {src_path}: {e}')
def generate_thumbnails():
"""Walk ART_BASE and generate WebP thumbs for any PNG that needs one."""
os.makedirs(THUMB_DIR, exist_ok=True)
todo = []
for fname in os.listdir(ART_BASE):
if not fname.lower().endswith('.png'):
continue
src = os.path.join(ART_BASE, fname)
thumb = os.path.join(THUMB_DIR, os.path.splitext(fname)[0] + '.webp')
if not os.path.isfile(thumb):
todo.append((src, thumb))
if todo:
print(f'Generating {len(todo)} thumbnails...')
for src, thumb in todo:
_make_thumb(src, thumb)
print('Thumbnails done.')
# Run thumbnail generation in background so startup isn't blocked
threading.Thread(target=generate_thumbnails, daemon=True).start()
# ── Settings helpers ───────────────────────────────────────────────────────
DEFAULT_PASSWORD = os.environ.get('ADMIN_PASSWORD', 'n64admin')
def _hash(pw):
return hashlib.sha256(pw.encode()).hexdigest()
def load_settings():
if os.path.isfile(SETTINGS_FILE):
with open(SETTINGS_FILE) as f:
return json.load(f)
# First-run: persist hashed default
s = {'password_hash': _hash(DEFAULT_PASSWORD)}
save_settings(s)
return s
def save_settings(s):
with open(SETTINGS_FILE, 'w') as f:
json.dump(s, f, indent=2)
def check_password(pw):
return _hash(pw) == load_settings().get('password_hash', '')
def is_authed():
return session.get('authed') is True
def require_auth(f):
"""Decorator: returns 401 JSON if not authenticated."""
from functools import wraps
@wraps(f)
def wrapped(*args, **kwargs):
if not is_authed():
return jsonify({'error': 'Unauthorized'}), 401
return f(*args, **kwargs)
return wrapped
# ── Game data helpers ──────────────────────────────────────────────────────
def load_games():
with open(DATA_FILE, 'r') as f:
return json.load(f)
def save_games(games):
with open(DATA_FILE, 'w') as f:
json.dump(games, f, indent=2)
# ── Static / index ─────────────────────────────────────────────────────────
@app.route(BASE_PREFIX + '/')
@app.route(BASE_PREFIX)
def index():
return send_from_directory('static', 'index.html')
# ── Cover art ──────────────────────────────────────────────────────────────
@app.route(BASE_PREFIX + '/art/<path:art_path>')
def serve_art(art_path):
full = os.path.join(ART_BASE, art_path)
if not os.path.isfile(full):
abort(404)
# Serve pre-generated WebP thumbnail if available (much smaller)
base_name = os.path.splitext(art_path)[0]
thumb_path = os.path.join(THUMB_DIR, base_name + '.webp')
if os.path.isfile(thumb_path):
resp = send_file(thumb_path, mimetype='image/webp')
else:
# Thumb not ready yet — serve original and kick off generation
resp = send_file(full, mimetype='image/png')
threading.Thread(target=_make_thumb, args=(full, thumb_path), daemon=True).start()
# Cache aggressively — images never change
resp.headers['Cache-Control'] = 'public, max-age=31536000, immutable'
return resp
# ── Auth endpoints ─────────────────────────────────────────────────────────
@app.route(BASE_PREFIX + '/api/auth/login', methods=['POST'])
def login():
data = request.get_json(silent=True) or {}
pw = data.get('password', '')
if check_password(pw):
session['authed'] = True
return jsonify({'ok': True})
return jsonify({'ok': False, 'error': 'Wrong password'}), 401
@app.route(BASE_PREFIX + '/api/auth/logout', methods=['POST'])
def logout():
session.clear()
return jsonify({'ok': True})
@app.route(BASE_PREFIX + '/api/auth/status', methods=['GET'])
def auth_status():
return jsonify({'authed': is_authed()})
@app.route(BASE_PREFIX + '/api/auth/change-password', methods=['POST'])
@require_auth
def change_password():
data = request.get_json(silent=True) or {}
current = data.get('current', '')
new_pw = data.get('new', '')
if not check_password(current):
return jsonify({'ok': False, 'error': 'Current password incorrect'}), 400
if len(new_pw) < 4:
return jsonify({'ok': False, 'error': 'Password must be at least 4 characters'}), 400
s = load_settings()
s['password_hash'] = _hash(new_pw)
save_settings(s)
return jsonify({'ok': True})
# ── Public API ────────────────────────────────────────────────────────────
@app.route(BASE_PREFIX + '/api/games', methods=['GET'])
def get_games():
games = load_games()
query = request.args.get('q', '').lower().strip()
if query:
games = [g for g in games if query in g['title'].lower()]
return jsonify(games)
@app.route(BASE_PREFIX + '/api/games/owned', methods=['GET'])
def get_owned():
games = load_games()
return jsonify([g for g in games if g.get('owned')])
CASE_COST = 7.00
@app.route(BASE_PREFIX + '/api/stats', methods=['GET'])
def get_stats():
games = load_games()
total = len(games)
owned = sum(1 for g in games if g.get('owned'))
collection_value = sum(
g.get('loose_price') or 0
for g in games if g.get('owned') and g.get('loose_price')
)
unowned_games = [g for g in games if not g.get('owned')]
needs_cost = sum(
(g.get('loose_price') or 0) + CASE_COST
for g in unowned_games
)
return jsonify({
'total': total,
'owned': owned,
'unowned': total - owned,
'collection_value': round(collection_value, 2),
'needs_cost': round(needs_cost, 2)
})
# ── Protected write API ───────────────────────────────────────────────────
@app.route(BASE_PREFIX + '/api/games/<int:game_id>/own', methods=['POST'])
@require_auth
def toggle_own(game_id):
games = load_games()
for g in games:
if g['id'] == game_id:
g['owned'] = not g.get('owned', False)
save_games(games)
if g['owned'] and not g.get('cover_art_local'):
info = scrape_game_info(g['title'])
if info:
games = load_games()
for gg in games:
if gg['id'] == game_id:
gg.update(info)
break
save_games(games)
g.update(info)
return jsonify(g)
return jsonify({'error': 'Game not found'}), 404
@app.route(BASE_PREFIX + '/api/games/<int:game_id>/flags', methods=['POST'])
@require_auth
def update_flags(game_id):
games = load_games()
data = request.get_json(silent=True) or {}
for g in games:
if g['id'] == game_id:
for field in ('has_case', 'cleaned', 'battery_replaced'):
if field in data:
g[field] = bool(data[field])
save_games(games)
return jsonify(g)
return jsonify({'error': 'Game not found'}), 404
@app.route(BASE_PREFIX + '/api/games/<int:game_id>/info', methods=['GET'])
def get_game_info(game_id):
games = load_games()
for g in games:
if g['id'] == game_id:
if not g.get('cover_art_local') and not g.get('cover_art_remote'):
info = scrape_game_info(g['title'])
if info:
g.update(info)
save_games(games)
return jsonify(g)
return jsonify({'error': 'Game not found'}), 404
@app.route(BASE_PREFIX + '/api/rarity/refresh', methods=['POST'])
@require_auth
def refresh_rarity():
try:
rarity_data = scrape_rarityguide()
if not rarity_data:
return jsonify({'error': 'Scrape returned no data'}), 502
games = load_games()
updated = 0
for g in games:
match = _match_rarity(g['title'], rarity_data)
if match:
g['rarity_pct'] = match['rarity_pct']
g['rarity_label'] = match['rarity_label']
updated += 1
elif 'rarity_pct' not in g:
g['rarity_pct'] = None
g['rarity_label'] = None
save_games(games)
return jsonify({'updated': updated, 'total': len(games)})
except Exception as e:
print(f'Rarity refresh error: {e}')
return jsonify({'error': str(e)}), 500
@app.route(BASE_PREFIX + '/api/prices/refresh', methods=['POST'])
@require_auth
def refresh_prices():
try:
prices = scrape_pricecharting()
if not prices:
return jsonify({'error': 'Scrape returned no data'}), 502
games = load_games()
updated = 0
for g in games:
matched = _match_price(g['title'], prices)
if matched is not None:
g['loose_price'] = matched
updated += 1
elif 'loose_price' not in g:
g['loose_price'] = None
save_games(games)
collection_value = round(sum(
g.get('loose_price') or 0
for g in games if g.get('owned') and g.get('loose_price')
), 2)
return jsonify({'updated': updated, 'total': len(games), 'collection_value': collection_value})
except Exception as e:
print(f'Price refresh error: {e}')
return jsonify({'error': str(e)}), 500
# ── Scrapers ───────────────────────────────────────────────────────────────
def scrape_rarityguide():
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'}
base = 'https://www.rarityguide.com/n64_view.php'
results = {}
for first in [None, 101, 201]:
params = {'SortDirection': 'desc', 'SortField': '5', 'recordsPerPage': '100'}
if first:
params['FirstRecord'] = str(first)
try:
resp = requests.get(base, params=params, headers=headers, timeout=20)
if resp.status_code != 200:
continue
except Exception:
continue
soup = BeautifulSoup(resp.text, 'html.parser')
for row in soup.find_all('tr'):
tds = row.find_all('td')
if len(tds) < 5:
continue
title = tds[1].get_text(strip=True)
if not title:
continue
img = tds[4].find('img')
if not img:
continue
alt = img.get('alt', '')
m = re.match(r'(\d+)\s*percent\s*\(([^)]+)\)', alt)
if not m:
continue
results[title] = {'rarity_pct': int(m.group(1)), 'rarity_label': m.group(2).strip()}
return results if results else None
def _match_rarity(title, rarity_data):
import difflib, unicodedata
def norm(s):
s = unicodedata.normalize('NFD', s)
s = ''.join(c for c in s if unicodedata.category(c) != 'Mn')
s = s.lower()
s = re.sub(r'[^a-z0-9\s\-]', '', s)
s = re.sub(r'\s+', ' ', s).strip()
for art in ('the ', 'a ', 'an '):
if s.startswith(art): s = s[len(art):]
return s
tn = norm(title)
for k, v in rarity_data.items():
if norm(k) == tn: return v
keys = list(rarity_data.keys())
nkeys = [norm(k) for k in keys]
matches = difflib.get_close_matches(tn, nkeys, n=1, cutoff=0.85)
if matches:
return rarity_data[keys[nkeys.index(matches[0])]]
return None
def scrape_pricecharting():
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'}
base_url = 'https://www.pricecharting.com/console/nintendo-64'
prices = {}
session = requests.Session()
resp = session.get(base_url, headers=headers, timeout=20)
if resp.status_code != 200:
return None
def parse_page(html):
soup = BeautifulSoup(html, 'html.parser')
table = soup.find('table', id='games_table')
if not table: return soup, False
for row in table.find_all('tr'):
title_td = row.find('td', class_='title')
price_td = row.find('td', class_='used_price')
if not title_td or not price_td: continue
a = title_td.find('a')
if not a: continue
span = price_td.find('span', class_='js-price')
price_text = span.get_text(strip=True) if span else 'N/A'
try:
prices[a.get_text(strip=True)] = float(price_text.replace('$','').replace(',',''))
except (ValueError, AttributeError):
prices[a.get_text(strip=True)] = None
form = soup.find('form', {'id': re.compile(r'pagination')}) or soup.find('input', {'name': 'cursor'})
if form:
cursor_input = soup.find('input', {'name': 'cursor'})
return soup, cursor_input
return soup, None
soup, cursor_input = parse_page(resp.text)
while cursor_input:
cursor_val = cursor_input.get('value', '')
try:
resp = session.post(base_url, headers=headers, timeout=20,
data={'cursor': cursor_val, 'q': '', 'sort': 'name'})
if resp.status_code != 200: break
except Exception: break
_, cursor_input = parse_page(resp.text)
return prices
def _normalize(s):
s = s.lower()
s = re.sub(r'[éèê]', 'e', s); s = re.sub(r'[óò]', 'o', s)
s = re.sub(r'[úù]', 'u', s); s = re.sub(r'°', '', s)
s = re.sub(r'[^a-z0-9\s]', '', s)
s = re.sub(r'\s+', ' ', s).strip()
for art in ('the ', 'a ', 'an '):
if s.startswith(art): s = s[len(art):]
return s
def _match_price(title, prices):
if title in prices: return prices[title]
tl = title.lower()
for k, v in prices.items():
if k.lower() == tl: return v
tn = _normalize(title)
for k, v in prices.items():
if _normalize(k) == tn: return v
return None
def scrape_game_info(title):
try:
from urllib.parse import quote
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
resp = requests.get(f'https://gamesdb.launchbox-app.com/games/results/{quote(title)}',
headers=headers, timeout=15)
if resp.status_code != 200: return None
soup = BeautifulSoup(resp.text, 'html.parser')
title_lower = title.lower().strip()
clean_title = re.sub(r'[^a-z0-9\s]', '', title_lower).strip()
game_link = None
for link in soup.find_all('a', href=True):
href = link['href']
if '/games/details/' not in href: continue
link_text = link.get_text(strip=True)
if 'Nintendo 64' not in link_text or 'Released' not in link_text: continue
if re.sub(r'[^a-z0-9\s]', '', link_text.lower()).startswith(clean_title):
game_link = href; break
if not game_link: return None
resp2 = requests.get(f'https://gamesdb.launchbox-app.com{game_link}', headers=headers, timeout=15)
if resp2.status_code != 200: return None
soup2 = BeautifulSoup(resp2.text, 'html.parser')
info = {}
for img in soup2.find_all('img'):
if 'launchbox-app.com' in img.get('src','') and 'm-auto' in img.get('class',[]):
info['cover_art_remote'] = img['src']; break
for dt in soup2.find_all('dt'):
dd = dt.find_next_sibling('dd')
if not dd: continue
label = dt.get_text(strip=True)
value = dd.get_text(strip=True)
if label == 'Developers': info['developer'] = value[:100]
elif label == 'Genre': info['genres'] = value[:100]
elif label == 'ESRB': info['rating'] = value[:50]
elif label == 'Max Players': info['players'] = value[:50]
if not info.get('description'):
for p in soup2.find_all('p'):
text = p.get_text(strip=True)
if len(text) > 80:
info['description'] = text[:500]; break
return info if info else None
except Exception as e:
print(f"Scraping error for '{title}': {e}")
return None
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=True)

59
build_coverart_mapping.py Normal file
View file

@ -0,0 +1,59 @@
import json
import os
import re
COVERART_DIR = "/Users/matt/Desktop/VideoGameList/Coverart"
GAMES_JSON = "/Users/matt/Desktop/VideoGameList/games.json"
def title_to_filename(title):
"""Transform a game title to its expected Coverart filename."""
transformed = title.replace("'", "_").replace(":", "_")
return f"{transformed}-01.png"
def main():
# 1. Build a set of available Coverart filenames
coverart_files = set(os.listdir(COVERART_DIR))
print(f"Coverart files available: {len(coverart_files)}")
# 2. Load games.json
with open(GAMES_JSON, "r", encoding="utf-8") as f:
games = json.load(f)
print(f"Games in JSON: {len(games)}")
matched = []
unmatched = []
cleared = []
for game in games:
title = game.get("title", game.get("name", ""))
expected = title_to_filename(title)
current = game.get("cover_art_local", "")
if expected in coverart_files:
game["cover_art_local"] = f"coverart/{expected}"
matched.append(title)
else:
# Clear any stale metadata/ paths; leave truly empty ones alone
if current.startswith("metadata/"):
game["cover_art_local"] = ""
cleared.append(title)
unmatched.append(title)
# 3. Save updated games.json
with open(GAMES_JSON, "w", encoding="utf-8") as f:
json.dump(games, f, indent=2, ensure_ascii=False)
# 4. Report
print(f"\nMatched : {len(matched)}")
print(f"Unmatched : {len(unmatched)}")
print(f"Cleared (metadata/ paths removed): {len(cleared)}")
if unmatched:
print("\nUnmatched titles:")
for t in unmatched:
expected = title_to_filename(t)
print(f" Title : {t!r}")
print(f" Expected : {expected!r}")
if __name__ == "__main__":
main()

13
docker-compose.yml Normal file
View file

@ -0,0 +1,13 @@
services:
n64-tracker:
build: .
ports:
- "8585:5000"
volumes:
# Persistent game data (ownership state survives rebuilds)
- ./data:/app/data
# Local Coverart folder with cover art (read-only bind mount)
- ./Coverart:/app/coverart:ro
env_file: .env
restart: unless-stopped

187
fix_coverart_mapping.py Normal file
View file

@ -0,0 +1,187 @@
import json
import os
import re
import unicodedata
COVERART_DIR = "/Users/matt/Desktop/VideoGameList/Coverart"
GAMES_JSON = "/Users/matt/Desktop/VideoGameList/games.json"
# ---------------------------------------------------------------------------
# Manual overrides: games.json title -> coverart stem (without -01.png)
# ---------------------------------------------------------------------------
MANUAL_OVERRIDES = {
"1080° Snowboarding": "1080 Snowboarding",
"Aero Fighters Assault": "AeroFighters Assault",
"All Star Tennis '99": "All Star Tennis 99",
"All-Star Baseball '99": "All-Star Baseball 99",
"Bass Hunter 64": "In-Fisherman Bass Hunter 64",
"Beetle Adventure Racing": "Beetle Adventure Racing!",
"Bomberman 64 1997": "Bomberman 64",
"Bust-A-Move 3 DX": "Bust-A-Move _99",
"ClayFighter 63\u2153": "Clay Fighter 63 1_3",
"ClayFighter: Sculptor's Cut": "Clay Fighter_ Sculptor_s Cut",
"Daffy Duck Starring as Duck Dodgers": "Looney Tunes_ Duck Dodgers_ Starring Daffy Duck",
"Daikatana": "John Romero_s Daikatana",
"Disney's Tarzan": "Tarzan",
"Donald Duck: Goin Quackers": "Donald Duck_ Goin_ Quackers",
"Duke Nukem Zero Hour": "Duke Nukem_ Zero Hour",
"Extreme-G 2": "Extreme-G_ XG2",
"FIFA '99": "FIFA 99",
"FIFA 64": "FIFA Soccer 64",
"FIFA: Road to World Cup '98": "FIFA_ Road to World Cup 98",
"G.A.S.P!! Fighters' NEXTream": "Deadly Arts",
"Holy Magic Century": "Quest 64",
"Lego Racers": "LEGO Racers",
"Madden NFL '99": "Madden NFL 99",
"Michael Owens WLS 2000": "Mia Hamm Soccer 64",
"Ms. Pac-Man: Maze Madness": "Ms. Pac-Man Maze Madness",
"Mystical Ninja Starring Goemon 2": "Goemon_s Great Adventure",
"NASCAR '99": "Nascar 99",
"NBA In The Zone '98": "NBA in the Zone _98",
"NBA In The Zone 2000": "NBA in the Zone 2000",
"NBA Jam '99": "NBA Jam 99",
"NBA Live '99": "NBA Live 99",
"NBA Pro '99": "NBA Hangtime",
"NFL Quarterback Club '98": "NFL Quarterback Club 98",
"NFL Quarterback Club '99": "NFL Quarterback Club 99",
"NFL Quarterback Club 2001": "NFL QB Club 2001",
"NHL '99": "NHL 99",
"NHL Breakaway '98": "NHL Breakaway 98",
"NHL Breakaway '99": "NHL Breakaway 99",
"NHL Pro '99": "NHL Blades of Steel _99",
"Olympic Hockey '98": "Olympic Hockey 98",
"Operation WinBack": "WinBack_ Covert Operations",
"Pokémon Puzzle League": "Pokemon Puzzle League",
"Pokémon Snap": "Pokemon Snap",
"Pokémon Stadium": "Pokemon Stadium",
"Pokémon Stadium 2": "Pokemon Stadium 2",
"Power Rangers Lightspeed Rescue": "Power Rangers_ Lightspeed Rescue",
"Quake 64": "Quake",
"Rainbow Six": "Tom Clancy_s Rainbow Six",
"Rampage World Tour": "Rampage_ World Tour",
"Ridge Racer 64": "RR64_ Ridge Racer 64",
"Rugrats: Treasure Hunt": "Rugrats_ Scavenger Hunt",
"Rush 2: Extreme Racing": "Rush 2_ Extreme Racing USA",
"San Francisco Rush": "San Francisco Rush_ Extreme Racing",
"Shadowgate 64: Trials of the Four Towers": "ShadowGate 64_ Trials of the Four Towers",
"Star Wars Episode I: Racer": "Star Wars Episode I_ Racer",
"Star Wars: Episode I Battle for Naboo": "Star Wars_ Episode I_ Battle for Naboo",
"Starcraft 64": "StarCraft 64",
"Supercross 2000": "SuperCross 2000",
"Superman 64": "Superman",
"The Powerpuff Girls: Chemical X-Traction": "The Powerpuff Girls_ Chemical X-traction",
"Tony Hawk's Skateboarding": "Tony Hawk_s Pro Skater",
"Toy Story 2: Buzz Lightyear to the Rescue":"Toy Story 2_ Buzz Lightyear to the Rescue!",
"Twisted Edge: Extreme Snowboarding": "Twisted Edge Extreme Snowboarding",
"V-Rally Edition '99": "V-Rally Edition 99",
"WCW vs. nWo: World Tour": "WCW Vs. nWo_ World Tour",
"WCW/nWo Revenge": "WCW_nWo Revenge",
"WipeOut 64": "Wipeout 64",
"World Cup '98": "World Cup 98",
"Xena: Warrior Princess": "Xena_ Warrior Princess_ The Talisman of Fate",
}
# ---------------------------------------------------------------------------
# Normalisation helper
# ---------------------------------------------------------------------------
def normalize(s):
"""Strip accents, remove degree/punctuation, lowercase, collapse spaces."""
# Decompose unicode (é -> e + combining accent) then strip combining chars
s = unicodedata.normalize("NFD", s)
s = "".join(c for c in s if unicodedata.category(c) != "Mn")
# Remove degree symbol and vulgar fraction one-third
s = s.replace("°", "").replace("\u2153", "")
# Replace underscores (used in filenames instead of ' : etc.)
s = s.replace("_", " ")
# Remove punctuation / special chars; keep alphanumerics and spaces
s = re.sub(r"[^a-z0-9 ]", "", s.lower())
# Collapse whitespace
s = re.sub(r"\s+", "", s)
return s
# ---------------------------------------------------------------------------
# Build coverart lookup { stem -> filename_with_extension }
# ---------------------------------------------------------------------------
coverart_files = {} # stem (without -01.png) -> full filename
for fname in os.listdir(COVERART_DIR):
if fname.endswith("-01.png"):
stem = fname[:-len("-01.png")]
coverart_files[stem] = fname
# Normalised lookup: normalized_stem -> stem
norm_to_stem = {normalize(stem): stem for stem in coverart_files}
# ---------------------------------------------------------------------------
# Load games
# ---------------------------------------------------------------------------
with open(GAMES_JSON, "r", encoding="utf-8") as f:
games = json.load(f)
unmatched_before = [g for g in games if not g.get("cover_art_local")]
print(f"Games without cover_art_local before fix: {len(unmatched_before)}")
print()
matched = []
unmatched = []
for game in games:
if game.get("cover_art_local"):
continue # already set, skip
title = game.get("title", "")
# --- 1. Manual override ---
if title in MANUAL_OVERRIDES:
stem = MANUAL_OVERRIDES[title]
if stem in coverart_files:
game["cover_art_local"] = f"Coverart/{coverart_files[stem]}"
matched.append((title, coverart_files[stem], "manual override"))
continue
else:
print(f" WARNING: override stem not found in Coverart: {stem!r}")
# --- 2. Fuzzy / normalised match ---
norm_title = normalize(title)
if norm_title in norm_to_stem:
stem = norm_to_stem[norm_title]
game["cover_art_local"] = f"Coverart/{coverart_files[stem]}"
matched.append((title, coverart_files[stem], "fuzzy match"))
continue
# --- 3. Try stripping trailing year (4 digits) ---
title_no_year = re.sub(r"\s+\d{4}$", "", title).strip()
norm_no_year = normalize(title_no_year)
if norm_no_year != norm_title and norm_no_year in norm_to_stem:
stem = norm_to_stem[norm_no_year]
game["cover_art_local"] = f"Coverart/{coverart_files[stem]}"
matched.append((title, coverart_files[stem], "fuzzy match (no year)"))
continue
unmatched.append(title)
# ---------------------------------------------------------------------------
# Report
# ---------------------------------------------------------------------------
print("=" * 70)
print(f"MATCHED ({len(matched)}):")
print("=" * 70)
for title, fname, method in matched:
print(f" [{method}]")
print(f" Game: {title!r}")
print(f" File: {fname!r}")
print()
print("=" * 70)
print(f"STILL UNMATCHED ({len(unmatched)}):")
print("=" * 70)
for t in unmatched:
print(f" {t!r}")
# ---------------------------------------------------------------------------
# Save updated games.json
# ---------------------------------------------------------------------------
with open(GAMES_JSON, "w", encoding="utf-8") as f:
json.dump(games, f, ensure_ascii=False, indent=2)
print()
print(f"Saved updated games.json (matched {len(matched)}/{len(unmatched_before)} previously unset entries)")

5626
games.json Normal file
View file

File diff suppressed because it is too large Load diff

1493
n64_dkoldies.csv Normal file
View file

File diff suppressed because it is too large Load diff

5
requirements.txt Normal file
View file

@ -0,0 +1,5 @@
flask==3.0.0
requests==2.31.0
beautifulsoup4==4.12.2
gunicorn==21.2.0
Pillow==10.3.0

163
scrape_prices.py Normal file
View file

@ -0,0 +1,163 @@
import requests
from bs4 import BeautifulSoup
import json
import re
import time
BASE_URL = "https://www.pricecharting.com/console/nintendo-64"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Referer": "https://www.pricecharting.com/console/nintendo-64",
}
def parse_price(price_str):
if not price_str:
return None
cleaned = re.sub(r"[^\d.]", "", price_str.strip())
try:
return float(cleaned) if cleaned else None
except ValueError:
return None
def parse_table(soup):
"""Return list of (title, loose_price) tuples from games_table."""
table = soup.find("table", id="games_table")
if not table:
return []
tbody = table.find("tbody") or table
rows = tbody.find_all("tr")
games = []
for row in rows:
title_cell = row.find("td", class_=lambda c: c and "title" in c.split())
if not title_cell:
continue
a_tag = title_cell.find("a")
title = a_tag.get_text(strip=True) if a_tag else None
if not title:
continue
loose_cell = row.find("td", class_=lambda c: c and "used_price" in c.split())
if loose_cell:
span = loose_cell.find("span", class_="js-price")
price_text = span.get_text(strip=True) if span else loose_cell.get_text(strip=True)
else:
price_text = None
games.append((title, parse_price(price_text)))
return games
def get_cursor_form(soup):
"""Find the 'more results' form and extract its hidden fields + cursor value."""
for form in soup.find_all("form"):
submit = form.find("input", type="submit")
if submit and "more" in (submit.get("value") or "").lower():
fields = {}
for inp in form.find_all("input"):
name = inp.get("name")
value = inp.get("value", "")
if name:
fields[name] = value
return fields
return None
# ---- Initial GET ----
print("=" * 60)
print("N64 PRICE SCRAPER - PriceCharting.com")
print("=" * 60)
params = {"q": "", "sort": "name", "dir": "asc", "status": "", "minimum-rating": ""}
print(f"\nFetching initial page...")
r = requests.get(BASE_URL, params=params, headers=HEADERS, timeout=30)
print(f" Status: {r.status_code} | {len(r.text)} bytes")
soup = BeautifulSoup(r.text, "html.parser")
# Extract reported total
total_hint = None
for m in re.finditer(r"(\d+)\s*items", soup.get_text()):
total_hint = int(m.group(1))
print(f" Reported total items on page: {total_hint}")
all_results = {}
batch = parse_table(soup)
for title, price in batch:
all_results[title] = price
print(f" Parsed {len(batch)} games from initial page | Running total: {len(all_results)}")
# ---- Cursor-based pagination via POST ----
fetch_num = 1
while True:
cursor_fields = get_cursor_form(soup)
if not cursor_fields:
print("\n No 'more results' form found - done paginating.")
break
cursor = cursor_fields.get("cursor", "?")
fetch_num += 1
print(f"\nFetching batch {fetch_num} (cursor={cursor})...")
# POST with form fields (same URL)
post_headers = {**HEADERS, "Content-Type": "application/x-www-form-urlencoded"}
r = requests.post(BASE_URL, data=cursor_fields, headers=post_headers, timeout=30)
print(f" Status: {r.status_code} | {len(r.text)} bytes")
soup = BeautifulSoup(r.text, "html.parser")
batch = parse_table(soup)
if not batch:
print(" No games in this response - stopping.")
break
new_count = 0
for title, price in batch:
if title not in all_results:
all_results[title] = price
new_count += 1
print(f" Parsed {len(batch)} games ({new_count} new) | Running total: {len(all_results)}")
if new_count == 0:
print(" No new games - stopping.")
break
time.sleep(0.5)
# ---- Results ----
print(f"\n{'=' * 60}")
print(f"SCRAPE COMPLETE - {len(all_results)} unique games collected")
print(f"{'=' * 60}\n")
print("HTML STRUCTURE OF FIRST 3 ROWS (already shown above in detail)")
print("Key structure confirmed:")
print(" <tr data-product='ID'>")
print(" <td class='title'><a href='...'>GAME NAME</a></td>")
print(" <td class='price numeric used_price'><span class='js-price'>$XX.XX</span></td>")
print(" <td class='price numeric cib_price'>...</td>")
print(" <td class='price numeric new_price'>...</td>")
print(" </tr>\n")
print(f"{'=' * 60}")
print("FIRST 20 RESULTS:")
print(f"{'=' * 60}")
for i, (title, price) in enumerate(list(all_results.items())[:20]):
price_str = f"${price:.2f}" if price is not None else "N/A"
print(f"{i+1:3}. {title:<55} {price_str}")
# Save JSON
output_path = "/tmp/pc_prices.json"
with open(output_path, "w", encoding="utf-8") as f:
json.dump(all_results, f, indent=2, ensure_ascii=False)
prices_only = [p for p in all_results.values() if p is not None]
no_price = sum(1 for p in all_results.values() if p is None)
print(f"\n{'=' * 60}")
print("SUMMARY STATS")
print(f"{'=' * 60}")
print(f" Total games saved: {len(all_results)}")
print(f" Games with price data: {len(prices_only)}")
print(f" Games with N/A price: {no_price}")
if prices_only:
print(f" Cheapest game: ${min(prices_only):.2f}")
print(f" Most expensive game: ${max(prices_only):.2f}")
print(f" Average loose price: ${sum(prices_only)/len(prices_only):.2f}")
print(f"\nSaved to: {output_path}")

138
scrape_rarity.py Normal file
View file

@ -0,0 +1,138 @@
"""
scrape_rarity.py
Fetches N64 rarity data from rarityguide.com and saves to /tmp/rarity.json
"""
import requests
from bs4 import BeautifulSoup
import json
import re
import warnings
warnings.filterwarnings("ignore") # suppress urllib3/ssl noise
HEADERS = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120 Safari/537.36"
}
BASE_URL = "https://www.rarityguide.com/n64_view.php"
# ── helpers ────────────────────────────────────────────────────────────────────
def fetch_html(params):
resp = requests.get(BASE_URL, headers=HEADERS, params=params, timeout=30)
resp.raise_for_status()
return resp.text
def parse_rarity_alt(alt_text):
"""'92 percent (Ultra Rare)' -> (92.0, 'Ultra Rare')"""
if not alt_text:
return None, None
m_pct = re.search(r'(\d+(?:\.\d+)?)\s*percent', alt_text, re.IGNORECASE)
m_lbl = re.search(r'\(([^)]+)\)', alt_text)
if m_pct:
return float(m_pct.group(1)), (m_lbl.group(1).strip() if m_lbl else "")
return None, None
def extract_games(html):
"""Return dict of {title: {rarity_pct, rarity_label}} from one page."""
soup = BeautifulSoup(html, "html.parser")
pct_imgs = soup.find_all("img", src=re.compile(r"percent\d+\.gif"))
page_results = {}
for img in pct_imgs:
alt = img.get("alt", "")
pct, label = parse_rarity_alt(alt)
if pct is None:
continue
tr = img.find_parent("tr")
if not tr:
continue
tds = tr.find_all("td", recursive=False)
# Title is in td[1] (td[0] is a blank selector cell)
if len(tds) < 2:
continue
title = tds[1].get_text(strip=True)
if title:
page_results[title] = {"rarity_pct": pct, "rarity_label": label}
return page_results
def print_raw_rows(html, n=3):
"""Print raw HTML for the first n game rows."""
soup = BeautifulSoup(html, "html.parser")
pct_imgs = soup.find_all("img", src=re.compile(r"percent\d+\.gif"))
print(f"\n--- First {n} raw game <tr> rows ---")
for i, img in enumerate(pct_imgs[:n]):
tr = img.find_parent("tr")
print(f"\n[Data Row {i}]\n{tr}\n")
# ── main ───────────────────────────────────────────────────────────────────────
# Page 1 (FirstRecord omitted = 1)
print("Fetching page 1 (FirstRecord=1)...")
html1 = fetch_html({"SortDirection": "desc", "SortField": "5", "recordsPerPage": "100"})
with open("/tmp/rarity_raw.html", "w", encoding="utf-8") as f:
f.write(html1)
print(f"Saved raw HTML ({len(html1):,} bytes) to /tmp/rarity_raw.html")
# ── structure diagnostics ──────────────────────────────────────────────────────
soup_diag = BeautifulSoup(html1, "html.parser")
all_tables = soup_diag.find_all("table")
print(f"\nTotal <table> tags: {len(all_tables)}")
for i, t in enumerate(all_tables):
print(f" Table {i}: id={t.get('id')!r} class={t.get('class')!r} rows={len(t.find_all('tr'))}")
pct_imgs_diag = soup_diag.find_all("img", src=re.compile(r"percent\d+\.gif"))
print(f"\nPercent <img> tags found: {len(pct_imgs_diag)}")
print(f" (recordsPerPage=300 is ignored by server; cap is 100 per page)")
# Print first 3 raw data rows
print_raw_rows(html1, n=3)
# ── collect all pages ──────────────────────────────────────────────────────────
all_results = {}
all_results.update(extract_games(html1))
print(f"\nPage 1: {len(all_results)} games collected.")
# Pages 2 and 3 via FirstRecord offset
for first_record in [101, 201]:
print(f"Fetching page with FirstRecord={first_record}...")
phtml = fetch_html({
"FirstRecord": str(first_record),
"recordsPerPage": "100",
"SortDirection": "desc",
"SortField": "5"
})
page_games = extract_games(phtml)
new_games = {k: v for k, v in page_games.items() if k not in all_results}
all_results.update(new_games)
print(f" -> {len(page_games)} on this page, {len(new_games)} new. Running total: {len(all_results)}")
if len(new_games) == 0:
print(" No new games — stopping pagination.")
break
# ── save JSON ──────────────────────────────────────────────────────────────────
with open("/tmp/rarity.json", "w", encoding="utf-8") as f:
json.dump(all_results, f, indent=2, ensure_ascii=False)
print(f"\nSaved {len(all_results)} games to /tmp/rarity.json")
# ── final stats ───────────────────────────────────────────────────────────────
if all_results:
pcts = [v["rarity_pct"] for v in all_results.values()]
max_game = max(all_results, key=lambda k: all_results[k]["rarity_pct"])
min_game = min(all_results, key=lambda k: all_results[k]["rarity_pct"])
from collections import Counter
label_counts = Counter(v["rarity_label"] for v in all_results.values())
print(f"""
=== FINAL STATS ===
Total games : {len(all_results)}
Max rarity : {all_results[max_game]['rarity_pct']:.0f}% ({all_results[max_game]['rarity_label']}) -> {max_game}
Min rarity : {all_results[min_game]['rarity_pct']:.0f}% ({all_results[min_game]['rarity_label']}) -> {min_game}
Avg rarity : {sum(pcts)/len(pcts):.1f}%
Rarity label distribution:""")
for label, count in sorted(label_counts.items(), key=lambda x: -x[1]):
print(f" {label or '(unlabeled)':<20} {count:>4} games")

1902
static/index.html Normal file
View file

File diff suppressed because it is too large Load diff