Compare commits

..

2 Commits

Author SHA1 Message Date
Samuel Enocsson 31d80273b8 chore: address review feedback on predicted rating logging (#11)
- align diagnostic endpoint date windows with getNextPDGAUpdateDate
- pass pdgaNumber to initial target-rating calculation for log attribution
- avoid double log when lazy-recompute returns 0
- log full error object in admin endpoint catch
2026-05-25 22:02:42 +02:00
Samuel Enocsson 83ba20d428 chore: add diagnostic logging for predicted rating lifecycle (#11)
Add structured Pino logging across four hot-spots to make the
predicted rating data-loss visible in production logs:

- rating-calculator.js: log on calculation start, each 0-return early
  exit (no_rounds / no_rounds_after_date_filter / no_eligible_rounds),
  and on successful calculation with full algorithm branch metadata.
  Accepts optional context={pdgaNumber,callsite} to correlate logs.

- models/player.js: savePlayerToDB now logs before INSERT OR REPLACE,
  explicitly naming the derived columns that will be reset to NULL.
  savePredictedRatingToDB reads the old value first and emits a warn
  when a positive predicted_rating is overwritten with 0.

- services/player-service.js: structured debug/info/warn logs in
  getPlayerDataFromDB (lazy recompute trigger, failed recompute) and
  getPredictedRatingFromDB (no round history, 0 despite having rounds).

- routes/players.js: refresh-round-history route now logs each step
  (official_history_scrape, rounds_scrape_save, predicted_calc) with
  success/count fields; outer catch uses structured error log.
  refresh-all and refresh-player log their INSERT OR REPLACE trigger.

Also adds GET /api/admin/player-state/:pdgaNumber — a read-only
diagnostic endpoint returning raw DB columns plus round count and
date-window breakdowns.

Closes #11
2026-05-25 21:55:53 +02:00
12 changed files with 206 additions and 284 deletions
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "pdga-ratings", "name": "pdga-ratings",
"version": "1.4.2", "version": "1.4.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "pdga-ratings", "name": "pdga-ratings",
"version": "1.4.2", "version": "1.4.0",
"dependencies": { "dependencies": {
"ejs": "^4.0.1", "ejs": "^4.0.1",
"express": "^4.18.2", "express": "^4.18.2",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "pdga-ratings", "name": "pdga-ratings",
"version": "1.4.2", "version": "1.4.0",
"description": "PDGA rating scraper and display", "description": "PDGA rating scraper and display",
"main": "server.js", "main": "server.js",
"scripts": { "scripts": {
-32
View File
@@ -342,38 +342,6 @@
transform: rotate(180deg); transform: rotate(180deg);
} }
/* Refresh button: hidden by default, revealed only when the card is open.
Larger than the desktop icon to give a comfortable touch target (≥44px). */
.m-card__head .m-refresh-icon {
display: none;
}
.m-card.is-open .m-card__head .m-refresh-icon {
display: grid;
width: 44px;
height: 44px;
margin-left: 0;
font-size: 15px;
opacity: 0.7;
flex-shrink: 0;
}
.m-card.is-open .m-card__head .m-refresh-icon:active {
opacity: 1;
color: var(--accent);
}
/* Spin only the icon glyph, not the 44px button box — otherwise the button's
lingering touch-hover frame (background + border) rotates too, which looks odd. */
.m-card.is-open .m-card__head .m-refresh-icon.spinning {
animation: none;
}
.m-card.is-open .m-card__head .m-refresh-icon.spinning i {
display: inline-block;
animation: spin 0.8s linear infinite;
}
.m-card__body { .m-card__body {
display: grid; display: grid;
grid-template-columns: minmax(0, 1fr) auto; grid-template-columns: minmax(0, 1fr) auto;
+4 -10
View File
@@ -131,16 +131,10 @@ async function clearCache() {
// Refreshes both the current rating and the prediction in one click, then // Refreshes both the current rating and the prediction in one click, then
// re-swaps the table so every derived value (deltas, pills, sparkline) reflects // re-swaps the table so every derived value (deltas, pills, sparkline) reflects
// the new state. Cheaper than fine-grained DOM updates and guaranteed consistent // the new state. Cheaper than fine-grained DOM updates and guaranteed consistent
// because the server renders the truth. The mobile cards partial is included // because the server renders the truth.
// inside ratings-table, so swapping #ratings-table re-renders both views at once.
async function refreshPlayerData(pdgaNumber) { async function refreshPlayerData(pdgaNumber) {
// The desktop row exists in the DOM even on mobile (hidden via CSS), so spin const icon = document.querySelector(`#row-${pdgaNumber} .cell-actions .refresh-icon`);
// both possible icons; only the one visible in the active viewport is seen. if (icon) icon.classList.add('spinning');
const icons = [
document.querySelector(`#row-${pdgaNumber} .cell-actions .refresh-icon`),
document.querySelector(`#m-card-${pdgaNumber} .m-refresh-icon`)
].filter(Boolean);
icons.forEach(icon => icon.classList.add('spinning'));
try { try {
await Promise.allSettled([ await Promise.allSettled([
fetch(`/api/refresh-player/${pdgaNumber}`, { method: 'POST' }), fetch(`/api/refresh-player/${pdgaNumber}`, { method: 'POST' }),
@@ -150,7 +144,7 @@ async function refreshPlayerData(pdgaNumber) {
} catch (error) { } catch (error) {
console.error('Error refreshing player data:', error); console.error('Error refreshing player data:', error);
} finally { } finally {
icons.forEach(icon => icon.classList.remove('spinning')); if (icon) icon.classList.remove('spinning');
} }
} }
-10
View File
@@ -76,16 +76,6 @@ function initializeDatabase() {
else logger.info('Successfully added cutoff_rating column'); else logger.info('Successfully added cutoff_rating column');
}); });
} }
const hasPredictedCalculatedAt = columns.some(col => col.name === 'predicted_calculated_at');
if (!hasPredictedCalculatedAt) {
logger.info('Adding predicted_calculated_at column to players table...');
db.run(`ALTER TABLE players ADD COLUMN predicted_calculated_at DATETIME DEFAULT NULL`, (err) => {
if (err) logger.error('Error adding predicted_calculated_at column:', err.message);
else logger.info('Successfully added predicted_calculated_at column');
});
}
}); });
}); });
+27 -29
View File
@@ -1,5 +1,6 @@
const { db } = require('../db'); const { db } = require('../db');
const { parseDate } = require('../services/rating-calculator'); const { parseDate } = require('../services/rating-calculator');
const logger = require('../logger');
function getPlayerFromDB(pdgaNumber) { function getPlayerFromDB(pdgaNumber) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@@ -16,28 +17,14 @@ function getPlayerFromDB(pdgaNumber) {
function savePlayerToDB(playerData) { function savePlayerToDB(playerData) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
logger.info({ pdgaNumber: playerData.pdgaNumber, name: playerData.name, currentRating: playerData.rating, ratingChange: playerData.ratingChange, resetColumns: ['predicted_rating', 'std_dev', 'excluded_rounds_count', 'cutoff_rating', 'last_round_update'] }, 'INSERT OR REPLACE on players — derived columns will be reset to NULL');
db.run( db.run(
// UPSERT (not INSERT OR REPLACE): updating in place preserves columns not `INSERT OR REPLACE INTO players (pdga_number, name, current_rating, rating_change, last_updated)
// listed here — predicted_rating, std_dev, last_round_update, VALUES (?, ?, ?, ?, datetime('now'))`,
// excluded_rounds_count, cutoff_rating. INSERT OR REPLACE would delete the
// existing row and reset those to their DEFAULT (NULL).
`INSERT INTO players (pdga_number, name, current_rating, rating_change, last_updated)
VALUES (?, ?, ?, ?, datetime('now'))
ON CONFLICT(pdga_number) DO UPDATE SET
name = excluded.name,
current_rating = excluded.current_rating,
rating_change = excluded.rating_change,
last_updated = excluded.last_updated`,
[playerData.pdgaNumber, playerData.name, playerData.rating, playerData.ratingChange], [playerData.pdgaNumber, playerData.name, playerData.rating, playerData.ratingChange],
function(err) { function(err) {
if (err) return reject(err); if (err) reject(err);
// node-sqlite3 leaves lastID = 0 when ON CONFLICT triggers an UPDATE. else resolve(this.lastID);
// Fall back to a SELECT to get the real id in that case.
if (this.lastID !== 0) return resolve(this.lastID);
db.get('SELECT id FROM players WHERE pdga_number = ?', [playerData.pdgaNumber], (err2, row) => {
if (err2) reject(err2);
else resolve(row ? row.id : 0);
});
} }
); );
}); });
@@ -187,17 +174,28 @@ function saveRoundHistoryToDB(pdgaNumber, roundData, isIncremental = false) {
}); });
} }
function savePredictedRatingToDB(pdgaNumber, predictedRating, stdDev = null, excludedRoundsCount = null, cutoffRating = null, calculatedAt = null) { function savePredictedRatingToDB(pdgaNumber, predictedRating, stdDev = null, excludedRoundsCount = null, cutoffRating = null, callsite = 'unknown') {
const timestamp = calculatedAt || new Date().toISOString();
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
db.run( db.get('SELECT predicted_rating AS oldValue FROM players WHERE pdga_number = ?', [pdgaNumber], (selectErr, row) => {
'UPDATE players SET predicted_rating = ?, std_dev = ?, excluded_rounds_count = ?, cutoff_rating = ?, predicted_calculated_at = ? WHERE pdga_number = ?', if (selectErr) return reject(selectErr);
[predictedRating, stdDev, excludedRoundsCount, cutoffRating, timestamp, pdgaNumber], const oldValue = row ? row.oldValue : null;
function(err) {
if (err) reject(err); db.run(
else resolve(); 'UPDATE players SET predicted_rating = ?, std_dev = ?, excluded_rounds_count = ?, cutoff_rating = ? WHERE pdga_number = ?',
} [predictedRating, stdDev, excludedRoundsCount, cutoffRating, pdgaNumber],
); function(err) {
if (err) return reject(err);
const logData = { pdgaNumber, oldValue, newValue: predictedRating, callsite };
if (predictedRating === 0 && oldValue != null && oldValue > 0) {
logger.warn(logData, 'predicted rating overwritten with 0');
} else {
logger.info(logData, 'predicted rating saved');
}
resolve();
}
);
});
}); });
} }
+64 -27
View File
@@ -7,7 +7,7 @@ const { getOfficialRatingHistory, getOptimizedPlayerRounds } = require('../scrap
const { launchBrowser } = require('../scrapers/browser'); const { launchBrowser } = require('../scrapers/browser');
const { getPlayerDataFromDB, scrapePDGARating, getAllRatingsFromDB, refreshAllPlayersInDB, getPredictedRatingFromDB, formatDisplayDate } = require('../services/player-service'); const { getPlayerDataFromDB, scrapePDGARating, getAllRatingsFromDB, refreshAllPlayersInDB, getPredictedRatingFromDB, formatDisplayDate } = require('../services/player-service');
const { getTopbarLocals } = require('../services/topbar-service'); const { getTopbarLocals } = require('../services/topbar-service');
const { calculatePredictedRating } = require('../services/rating-calculator'); const { calculatePredictedRating, getNextPDGAUpdateDate } = require('../services/rating-calculator');
const { calculateRequiredAverage } = require('../services/target-rating-calculator'); const { calculateRequiredAverage } = require('../services/target-rating-calculator');
const logger = require('../logger'); const logger = require('../logger');
@@ -19,6 +19,7 @@ router.post('/api/refresh-all', async (req, res, next) => {
return res.status(409).json({ error: 'Refresh already in progress' }); return res.status(409).json({ error: 'Refresh already in progress' });
} }
refreshInProgress = true; refreshInProgress = true;
logger.info({ pdgaNumber: 'all' }, 'refresh-all triggered — will invoke savePlayerToDB for all players (INSERT OR REPLACE wipes derived columns)');
try { try {
try { try {
await refreshAllPlayersInDB(); await refreshAllPlayersInDB();
@@ -253,7 +254,7 @@ router.post('/api/add-player', async (req, res) => {
router.post('/api/refresh-player/:pdgaNumber', async (req, res) => { router.post('/api/refresh-player/:pdgaNumber', async (req, res) => {
try { try {
const { pdgaNumber } = req.params; const { pdgaNumber } = req.params;
logger.info(`Manually refreshing player data for PDGA ${pdgaNumber}`); logger.info({ pdgaNumber }, 'refresh-player triggered — will invoke savePlayerToDB (INSERT OR REPLACE wipes derived columns)');
const html = await fetchPlayerDataHTTP(pdgaNumber); const html = await fetchPlayerDataHTTP(pdgaNumber);
const playerData = parsePlayerData(html, pdgaNumber); const playerData = parsePlayerData(html, pdgaNumber);
@@ -349,48 +350,45 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
const isIncremental = !!sinceDate; const isIncremental = !!sinceDate;
logger.info(`${isIncremental ? 'Incrementally updating' : 'Fully refreshing'} round history for PDGA ${pdgaNumber}${sinceDate ? ` since ${sinceDate.toDateString()}` : ''}`); logger.info({ pdgaNumber, lastRoundUpdate, isIncremental }, 'refresh-round-history started');
browser = await launchBrowser(); browser = await launchBrowser();
let officialHistory; let officialHistory;
try { try {
officialHistory = await getOfficialRatingHistory(browser, pdgaNumber); officialHistory = await getOfficialRatingHistory(browser, pdgaNumber);
if (officialHistory.length > 0) { if (officialHistory.length > 0) {
await saveRatingHistoryToDB(pdgaNumber, officialHistory); await saveRatingHistoryToDB(pdgaNumber, officialHistory);
} }
logger.info({ pdgaNumber, step: 'official_history_scrape', success: officialHistory.length > 0, count: officialHistory.length }, 'official history scrape completed');
} catch (historyError) { } catch (historyError) {
logger.error('Failed to fetch official history:', historyError.message); logger.warn({ pdgaNumber, step: 'official_history_scrape', success: false, error: historyError.message }, 'official history scrape failed');
officialHistory = []; officialHistory = [];
} }
let allRounds = []; let allRounds = [];
try { try {
logger.info(`Using optimized approach: /details + new tournaments only for PDGA ${pdgaNumber}...`);
allRounds = await getOptimizedPlayerRounds(browser, pdgaNumber); allRounds = await getOptimizedPlayerRounds(browser, pdgaNumber);
if (allRounds.length > 0) { if (allRounds.length > 0) {
const roundsForDB = allRounds.map(round => ({ const roundsForDB = allRounds.map(round => ({
rating: round.rating, rating: round.rating,
date: round.date, date: round.date,
competition: round.competition competition: round.competition
})); }));
await saveRoundHistoryToDB(pdgaNumber, roundsForDB, false); await saveRoundHistoryToDB(pdgaNumber, roundsForDB, false);
logger.info(`✓ Saved ${allRounds.length} rounds using optimized approach`);
await updateLastRoundUpdateDate(pdgaNumber); await updateLastRoundUpdateDate(pdgaNumber);
} else {
logger.info(' No rounds found');
} }
logger.info({ pdgaNumber, step: 'rounds_scrape_save', success: allRounds.length > 0, count: allRounds.length, lastRoundUpdateTouched: allRounds.length > 0 }, 'rounds scrape and save completed');
} catch (detailsError) { } catch (detailsError) {
logger.error('Failed to fetch rounds using optimized approach:', detailsError.message); logger.warn({ pdgaNumber, step: 'rounds_scrape_save', success: false, error: detailsError.message }, 'rounds scrape and save failed');
allRounds = []; allRounds = [];
} }
await browser.close(); await browser.close();
browser = null; browser = null;
const dbRounds = await getRoundHistoryFromDB(pdgaNumber); const dbRounds = await getRoundHistoryFromDB(pdgaNumber);
const roundsForPrediction = dbRounds.map(round => ({ const roundsForPrediction = dbRounds.map(round => ({
rating: round.rating, rating: round.rating,
@@ -398,9 +396,10 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
competition: round.competition_name competition: round.competition_name
})); }));
const result = calculatePredictedRating(roundsForPrediction); const result = calculatePredictedRating(roundsForPrediction, { pdgaNumber, callsite: 'refresh-round-history' });
logger.info({ pdgaNumber, step: 'predicted_calc', rating: result.rating, stdDev: result.stdDev, excludedRoundsCount: result.excludedRoundsCount }, 'predicted rating calculation step completed');
await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating); await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating, 'refresh-round-history');
const officialCount = allRounds.filter(r => r.source === 'official').length; const officialCount = allRounds.filter(r => r.source === 'official').length;
const newCount = allRounds.filter(r => r.source === 'new').length; const newCount = allRounds.filter(r => r.source === 'new').length;
@@ -418,10 +417,8 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
message: `Used /details (${officialCount} rounds) + new tournaments (${newCount} rounds)` message: `Used /details (${officialCount} rounds) + new tournaments (${newCount} rounds)`
}); });
} catch (error) { } catch (error) {
logger.error(`=== Error refreshing round history for PDGA ${pdgaNumber} ===`); logger.error({ pdgaNumber, step: 'unknown_or_orchestration', errorType: error.constructor.name, errorMessage: error.message }, 'refresh-round-history failed');
logger.error('Error type:', error.constructor.name);
logger.error('Error message:', error.message);
if (browser) { if (browser) {
try { try {
await browser.close(); await browser.close();
@@ -429,14 +426,14 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
logger.error('Error closing browser:', closeError.message); logger.error('Error closing browser:', closeError.message);
} }
} }
res.status(500).json({ res.status(500).json({
error: 'Failed to refresh round history', error: 'Failed to refresh round history',
details: error.message, details: error.message,
errorType: error.constructor.name, errorType: error.constructor.name,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
suggestion: error.message.includes('socket hang up') ? suggestion: error.message.includes('socket hang up') ?
'Rate limited by PDGA - try again in a few minutes.' : 'Rate limited by PDGA - try again in a few minutes.' :
error.message.includes('timeout') ? error.message.includes('timeout') ?
'PDGA pages are loading slowly - try again later.' : 'PDGA pages are loading slowly - try again later.' :
'Tournament scraping failed - check server logs for details' 'Tournament scraping failed - check server logs for details'
@@ -444,6 +441,46 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
} }
}); });
router.get('/api/admin/player-state/:pdgaNumber', async (req, res) => {
const { pdgaNumber } = req.params;
try {
const player = await getPlayerFromDB(pdgaNumber);
if (!player) {
return res.status(404).json({ error: 'Player not found', pdgaNumber: parseInt(pdgaNumber) });
}
const rounds = await getRoundHistoryFromDB(pdgaNumber);
const cutoff = getNextPDGAUpdateDate();
const twelveMonthsAgo = new Date(cutoff); twelveMonthsAgo.setFullYear(cutoff.getFullYear() - 1);
const twentyFourMonthsAgo = new Date(cutoff); twentyFourMonthsAgo.setFullYear(cutoff.getFullYear() - 2);
const roundsInLast12mo = rounds.filter(r => new Date(r.date) >= twelveMonthsAgo).length;
const roundsInLast24mo = rounds.filter(r => new Date(r.date) >= twentyFourMonthsAgo).length;
const dates = rounds.map(r => r.date).sort();
res.json({
pdgaNumber: player.pdga_number,
name: player.name,
currentRating: player.current_rating,
ratingChange: player.rating_change,
predictedRating: player.predicted_rating,
stdDev: player.std_dev,
excludedRoundsCount: player.excluded_rounds_count,
cutoffRating: player.cutoff_rating,
lastUpdated: player.last_updated,
lastRoundUpdate: player.last_round_update,
cutoffDate: cutoff.toISOString(),
roundCount: rounds.length,
roundsInLast12mo,
roundsInLast24mo,
oldestRound: dates[0] ?? null,
newestRound: dates[dates.length - 1] ?? null
});
} catch (err) {
logger.error({ err, pdgaNumber }, 'admin player-state endpoint failed');
res.status(500).json({ error: 'Failed to fetch player state', details: err.message });
}
});
router.post('/api/calculate-target-rating/:pdgaNumber', async (req, res) => { router.post('/api/calculate-target-rating/:pdgaNumber', async (req, res) => {
const { pdgaNumber } = req.params; const { pdgaNumber } = req.params;
const pdgaNum = parseInt(pdgaNumber, 10); const pdgaNum = parseInt(pdgaNumber, 10);
@@ -483,7 +520,7 @@ router.post('/api/calculate-target-rating/:pdgaNumber', async (req, res) => {
competition: r.competition_name competition: r.competition_name
})); }));
const result = calculateRequiredAverage(roundRatings, target, numRounds); const result = calculateRequiredAverage(roundRatings, target, numRounds, pdgaNum);
logger.info(`Target rating calc for PDGA ${pdgaNum}: target=${target} rounds=${numRounds} -> avg=${result.requiredAverage}`); logger.info(`Target rating calc for PDGA ${pdgaNum}: target=${target} rounds=${numRounds} -> avg=${result.requiredAverage}`);
+29 -102
View File
@@ -156,154 +156,81 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) {
logger.info(`Looking for tournaments after ${afterDate.toDateString()}...`); logger.info(`Looking for tournaments after ${afterDate.toDateString()}...`);
const { urls: newTournamentUrls, counts } = await page.evaluate((afterTimestamp) => { const newTournamentUrls = await page.evaluate((afterTimestamp) => {
const afterDate = new Date(afterTimestamp); const afterDate = new Date(afterTimestamp);
const tables = document.querySelectorAll('table[id*="player-results"]'); const tables = document.querySelectorAll('table[id*="player-results"]');
const urls = []; const urls = [];
const seenUrls = new Set();
let table = 0; tables.forEach(table => {
let recentEvents = 0; const rows = table.querySelectorAll('tbody tr');
let recentEventsAnchorsSeen = 0;
let recentEventsSkippedDuplicates = 0;
tables.forEach(tbl => {
const rows = tbl.querySelectorAll('tbody tr');
rows.forEach(row => { rows.forEach(row => {
const dateCell = row.querySelector('.dates'); const dateCell = row.querySelector('.dates');
const tournamentCell = row.querySelector('.tournament a'); const tournamentCell = row.querySelector('.tournament a');
if (dateCell && tournamentCell) { if (dateCell && tournamentCell) {
const dateText = dateCell.innerText.trim(); const dateText = dateCell.innerText.trim();
const dateMatch = dateText.match(/\d{1,2}-[A-Za-z]{3}-\d{4}/); const dateMatch = dateText.match(/\d{1,2}-[A-Za-z]{3}-\d{4}/);
if (dateMatch) { if (dateMatch) {
const dateStr = dateMatch[0]; const dateStr = dateMatch[0];
const date = new Date(dateStr); const date = new Date(dateStr);
if (date > afterDate) { if (date > afterDate) {
const href = tournamentCell.getAttribute('href'); const href = tournamentCell.getAttribute('href');
if (href) { if (href) {
const absoluteUrl = new URL(href, location.origin).href; urls.push({
if (!seenUrls.has(absoluteUrl)) { url: `https://www.pdga.com${href}`,
seenUrls.add(absoluteUrl); date: dateStr,
urls.push({ name: tournamentCell.innerText.trim()
url: absoluteUrl, });
date: dateStr,
name: tournamentCell.innerText.trim(),
source: 'table'
});
table++;
}
} }
} }
} }
} }
}); });
}); });
const recentAnchors = document.querySelectorAll('.recent-events a[href*="/tour/event/"]'); return urls;
recentAnchors.forEach(anchor => {
recentEventsAnchorsSeen++;
const href = anchor.getAttribute('href');
if (href) {
const absoluteUrl = new URL(href, location.origin).href;
if (seenUrls.has(absoluteUrl)) {
recentEventsSkippedDuplicates++;
} else {
seenUrls.add(absoluteUrl);
urls.push({
url: absoluteUrl,
date: null,
name: anchor.innerText.trim() || 'Recent event',
source: 'recent-events'
});
recentEvents++;
}
}
});
return { urls, counts: { table, recentEvents, recentEventsAnchorsSeen, recentEventsSkippedDuplicates } };
}, afterDate.getTime()); }, afterDate.getTime());
logger.info({ logger.info(`Found ${newTournamentUrls.length} new tournaments after ${afterDate.toDateString()}`);
pdgaNumber,
afterDate: afterDate.toISOString(),
tableMatches: counts.table,
recentEventsMatches: counts.recentEvents,
recentEventsAnchorsSeen: counts.recentEventsAnchorsSeen,
recentEventsSkippedDuplicates: counts.recentEventsSkippedDuplicates,
totalUrlsToScrape: newTournamentUrls.length
}, 'new tournament URL discovery completed');
for (const tournamentData of newTournamentUrls) { for (const tournamentData of newTournamentUrls) {
try { try {
if (tournamentData.source === 'recent-events') { logger.info(`Scraping new tournament: ${tournamentData.name} (${tournamentData.date})`);
logger.debug({ pdgaNumber, url: tournamentData.url }, 'recent-events: scraping tournament');
} else {
logger.info(`Scraping new tournament: ${tournamentData.name} (${tournamentData.date})`);
}
await page.goto(tournamentData.url, { waitUntil: 'domcontentloaded', timeout: 30000 }); await page.goto(tournamentData.url, { waitUntil: 'domcontentloaded', timeout: 30000 });
await new Promise(r => setTimeout(r, 500)); await new Promise(r => setTimeout(r, 500));
let parsedDate;
if (tournamentData.date !== null) {
parsedDate = parseDate(tournamentData.date);
} else {
const eventDateStr = await page.evaluate(() => {
const body = document.body ? document.body.innerText : '';
const m = body.match(/\d{1,2}\s+to\s+\d{1,2}-[A-Za-z]{3}-\d{4}/)
|| body.match(/\d{1,2}-[A-Za-z]{3}-\d{4}/);
return m ? m[0] : null;
});
if (eventDateStr) {
parsedDate = parseDate(eventDateStr);
if (!(parsedDate > afterDate)) {
logger.warn({
pdgaNumber,
url: tournamentData.url,
eventDateStr,
parsedDate: parsedDate ? parsedDate.toISOString() : null,
afterDate: afterDate.toISOString()
}, 'recent-events: extracted event date is not newer than afterDate, likely captured a non-tournament date — skipping');
continue;
}
logger.debug({ pdgaNumber, url: tournamentData.url, eventDateStr }, 'recent-events: extracted date from event page');
} else {
logger.warn({ pdgaNumber, url: tournamentData.url }, 'recent-events: could not extract date from event page, skipping tournament');
continue;
}
}
const roundRatings = await page.evaluate((pdgaNum) => { const roundRatings = await page.evaluate((pdgaNum) => {
const rows = document.querySelectorAll('tr'); const rows = document.querySelectorAll('tr');
for (const row of rows) { for (const row of rows) {
const cells = row.querySelectorAll('td'); const cells = row.querySelectorAll('td');
const hasPlayerNumber = Array.from(cells).some(cell => const hasPlayerNumber = Array.from(cells).some(cell =>
cell.innerText && cell.innerText.includes(pdgaNum.toString()) cell.innerText && cell.innerText.includes(pdgaNum.toString())
); );
if (hasPlayerNumber) { if (hasPlayerNumber) {
const roundRatingCells = row.querySelectorAll('td.round-rating'); const roundRatingCells = row.querySelectorAll('td.round-rating');
const ratings = []; const ratings = [];
roundRatingCells.forEach(cell => { roundRatingCells.forEach(cell => {
const rating = parseInt(cell.innerText.trim()); const rating = parseInt(cell.innerText.trim());
if (!isNaN(rating) && rating > 0) { if (!isNaN(rating) && rating > 0) {
ratings.push(rating); ratings.push(rating);
} }
}); });
return ratings; return ratings;
} }
} }
return []; return [];
}, pdgaNumber); }, pdgaNumber);
if (roundRatings.length > 0) { if (roundRatings.length > 0) {
const parsedDate = parseDate(tournamentData.date);
roundRatings.forEach(rating => { roundRatings.forEach(rating => {
newRounds.push({ newRounds.push({
rating, rating,
@@ -311,10 +238,10 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) {
competition: tournamentData.name competition: tournamentData.name
}); });
}); });
logger.info(`Found ${roundRatings.length} round ratings for ${tournamentData.name}`); logger.info(`Found ${roundRatings.length} round ratings for ${tournamentData.name}`);
} }
} catch (error) { } catch (error) {
logger.error(`Error scraping tournament ${tournamentData.name}: ${error.message}`); logger.error(`Error scraping tournament ${tournamentData.name}: ${error.message}`);
} }
+33 -38
View File
@@ -1,7 +1,7 @@
const { db } = require('../db'); const { db } = require('../db');
const { getPlayerFromDB, getRoundHistoryFromDB, savePredictedRatingToDB, savePlayerToDB, getMonthlyHistory, getAllMonthlyHistoriesFromDB, getAllRatingHistoriesFromDB } = require('../models/player'); const { getPlayerFromDB, getRoundHistoryFromDB, savePredictedRatingToDB, savePlayerToDB, getMonthlyHistory, getAllMonthlyHistoriesFromDB, getAllRatingHistoriesFromDB } = require('../models/player');
const { fetchPlayerDataHTTP, parsePlayerData } = require('../scrapers/player-http'); const { fetchPlayerDataHTTP, parsePlayerData } = require('../scrapers/player-http');
const { calculatePredictedRating, getPreviousPDGAUpdateDate } = require('./rating-calculator'); const { calculatePredictedRating } = require('./rating-calculator');
const logger = require('../logger'); const logger = require('../logger');
function formatDisplayDate(dateStr) { function formatDisplayDate(dateStr) {
@@ -36,43 +36,32 @@ async function getPlayerDataFromDB(pdgaNumber, { includeMonthlyHistory = true }
try { try {
const cachedPlayer = await getPlayerFromDB(pdgaNumber); const cachedPlayer = await getPlayerFromDB(pdgaNumber);
if (cachedPlayer) { if (cachedPlayer) {
logger.debug(`Loading PDGA ${pdgaNumber} from DB (source of truth)`); logger.debug({ pdgaNumber }, 'Loading player from DB (source of truth)');
let predictedRating = cachedPlayer.predicted_rating; let predictedRating = cachedPlayer.predicted_rating;
let stdDev = cachedPlayer.std_dev; let stdDev = cachedPlayer.std_dev;
let excludedRoundsCount = cachedPlayer.excluded_rounds_count; let excludedRoundsCount = cachedPlayer.excluded_rounds_count;
let cutoffRating = cachedPlayer.cutoff_rating; let cutoffRating = cachedPlayer.cutoff_rating;
let predictedCalculatedAtRaw = cachedPlayer.predicted_calculated_at; let recomputeAttempted = false;
if (!predictedRating || predictedRating === 0) { if (!predictedRating || predictedRating === 0) {
recomputeAttempted = true;
logger.debug({ pdgaNumber, dbValue: cachedPlayer.predicted_rating }, 'lazy recompute triggered for predicted_rating');
predictedRating = await getPredictedRatingFromDB(pdgaNumber); predictedRating = await getPredictedRatingFromDB(pdgaNumber);
const updatedPlayer = await getPlayerFromDB(pdgaNumber); const updatedPlayer = await getPlayerFromDB(pdgaNumber);
stdDev = updatedPlayer?.std_dev; stdDev = updatedPlayer?.std_dev;
excludedRoundsCount = updatedPlayer?.excluded_rounds_count; excludedRoundsCount = updatedPlayer?.excluded_rounds_count;
cutoffRating = updatedPlayer?.cutoff_rating; cutoffRating = updatedPlayer?.cutoff_rating;
predictedCalculatedAtRaw = updatedPlayer?.predicted_calculated_at; if (!predictedRating || predictedRating === 0) {
} logger.info({ pdgaNumber, recomputedValue: predictedRating }, 'lazy recompute did not yield a positive predicted rating');
}
// Staleness-check: invalidate cached predicted_rating if the PDGA cycle has
// rolled over since it was calculated. Don't recompute — round_history may be
// equally stale. UI will show "—" until the next manual refresh.
const predictedCalculatedAt = predictedCalculatedAtRaw
? new Date(predictedCalculatedAtRaw)
: null;
const previousUpdate = getPreviousPDGAUpdateDate();
const hasPredicted = predictedRating !== null && predictedRating !== 0;
const isStale = hasPredicted && (
predictedCalculatedAt === null || predictedCalculatedAt < previousUpdate
);
if (isStale) {
logger.debug(`PDGA ${pdgaNumber}: predicted rating stale (calculated ${predictedCalculatedAt?.toISOString() ?? 'unknown'}, cycle rolled ${previousUpdate.toISOString()})`);
predictedRating = null;
stdDev = null;
excludedRoundsCount = null;
cutoffRating = null;
} }
const rating = cachedPlayer.current_rating; const rating = cachedPlayer.current_rating;
const rawRatingChange = cachedPlayer.rating_change; const rawRatingChange = cachedPlayer.rating_change;
// Only warn about ≤0 if it wasn't already explained by a failed recompute (which has its own log)
if (!recomputeAttempted && predictedRating != null && predictedRating <= 0) {
logger.warn({ pdgaNumber, dbValue: predictedRating }, 'predicted rating present but <= 0 in DB without recompute — rendered as empty');
}
const resolvedPredicted = predictedRating > 0 ? predictedRating : null; const resolvedPredicted = predictedRating > 0 ? predictedRating : null;
const resolvedStdDev = stdDev > 0 ? stdDev : null; const resolvedStdDev = stdDev > 0 ? stdDev : null;
@@ -161,22 +150,28 @@ async function scrapePDGARating(pdgaNumber, retries = 3) {
async function getPredictedRatingFromDB(pdgaNumber) { async function getPredictedRatingFromDB(pdgaNumber) {
try { try {
const roundHistory = await getRoundHistoryFromDB(pdgaNumber); const roundHistory = await getRoundHistoryFromDB(pdgaNumber);
if (roundHistory.length > 0) { if (roundHistory.length === 0) {
logger.debug(`Using ${roundHistory.length} cached rounds for PDGA ${pdgaNumber} prediction`); logger.info({ pdgaNumber, reason: 'no_round_history_in_db' }, 'predicted recompute returning 0 — no round history available');
return 0;
const roundRatings = roundHistory.map(round => ({
rating: round.rating,
date: new Date(round.date),
competition: round.competition_name || 'Unknown'
}));
const result = calculatePredictedRating(roundRatings);
await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating);
return result.rating;
} }
return 0;
logger.debug({ pdgaNumber, cachedRounds: roundHistory.length }, 'Using cached rounds for prediction');
const roundRatings = roundHistory.map(round => ({
rating: round.rating,
date: new Date(round.date),
competition: round.competition_name || 'Unknown'
}));
const result = calculatePredictedRating(roundRatings, { pdgaNumber, callsite: 'getPredictedRatingFromDB' });
if (result.rating === 0) {
logger.warn({ pdgaNumber, roundsInDb: roundHistory.length }, 'predicted recompute returned 0 despite having rounds in DB');
}
await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating, 'lazy-recompute');
return result.rating;
} catch (err) { } catch (err) {
logger.error(`Error getting predicted rating from DB for ${pdgaNumber}:`, err.message); logger.error(`Error getting predicted rating from DB for ${pdgaNumber}:`, err.message);
return 0; return 0;
+44 -26
View File
@@ -1,3 +1,5 @@
const logger = require('../logger');
function parseDate(dateStr) { function parseDate(dateStr) {
const multiDayMatch = dateStr.match(/^(\d{1,2})(-([A-Za-z]{3}))?(\s+to\s+)(\d{1,2})-([A-Za-z]{3})-(\d{4})$/); const multiDayMatch = dateStr.match(/^(\d{1,2})(-([A-Za-z]{3}))?(\s+to\s+)(\d{1,2})-([A-Za-z]{3})-(\d{4})$/);
if (multiDayMatch) { if (multiDayMatch) {
@@ -37,41 +39,37 @@ function parseDate(dateStr) {
return new Date(dateStr); return new Date(dateStr);
} }
function secondTuesdayOf(year, month) {
const firstDay = new Date(year, month, 1);
const daysUntilTuesday = (2 - firstDay.getDay() + 7) % 7;
const firstTuesday = new Date(year, month, 1 + daysUntilTuesday);
const secondTuesday = new Date(firstTuesday);
secondTuesday.setDate(firstTuesday.getDate() + 7);
return secondTuesday;
}
function getNextPDGAUpdateDate() { function getNextPDGAUpdateDate() {
const today = new Date(); const today = new Date();
const currentMonth = today.getMonth(); const currentMonth = today.getMonth();
const currentYear = today.getFullYear(); const currentYear = today.getFullYear();
const secondTuesday = secondTuesdayOf(currentYear, currentMonth); const firstDayOfMonth = new Date(currentYear, currentMonth, 1);
const firstTuesday = new Date(firstDayOfMonth);
const daysUntilTuesday = (2 - firstDayOfMonth.getDay() + 7) % 7;
firstTuesday.setDate(1 + daysUntilTuesday);
const secondTuesday = new Date(firstTuesday);
secondTuesday.setDate(firstTuesday.getDate() + 7);
if (today <= secondTuesday) { if (today <= secondTuesday) {
return secondTuesday; return secondTuesday;
} else { } else {
const nextMonth = currentMonth === 11 ? 0 : currentMonth + 1; const nextMonth = currentMonth === 11 ? 0 : currentMonth + 1;
const nextYear = currentMonth === 11 ? currentYear + 1 : currentYear; const nextYear = currentMonth === 11 ? currentYear + 1 : currentYear;
return secondTuesdayOf(nextYear, nextMonth);
}
}
function getPreviousPDGAUpdateDate() { const firstDayNextMonth = new Date(nextYear, nextMonth, 1);
const today = new Date(); const firstTuesdayNext = new Date(firstDayNextMonth);
const year = today.getFullYear();
const month = today.getMonth(); const daysUntilTuesdayNext = (2 - firstDayNextMonth.getDay() + 7) % 7;
const secondTuesday = secondTuesdayOf(year, month); firstTuesdayNext.setDate(1 + daysUntilTuesdayNext);
if (today > secondTuesday) return secondTuesday;
// Otherwise: last month's second Tuesday const secondTuesdayNext = new Date(firstTuesdayNext);
const prevMonth = month === 0 ? 11 : month - 1; secondTuesdayNext.setDate(firstTuesdayNext.getDate() + 7);
const prevYear = month === 0 ? year - 1 : year;
return secondTuesdayOf(prevYear, prevMonth); return secondTuesdayNext;
}
} }
function calculateStandardDeviation(ratings) { function calculateStandardDeviation(ratings) {
@@ -83,15 +81,19 @@ function calculateStandardDeviation(ratings) {
return Math.sqrt(variance); return Math.sqrt(variance);
} }
function calculatePredictedRating(roundRatings) { function calculatePredictedRating(roundRatings, context = {}) {
const pdgaNumber = context.pdgaNumber ?? null;
const callsite = context.callsite ?? 'unknown';
const debugLog = []; const debugLog = [];
debugLog.push('=== PDGA RATING CALCULATION (Following Official Rules) ==='); debugLog.push('=== PDGA RATING CALCULATION (Following Official Rules) ===');
if (!roundRatings || roundRatings.length === 0) { if (!roundRatings || roundRatings.length === 0) {
debugLog.push('❌ No rounds provided for prediction'); debugLog.push('❌ No rounds provided for prediction');
logger.warn({ pdgaNumber, callsite, reason: 'no_rounds' }, 'predicted rating computed as 0');
return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null }; return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null };
} }
logger.info({ pdgaNumber, callsite, inputRounds: roundRatings.length }, 'predicted rating calculation started');
debugLog.push(`📊 Starting with ${roundRatings.length} total rounds`); debugLog.push(`📊 Starting with ${roundRatings.length} total rounds`);
const nextUpdateDate = getNextPDGAUpdateDate(); const nextUpdateDate = getNextPDGAUpdateDate();
@@ -104,6 +106,7 @@ function calculatePredictedRating(roundRatings) {
if (allSortedRounds.length === 0) { if (allSortedRounds.length === 0) {
debugLog.push('❌ No valid rounds after filtering for update date'); debugLog.push('❌ No valid rounds after filtering for update date');
logger.warn({ pdgaNumber, callsite, reason: 'no_rounds_after_date_filter', inputRounds: roundRatings.length, nextUpdateDate: nextUpdateDate.toISOString() }, 'predicted rating computed as 0');
return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null }; return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null };
} }
@@ -121,8 +124,9 @@ function calculatePredictedRating(roundRatings) {
debugLog.push('🗓️ 12-MONTH FILTERING:'); debugLog.push('🗓️ 12-MONTH FILTERING:');
debugLog.push(`✅ Rounds in last 12 months: ${eligibleRounds.length}`); debugLog.push(`✅ Rounds in last 12 months: ${eligibleRounds.length}`);
let twentyFourMonthsBeforeUpdate = null;
if (eligibleRounds.length < 8) { if (eligibleRounds.length < 8) {
const twentyFourMonthsBeforeUpdate = new Date(nextUpdateDate); twentyFourMonthsBeforeUpdate = new Date(nextUpdateDate);
twentyFourMonthsBeforeUpdate.setFullYear(twentyFourMonthsBeforeUpdate.getFullYear() - 2); twentyFourMonthsBeforeUpdate.setFullYear(twentyFourMonthsBeforeUpdate.getFullYear() - 2);
eligibleRounds = allSortedRounds.filter(r => r.date >= twentyFourMonthsBeforeUpdate); eligibleRounds = allSortedRounds.filter(r => r.date >= twentyFourMonthsBeforeUpdate);
@@ -131,6 +135,7 @@ function calculatePredictedRating(roundRatings) {
if (eligibleRounds.length === 0) { if (eligibleRounds.length === 0) {
debugLog.push('❌ No eligible rounds found'); debugLog.push('❌ No eligible rounds found');
logger.warn({ pdgaNumber, callsite, reason: 'no_eligible_rounds', afterDateFilter: allSortedRounds.length, twelveMonthCutoff: twelveMonthsBeforeUpdate.toISOString(), twentyFourMonthCutoff: twentyFourMonthsBeforeUpdate ? twentyFourMonthsBeforeUpdate.toISOString() : null }, 'predicted rating computed as 0');
return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null }; return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null };
} }
@@ -239,7 +244,20 @@ function calculatePredictedRating(roundRatings) {
debugLog.push(` Final Rating: ${finalRating}`); debugLog.push(` Final Rating: ${finalRating}`);
debugLog.push('=== END PDGA CALCULATION ==='); debugLog.push('=== END PDGA CALCULATION ===');
logger.info({
pdgaNumber,
callsite,
finalRating,
inputRounds: roundRatings.length,
afterDateFilter: allSortedRounds.length,
eligibleRounds: eligibleRounds.length,
outlierExclusionApplied: workingRatings.length >= 7,
doubleWeightingApplied: workingRatings.length >= 9,
excludedRoundsCount,
cutoffRating
}, 'predicted rating calculated');
return { rating: finalRating, stdDev: Math.round(stdDev), debugLog, excludedRoundsCount, cutoffRating }; return { rating: finalRating, stdDev: Math.round(stdDev), debugLog, excludedRoundsCount, cutoffRating };
} }
module.exports = { parseDate, getNextPDGAUpdateDate, getPreviousPDGAUpdateDate, calculatePredictedRating, calculateStandardDeviation }; module.exports = { parseDate, getNextPDGAUpdateDate, calculatePredictedRating, calculateStandardDeviation };
+2 -2
View File
@@ -1,14 +1,14 @@
const { calculatePredictedRating, getNextPDGAUpdateDate } = require('./rating-calculator'); const { calculatePredictedRating, getNextPDGAUpdateDate } = require('./rating-calculator');
const logger = require('../logger'); const logger = require('../logger');
function calculateRequiredAverage(roundRatings, targetRating, numRounds) { function calculateRequiredAverage(roundRatings, targetRating, numRounds, pdgaNumber) {
if (!Array.isArray(roundRatings) || roundRatings.length === 0) { if (!Array.isArray(roundRatings) || roundRatings.length === 0) {
const err = new Error('No round history'); const err = new Error('No round history');
err.code = 'NO_ROUNDS'; err.code = 'NO_ROUNDS';
throw err; throw err;
} }
const currentPredicted = calculatePredictedRating(roundRatings).rating; const currentPredicted = calculatePredictedRating(roundRatings, { pdgaNumber, callsite: 'target-rating-calculator' }).rating;
const nextUpdate = getNextPDGAUpdateDate(); const nextUpdate = getNextPDGAUpdateDate();
const syntheticDate = new Date(nextUpdate.getTime() - 24 * 60 * 60 * 1000); const syntheticDate = new Date(nextUpdate.getTime() - 24 * 60 * 60 * 1000);
-5
View File
@@ -65,11 +65,6 @@ function renderSparkline(values, opts) {
<span class="m-player-name"><%= player.name %></span> <span class="m-player-name"><%= player.name %></span>
<span class="m-pdga-num">#<%= player.pdgaNumber %></span> <span class="m-pdga-num">#<%= player.pdgaNumber %></span>
</div> </div>
<button class="icon-btn refresh-icon m-refresh-icon" type="button"
onclick="event.stopPropagation(); refreshPlayerData(<%= player.pdgaNumber %>)"
title="Refresh rating + prediction" aria-label="Refresh rating and prediction">
<i class="fas fa-sync-alt"></i>
</button>
<span class="m-chevron">&#9660;</span> <span class="m-chevron">&#9660;</span>
</div> </div>