Compare commits
2 Commits
main
..
31d80273b8
| Author | SHA1 | Date | |
|---|---|---|---|
| 31d80273b8 | |||
| 83ba20d428 |
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "pdga-ratings",
|
||||
"version": "1.4.2",
|
||||
"version": "1.4.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "pdga-ratings",
|
||||
"version": "1.4.2",
|
||||
"version": "1.4.0",
|
||||
"dependencies": {
|
||||
"ejs": "^4.0.1",
|
||||
"express": "^4.18.2",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "pdga-ratings",
|
||||
"version": "1.4.2",
|
||||
"version": "1.4.0",
|
||||
"description": "PDGA rating scraper and display",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
|
||||
@@ -342,38 +342,6 @@
|
||||
transform: rotate(180deg);
|
||||
}
|
||||
|
||||
/* Refresh button: hidden by default, revealed only when the card is open.
|
||||
Larger than the desktop icon to give a comfortable touch target (≥44px). */
|
||||
.m-card__head .m-refresh-icon {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.m-card.is-open .m-card__head .m-refresh-icon {
|
||||
display: grid;
|
||||
width: 44px;
|
||||
height: 44px;
|
||||
margin-left: 0;
|
||||
font-size: 15px;
|
||||
opacity: 0.7;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.m-card.is-open .m-card__head .m-refresh-icon:active {
|
||||
opacity: 1;
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
/* Spin only the icon glyph, not the 44px button box — otherwise the button's
|
||||
lingering touch-hover frame (background + border) rotates too, which looks odd. */
|
||||
.m-card.is-open .m-card__head .m-refresh-icon.spinning {
|
||||
animation: none;
|
||||
}
|
||||
|
||||
.m-card.is-open .m-card__head .m-refresh-icon.spinning i {
|
||||
display: inline-block;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
.m-card__body {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
|
||||
+4
-10
@@ -131,16 +131,10 @@ async function clearCache() {
|
||||
// Refreshes both the current rating and the prediction in one click, then
|
||||
// re-swaps the table so every derived value (deltas, pills, sparkline) reflects
|
||||
// the new state. Cheaper than fine-grained DOM updates and guaranteed consistent
|
||||
// because the server renders the truth. The mobile cards partial is included
|
||||
// inside ratings-table, so swapping #ratings-table re-renders both views at once.
|
||||
// because the server renders the truth.
|
||||
async function refreshPlayerData(pdgaNumber) {
|
||||
// The desktop row exists in the DOM even on mobile (hidden via CSS), so spin
|
||||
// both possible icons; only the one visible in the active viewport is seen.
|
||||
const icons = [
|
||||
document.querySelector(`#row-${pdgaNumber} .cell-actions .refresh-icon`),
|
||||
document.querySelector(`#m-card-${pdgaNumber} .m-refresh-icon`)
|
||||
].filter(Boolean);
|
||||
icons.forEach(icon => icon.classList.add('spinning'));
|
||||
const icon = document.querySelector(`#row-${pdgaNumber} .cell-actions .refresh-icon`);
|
||||
if (icon) icon.classList.add('spinning');
|
||||
try {
|
||||
await Promise.allSettled([
|
||||
fetch(`/api/refresh-player/${pdgaNumber}`, { method: 'POST' }),
|
||||
@@ -150,7 +144,7 @@ async function refreshPlayerData(pdgaNumber) {
|
||||
} catch (error) {
|
||||
console.error('Error refreshing player data:', error);
|
||||
} finally {
|
||||
icons.forEach(icon => icon.classList.remove('spinning'));
|
||||
if (icon) icon.classList.remove('spinning');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -76,16 +76,6 @@ function initializeDatabase() {
|
||||
else logger.info('Successfully added cutoff_rating column');
|
||||
});
|
||||
}
|
||||
|
||||
const hasPredictedCalculatedAt = columns.some(col => col.name === 'predicted_calculated_at');
|
||||
|
||||
if (!hasPredictedCalculatedAt) {
|
||||
logger.info('Adding predicted_calculated_at column to players table...');
|
||||
db.run(`ALTER TABLE players ADD COLUMN predicted_calculated_at DATETIME DEFAULT NULL`, (err) => {
|
||||
if (err) logger.error('Error adding predicted_calculated_at column:', err.message);
|
||||
else logger.info('Successfully added predicted_calculated_at column');
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+27
-29
@@ -1,5 +1,6 @@
|
||||
const { db } = require('../db');
|
||||
const { parseDate } = require('../services/rating-calculator');
|
||||
const logger = require('../logger');
|
||||
|
||||
function getPlayerFromDB(pdgaNumber) {
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -16,28 +17,14 @@ function getPlayerFromDB(pdgaNumber) {
|
||||
|
||||
function savePlayerToDB(playerData) {
|
||||
return new Promise((resolve, reject) => {
|
||||
logger.info({ pdgaNumber: playerData.pdgaNumber, name: playerData.name, currentRating: playerData.rating, ratingChange: playerData.ratingChange, resetColumns: ['predicted_rating', 'std_dev', 'excluded_rounds_count', 'cutoff_rating', 'last_round_update'] }, 'INSERT OR REPLACE on players — derived columns will be reset to NULL');
|
||||
db.run(
|
||||
// UPSERT (not INSERT OR REPLACE): updating in place preserves columns not
|
||||
// listed here — predicted_rating, std_dev, last_round_update,
|
||||
// excluded_rounds_count, cutoff_rating. INSERT OR REPLACE would delete the
|
||||
// existing row and reset those to their DEFAULT (NULL).
|
||||
`INSERT INTO players (pdga_number, name, current_rating, rating_change, last_updated)
|
||||
VALUES (?, ?, ?, ?, datetime('now'))
|
||||
ON CONFLICT(pdga_number) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
current_rating = excluded.current_rating,
|
||||
rating_change = excluded.rating_change,
|
||||
last_updated = excluded.last_updated`,
|
||||
`INSERT OR REPLACE INTO players (pdga_number, name, current_rating, rating_change, last_updated)
|
||||
VALUES (?, ?, ?, ?, datetime('now'))`,
|
||||
[playerData.pdgaNumber, playerData.name, playerData.rating, playerData.ratingChange],
|
||||
function(err) {
|
||||
if (err) return reject(err);
|
||||
// node-sqlite3 leaves lastID = 0 when ON CONFLICT triggers an UPDATE.
|
||||
// Fall back to a SELECT to get the real id in that case.
|
||||
if (this.lastID !== 0) return resolve(this.lastID);
|
||||
db.get('SELECT id FROM players WHERE pdga_number = ?', [playerData.pdgaNumber], (err2, row) => {
|
||||
if (err2) reject(err2);
|
||||
else resolve(row ? row.id : 0);
|
||||
});
|
||||
if (err) reject(err);
|
||||
else resolve(this.lastID);
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -187,17 +174,28 @@ function saveRoundHistoryToDB(pdgaNumber, roundData, isIncremental = false) {
|
||||
});
|
||||
}
|
||||
|
||||
function savePredictedRatingToDB(pdgaNumber, predictedRating, stdDev = null, excludedRoundsCount = null, cutoffRating = null, calculatedAt = null) {
|
||||
const timestamp = calculatedAt || new Date().toISOString();
|
||||
function savePredictedRatingToDB(pdgaNumber, predictedRating, stdDev = null, excludedRoundsCount = null, cutoffRating = null, callsite = 'unknown') {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.run(
|
||||
'UPDATE players SET predicted_rating = ?, std_dev = ?, excluded_rounds_count = ?, cutoff_rating = ?, predicted_calculated_at = ? WHERE pdga_number = ?',
|
||||
[predictedRating, stdDev, excludedRoundsCount, cutoffRating, timestamp, pdgaNumber],
|
||||
function(err) {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
}
|
||||
);
|
||||
db.get('SELECT predicted_rating AS oldValue FROM players WHERE pdga_number = ?', [pdgaNumber], (selectErr, row) => {
|
||||
if (selectErr) return reject(selectErr);
|
||||
const oldValue = row ? row.oldValue : null;
|
||||
|
||||
db.run(
|
||||
'UPDATE players SET predicted_rating = ?, std_dev = ?, excluded_rounds_count = ?, cutoff_rating = ? WHERE pdga_number = ?',
|
||||
[predictedRating, stdDev, excludedRoundsCount, cutoffRating, pdgaNumber],
|
||||
function(err) {
|
||||
if (err) return reject(err);
|
||||
|
||||
const logData = { pdgaNumber, oldValue, newValue: predictedRating, callsite };
|
||||
if (predictedRating === 0 && oldValue != null && oldValue > 0) {
|
||||
logger.warn(logData, 'predicted rating overwritten with 0');
|
||||
} else {
|
||||
logger.info(logData, 'predicted rating saved');
|
||||
}
|
||||
resolve();
|
||||
}
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+64
-27
@@ -7,7 +7,7 @@ const { getOfficialRatingHistory, getOptimizedPlayerRounds } = require('../scrap
|
||||
const { launchBrowser } = require('../scrapers/browser');
|
||||
const { getPlayerDataFromDB, scrapePDGARating, getAllRatingsFromDB, refreshAllPlayersInDB, getPredictedRatingFromDB, formatDisplayDate } = require('../services/player-service');
|
||||
const { getTopbarLocals } = require('../services/topbar-service');
|
||||
const { calculatePredictedRating } = require('../services/rating-calculator');
|
||||
const { calculatePredictedRating, getNextPDGAUpdateDate } = require('../services/rating-calculator');
|
||||
const { calculateRequiredAverage } = require('../services/target-rating-calculator');
|
||||
const logger = require('../logger');
|
||||
|
||||
@@ -19,6 +19,7 @@ router.post('/api/refresh-all', async (req, res, next) => {
|
||||
return res.status(409).json({ error: 'Refresh already in progress' });
|
||||
}
|
||||
refreshInProgress = true;
|
||||
logger.info({ pdgaNumber: 'all' }, 'refresh-all triggered — will invoke savePlayerToDB for all players (INSERT OR REPLACE wipes derived columns)');
|
||||
try {
|
||||
try {
|
||||
await refreshAllPlayersInDB();
|
||||
@@ -253,7 +254,7 @@ router.post('/api/add-player', async (req, res) => {
|
||||
router.post('/api/refresh-player/:pdgaNumber', async (req, res) => {
|
||||
try {
|
||||
const { pdgaNumber } = req.params;
|
||||
logger.info(`Manually refreshing player data for PDGA ${pdgaNumber}`);
|
||||
logger.info({ pdgaNumber }, 'refresh-player triggered — will invoke savePlayerToDB (INSERT OR REPLACE wipes derived columns)');
|
||||
|
||||
const html = await fetchPlayerDataHTTP(pdgaNumber);
|
||||
const playerData = parsePlayerData(html, pdgaNumber);
|
||||
@@ -349,48 +350,45 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
|
||||
|
||||
const isIncremental = !!sinceDate;
|
||||
|
||||
logger.info(`${isIncremental ? 'Incrementally updating' : 'Fully refreshing'} round history for PDGA ${pdgaNumber}${sinceDate ? ` since ${sinceDate.toDateString()}` : ''}`);
|
||||
logger.info({ pdgaNumber, lastRoundUpdate, isIncremental }, 'refresh-round-history started');
|
||||
|
||||
browser = await launchBrowser();
|
||||
|
||||
|
||||
let officialHistory;
|
||||
try {
|
||||
officialHistory = await getOfficialRatingHistory(browser, pdgaNumber);
|
||||
if (officialHistory.length > 0) {
|
||||
await saveRatingHistoryToDB(pdgaNumber, officialHistory);
|
||||
}
|
||||
logger.info({ pdgaNumber, step: 'official_history_scrape', success: officialHistory.length > 0, count: officialHistory.length }, 'official history scrape completed');
|
||||
} catch (historyError) {
|
||||
logger.error('Failed to fetch official history:', historyError.message);
|
||||
logger.warn({ pdgaNumber, step: 'official_history_scrape', success: false, error: historyError.message }, 'official history scrape failed');
|
||||
officialHistory = [];
|
||||
}
|
||||
|
||||
|
||||
let allRounds = [];
|
||||
try {
|
||||
logger.info(`Using optimized approach: /details + new tournaments only for PDGA ${pdgaNumber}...`);
|
||||
allRounds = await getOptimizedPlayerRounds(browser, pdgaNumber);
|
||||
|
||||
|
||||
if (allRounds.length > 0) {
|
||||
const roundsForDB = allRounds.map(round => ({
|
||||
rating: round.rating,
|
||||
date: round.date,
|
||||
competition: round.competition
|
||||
}));
|
||||
|
||||
|
||||
await saveRoundHistoryToDB(pdgaNumber, roundsForDB, false);
|
||||
logger.info(`✓ Saved ${allRounds.length} rounds using optimized approach`);
|
||||
|
||||
await updateLastRoundUpdateDate(pdgaNumber);
|
||||
} else {
|
||||
logger.info('ℹ No rounds found');
|
||||
}
|
||||
logger.info({ pdgaNumber, step: 'rounds_scrape_save', success: allRounds.length > 0, count: allRounds.length, lastRoundUpdateTouched: allRounds.length > 0 }, 'rounds scrape and save completed');
|
||||
} catch (detailsError) {
|
||||
logger.error('Failed to fetch rounds using optimized approach:', detailsError.message);
|
||||
logger.warn({ pdgaNumber, step: 'rounds_scrape_save', success: false, error: detailsError.message }, 'rounds scrape and save failed');
|
||||
allRounds = [];
|
||||
}
|
||||
|
||||
|
||||
await browser.close();
|
||||
browser = null;
|
||||
|
||||
|
||||
const dbRounds = await getRoundHistoryFromDB(pdgaNumber);
|
||||
const roundsForPrediction = dbRounds.map(round => ({
|
||||
rating: round.rating,
|
||||
@@ -398,9 +396,10 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
|
||||
competition: round.competition_name
|
||||
}));
|
||||
|
||||
const result = calculatePredictedRating(roundsForPrediction);
|
||||
const result = calculatePredictedRating(roundsForPrediction, { pdgaNumber, callsite: 'refresh-round-history' });
|
||||
logger.info({ pdgaNumber, step: 'predicted_calc', rating: result.rating, stdDev: result.stdDev, excludedRoundsCount: result.excludedRoundsCount }, 'predicted rating calculation step completed');
|
||||
|
||||
await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating);
|
||||
await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating, 'refresh-round-history');
|
||||
|
||||
const officialCount = allRounds.filter(r => r.source === 'official').length;
|
||||
const newCount = allRounds.filter(r => r.source === 'new').length;
|
||||
@@ -418,10 +417,8 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
|
||||
message: `Used /details (${officialCount} rounds) + new tournaments (${newCount} rounds)`
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(`=== Error refreshing round history for PDGA ${pdgaNumber} ===`);
|
||||
logger.error('Error type:', error.constructor.name);
|
||||
logger.error('Error message:', error.message);
|
||||
|
||||
logger.error({ pdgaNumber, step: 'unknown_or_orchestration', errorType: error.constructor.name, errorMessage: error.message }, 'refresh-round-history failed');
|
||||
|
||||
if (browser) {
|
||||
try {
|
||||
await browser.close();
|
||||
@@ -429,14 +426,14 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
|
||||
logger.error('Error closing browser:', closeError.message);
|
||||
}
|
||||
}
|
||||
|
||||
res.status(500).json({
|
||||
|
||||
res.status(500).json({
|
||||
error: 'Failed to refresh round history',
|
||||
details: error.message,
|
||||
errorType: error.constructor.name,
|
||||
timestamp: new Date().toISOString(),
|
||||
suggestion: error.message.includes('socket hang up') ?
|
||||
'Rate limited by PDGA - try again in a few minutes.' :
|
||||
suggestion: error.message.includes('socket hang up') ?
|
||||
'Rate limited by PDGA - try again in a few minutes.' :
|
||||
error.message.includes('timeout') ?
|
||||
'PDGA pages are loading slowly - try again later.' :
|
||||
'Tournament scraping failed - check server logs for details'
|
||||
@@ -444,6 +441,46 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/api/admin/player-state/:pdgaNumber', async (req, res) => {
|
||||
const { pdgaNumber } = req.params;
|
||||
try {
|
||||
const player = await getPlayerFromDB(pdgaNumber);
|
||||
if (!player) {
|
||||
return res.status(404).json({ error: 'Player not found', pdgaNumber: parseInt(pdgaNumber) });
|
||||
}
|
||||
const rounds = await getRoundHistoryFromDB(pdgaNumber);
|
||||
const cutoff = getNextPDGAUpdateDate();
|
||||
const twelveMonthsAgo = new Date(cutoff); twelveMonthsAgo.setFullYear(cutoff.getFullYear() - 1);
|
||||
const twentyFourMonthsAgo = new Date(cutoff); twentyFourMonthsAgo.setFullYear(cutoff.getFullYear() - 2);
|
||||
|
||||
const roundsInLast12mo = rounds.filter(r => new Date(r.date) >= twelveMonthsAgo).length;
|
||||
const roundsInLast24mo = rounds.filter(r => new Date(r.date) >= twentyFourMonthsAgo).length;
|
||||
const dates = rounds.map(r => r.date).sort();
|
||||
|
||||
res.json({
|
||||
pdgaNumber: player.pdga_number,
|
||||
name: player.name,
|
||||
currentRating: player.current_rating,
|
||||
ratingChange: player.rating_change,
|
||||
predictedRating: player.predicted_rating,
|
||||
stdDev: player.std_dev,
|
||||
excludedRoundsCount: player.excluded_rounds_count,
|
||||
cutoffRating: player.cutoff_rating,
|
||||
lastUpdated: player.last_updated,
|
||||
lastRoundUpdate: player.last_round_update,
|
||||
cutoffDate: cutoff.toISOString(),
|
||||
roundCount: rounds.length,
|
||||
roundsInLast12mo,
|
||||
roundsInLast24mo,
|
||||
oldestRound: dates[0] ?? null,
|
||||
newestRound: dates[dates.length - 1] ?? null
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error({ err, pdgaNumber }, 'admin player-state endpoint failed');
|
||||
res.status(500).json({ error: 'Failed to fetch player state', details: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/api/calculate-target-rating/:pdgaNumber', async (req, res) => {
|
||||
const { pdgaNumber } = req.params;
|
||||
const pdgaNum = parseInt(pdgaNumber, 10);
|
||||
@@ -483,7 +520,7 @@ router.post('/api/calculate-target-rating/:pdgaNumber', async (req, res) => {
|
||||
competition: r.competition_name
|
||||
}));
|
||||
|
||||
const result = calculateRequiredAverage(roundRatings, target, numRounds);
|
||||
const result = calculateRequiredAverage(roundRatings, target, numRounds, pdgaNum);
|
||||
|
||||
logger.info(`Target rating calc for PDGA ${pdgaNum}: target=${target} rounds=${numRounds} -> avg=${result.requiredAverage}`);
|
||||
|
||||
|
||||
@@ -156,154 +156,81 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) {
|
||||
|
||||
logger.info(`Looking for tournaments after ${afterDate.toDateString()}...`);
|
||||
|
||||
const { urls: newTournamentUrls, counts } = await page.evaluate((afterTimestamp) => {
|
||||
const newTournamentUrls = await page.evaluate((afterTimestamp) => {
|
||||
const afterDate = new Date(afterTimestamp);
|
||||
const tables = document.querySelectorAll('table[id*="player-results"]');
|
||||
const urls = [];
|
||||
const seenUrls = new Set();
|
||||
let table = 0;
|
||||
let recentEvents = 0;
|
||||
let recentEventsAnchorsSeen = 0;
|
||||
let recentEventsSkippedDuplicates = 0;
|
||||
|
||||
tables.forEach(tbl => {
|
||||
const rows = tbl.querySelectorAll('tbody tr');
|
||||
|
||||
tables.forEach(table => {
|
||||
const rows = table.querySelectorAll('tbody tr');
|
||||
rows.forEach(row => {
|
||||
const dateCell = row.querySelector('.dates');
|
||||
const tournamentCell = row.querySelector('.tournament a');
|
||||
|
||||
|
||||
if (dateCell && tournamentCell) {
|
||||
const dateText = dateCell.innerText.trim();
|
||||
const dateMatch = dateText.match(/\d{1,2}-[A-Za-z]{3}-\d{4}/);
|
||||
|
||||
|
||||
if (dateMatch) {
|
||||
const dateStr = dateMatch[0];
|
||||
const date = new Date(dateStr);
|
||||
|
||||
|
||||
if (date > afterDate) {
|
||||
const href = tournamentCell.getAttribute('href');
|
||||
if (href) {
|
||||
const absoluteUrl = new URL(href, location.origin).href;
|
||||
if (!seenUrls.has(absoluteUrl)) {
|
||||
seenUrls.add(absoluteUrl);
|
||||
urls.push({
|
||||
url: absoluteUrl,
|
||||
date: dateStr,
|
||||
name: tournamentCell.innerText.trim(),
|
||||
source: 'table'
|
||||
});
|
||||
table++;
|
||||
}
|
||||
urls.push({
|
||||
url: `https://www.pdga.com${href}`,
|
||||
date: dateStr,
|
||||
name: tournamentCell.innerText.trim()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const recentAnchors = document.querySelectorAll('.recent-events a[href*="/tour/event/"]');
|
||||
recentAnchors.forEach(anchor => {
|
||||
recentEventsAnchorsSeen++;
|
||||
const href = anchor.getAttribute('href');
|
||||
if (href) {
|
||||
const absoluteUrl = new URL(href, location.origin).href;
|
||||
if (seenUrls.has(absoluteUrl)) {
|
||||
recentEventsSkippedDuplicates++;
|
||||
} else {
|
||||
seenUrls.add(absoluteUrl);
|
||||
urls.push({
|
||||
url: absoluteUrl,
|
||||
date: null,
|
||||
name: anchor.innerText.trim() || 'Recent event',
|
||||
source: 'recent-events'
|
||||
});
|
||||
recentEvents++;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return { urls, counts: { table, recentEvents, recentEventsAnchorsSeen, recentEventsSkippedDuplicates } };
|
||||
|
||||
return urls;
|
||||
}, afterDate.getTime());
|
||||
|
||||
logger.info({
|
||||
pdgaNumber,
|
||||
afterDate: afterDate.toISOString(),
|
||||
tableMatches: counts.table,
|
||||
recentEventsMatches: counts.recentEvents,
|
||||
recentEventsAnchorsSeen: counts.recentEventsAnchorsSeen,
|
||||
recentEventsSkippedDuplicates: counts.recentEventsSkippedDuplicates,
|
||||
totalUrlsToScrape: newTournamentUrls.length
|
||||
}, 'new tournament URL discovery completed');
|
||||
|
||||
logger.info(`Found ${newTournamentUrls.length} new tournaments after ${afterDate.toDateString()}`);
|
||||
|
||||
for (const tournamentData of newTournamentUrls) {
|
||||
try {
|
||||
if (tournamentData.source === 'recent-events') {
|
||||
logger.debug({ pdgaNumber, url: tournamentData.url }, 'recent-events: scraping tournament');
|
||||
} else {
|
||||
logger.info(`Scraping new tournament: ${tournamentData.name} (${tournamentData.date})`);
|
||||
}
|
||||
|
||||
logger.info(`Scraping new tournament: ${tournamentData.name} (${tournamentData.date})`);
|
||||
|
||||
await page.goto(tournamentData.url, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
|
||||
let parsedDate;
|
||||
if (tournamentData.date !== null) {
|
||||
parsedDate = parseDate(tournamentData.date);
|
||||
} else {
|
||||
const eventDateStr = await page.evaluate(() => {
|
||||
const body = document.body ? document.body.innerText : '';
|
||||
const m = body.match(/\d{1,2}\s+to\s+\d{1,2}-[A-Za-z]{3}-\d{4}/)
|
||||
|| body.match(/\d{1,2}-[A-Za-z]{3}-\d{4}/);
|
||||
return m ? m[0] : null;
|
||||
});
|
||||
|
||||
if (eventDateStr) {
|
||||
parsedDate = parseDate(eventDateStr);
|
||||
if (!(parsedDate > afterDate)) {
|
||||
logger.warn({
|
||||
pdgaNumber,
|
||||
url: tournamentData.url,
|
||||
eventDateStr,
|
||||
parsedDate: parsedDate ? parsedDate.toISOString() : null,
|
||||
afterDate: afterDate.toISOString()
|
||||
}, 'recent-events: extracted event date is not newer than afterDate, likely captured a non-tournament date — skipping');
|
||||
continue;
|
||||
}
|
||||
logger.debug({ pdgaNumber, url: tournamentData.url, eventDateStr }, 'recent-events: extracted date from event page');
|
||||
} else {
|
||||
logger.warn({ pdgaNumber, url: tournamentData.url }, 'recent-events: could not extract date from event page, skipping tournament');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const roundRatings = await page.evaluate((pdgaNum) => {
|
||||
const rows = document.querySelectorAll('tr');
|
||||
|
||||
|
||||
for (const row of rows) {
|
||||
const cells = row.querySelectorAll('td');
|
||||
const hasPlayerNumber = Array.from(cells).some(cell =>
|
||||
const hasPlayerNumber = Array.from(cells).some(cell =>
|
||||
cell.innerText && cell.innerText.includes(pdgaNum.toString())
|
||||
);
|
||||
|
||||
|
||||
if (hasPlayerNumber) {
|
||||
const roundRatingCells = row.querySelectorAll('td.round-rating');
|
||||
const ratings = [];
|
||||
|
||||
|
||||
roundRatingCells.forEach(cell => {
|
||||
const rating = parseInt(cell.innerText.trim());
|
||||
if (!isNaN(rating) && rating > 0) {
|
||||
ratings.push(rating);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
return ratings;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return [];
|
||||
}, pdgaNumber);
|
||||
|
||||
|
||||
if (roundRatings.length > 0) {
|
||||
const parsedDate = parseDate(tournamentData.date);
|
||||
roundRatings.forEach(rating => {
|
||||
newRounds.push({
|
||||
rating,
|
||||
@@ -311,10 +238,10 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) {
|
||||
competition: tournamentData.name
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
logger.info(`Found ${roundRatings.length} round ratings for ${tournamentData.name}`);
|
||||
}
|
||||
|
||||
|
||||
} catch (error) {
|
||||
logger.error(`Error scraping tournament ${tournamentData.name}: ${error.message}`);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
const { db } = require('../db');
|
||||
const { getPlayerFromDB, getRoundHistoryFromDB, savePredictedRatingToDB, savePlayerToDB, getMonthlyHistory, getAllMonthlyHistoriesFromDB, getAllRatingHistoriesFromDB } = require('../models/player');
|
||||
const { fetchPlayerDataHTTP, parsePlayerData } = require('../scrapers/player-http');
|
||||
const { calculatePredictedRating, getPreviousPDGAUpdateDate } = require('./rating-calculator');
|
||||
const { calculatePredictedRating } = require('./rating-calculator');
|
||||
const logger = require('../logger');
|
||||
|
||||
function formatDisplayDate(dateStr) {
|
||||
@@ -36,43 +36,32 @@ async function getPlayerDataFromDB(pdgaNumber, { includeMonthlyHistory = true }
|
||||
try {
|
||||
const cachedPlayer = await getPlayerFromDB(pdgaNumber);
|
||||
if (cachedPlayer) {
|
||||
logger.debug(`Loading PDGA ${pdgaNumber} from DB (source of truth)`);
|
||||
logger.debug({ pdgaNumber }, 'Loading player from DB (source of truth)');
|
||||
|
||||
let predictedRating = cachedPlayer.predicted_rating;
|
||||
let stdDev = cachedPlayer.std_dev;
|
||||
let excludedRoundsCount = cachedPlayer.excluded_rounds_count;
|
||||
let cutoffRating = cachedPlayer.cutoff_rating;
|
||||
let predictedCalculatedAtRaw = cachedPlayer.predicted_calculated_at;
|
||||
let recomputeAttempted = false;
|
||||
if (!predictedRating || predictedRating === 0) {
|
||||
recomputeAttempted = true;
|
||||
logger.debug({ pdgaNumber, dbValue: cachedPlayer.predicted_rating }, 'lazy recompute triggered for predicted_rating');
|
||||
predictedRating = await getPredictedRatingFromDB(pdgaNumber);
|
||||
const updatedPlayer = await getPlayerFromDB(pdgaNumber);
|
||||
stdDev = updatedPlayer?.std_dev;
|
||||
excludedRoundsCount = updatedPlayer?.excluded_rounds_count;
|
||||
cutoffRating = updatedPlayer?.cutoff_rating;
|
||||
predictedCalculatedAtRaw = updatedPlayer?.predicted_calculated_at;
|
||||
}
|
||||
|
||||
// Staleness-check: invalidate cached predicted_rating if the PDGA cycle has
|
||||
// rolled over since it was calculated. Don't recompute — round_history may be
|
||||
// equally stale. UI will show "—" until the next manual refresh.
|
||||
const predictedCalculatedAt = predictedCalculatedAtRaw
|
||||
? new Date(predictedCalculatedAtRaw)
|
||||
: null;
|
||||
const previousUpdate = getPreviousPDGAUpdateDate();
|
||||
const hasPredicted = predictedRating !== null && predictedRating !== 0;
|
||||
const isStale = hasPredicted && (
|
||||
predictedCalculatedAt === null || predictedCalculatedAt < previousUpdate
|
||||
);
|
||||
if (isStale) {
|
||||
logger.debug(`PDGA ${pdgaNumber}: predicted rating stale (calculated ${predictedCalculatedAt?.toISOString() ?? 'unknown'}, cycle rolled ${previousUpdate.toISOString()})`);
|
||||
predictedRating = null;
|
||||
stdDev = null;
|
||||
excludedRoundsCount = null;
|
||||
cutoffRating = null;
|
||||
if (!predictedRating || predictedRating === 0) {
|
||||
logger.info({ pdgaNumber, recomputedValue: predictedRating }, 'lazy recompute did not yield a positive predicted rating');
|
||||
}
|
||||
}
|
||||
|
||||
const rating = cachedPlayer.current_rating;
|
||||
const rawRatingChange = cachedPlayer.rating_change;
|
||||
// Only warn about ≤0 if it wasn't already explained by a failed recompute (which has its own log)
|
||||
if (!recomputeAttempted && predictedRating != null && predictedRating <= 0) {
|
||||
logger.warn({ pdgaNumber, dbValue: predictedRating }, 'predicted rating present but <= 0 in DB without recompute — rendered as empty');
|
||||
}
|
||||
const resolvedPredicted = predictedRating > 0 ? predictedRating : null;
|
||||
const resolvedStdDev = stdDev > 0 ? stdDev : null;
|
||||
|
||||
@@ -161,22 +150,28 @@ async function scrapePDGARating(pdgaNumber, retries = 3) {
|
||||
async function getPredictedRatingFromDB(pdgaNumber) {
|
||||
try {
|
||||
const roundHistory = await getRoundHistoryFromDB(pdgaNumber);
|
||||
if (roundHistory.length > 0) {
|
||||
logger.debug(`Using ${roundHistory.length} cached rounds for PDGA ${pdgaNumber} prediction`);
|
||||
|
||||
const roundRatings = roundHistory.map(round => ({
|
||||
rating: round.rating,
|
||||
date: new Date(round.date),
|
||||
competition: round.competition_name || 'Unknown'
|
||||
}));
|
||||
|
||||
const result = calculatePredictedRating(roundRatings);
|
||||
|
||||
await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating);
|
||||
|
||||
return result.rating;
|
||||
if (roundHistory.length === 0) {
|
||||
logger.info({ pdgaNumber, reason: 'no_round_history_in_db' }, 'predicted recompute returning 0 — no round history available');
|
||||
return 0;
|
||||
}
|
||||
return 0;
|
||||
|
||||
logger.debug({ pdgaNumber, cachedRounds: roundHistory.length }, 'Using cached rounds for prediction');
|
||||
|
||||
const roundRatings = roundHistory.map(round => ({
|
||||
rating: round.rating,
|
||||
date: new Date(round.date),
|
||||
competition: round.competition_name || 'Unknown'
|
||||
}));
|
||||
|
||||
const result = calculatePredictedRating(roundRatings, { pdgaNumber, callsite: 'getPredictedRatingFromDB' });
|
||||
|
||||
if (result.rating === 0) {
|
||||
logger.warn({ pdgaNumber, roundsInDb: roundHistory.length }, 'predicted recompute returned 0 despite having rounds in DB');
|
||||
}
|
||||
|
||||
await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating, 'lazy-recompute');
|
||||
|
||||
return result.rating;
|
||||
} catch (err) {
|
||||
logger.error(`Error getting predicted rating from DB for ${pdgaNumber}:`, err.message);
|
||||
return 0;
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
const logger = require('../logger');
|
||||
|
||||
function parseDate(dateStr) {
|
||||
const multiDayMatch = dateStr.match(/^(\d{1,2})(-([A-Za-z]{3}))?(\s+to\s+)(\d{1,2})-([A-Za-z]{3})-(\d{4})$/);
|
||||
if (multiDayMatch) {
|
||||
@@ -37,41 +39,37 @@ function parseDate(dateStr) {
|
||||
return new Date(dateStr);
|
||||
}
|
||||
|
||||
function secondTuesdayOf(year, month) {
|
||||
const firstDay = new Date(year, month, 1);
|
||||
const daysUntilTuesday = (2 - firstDay.getDay() + 7) % 7;
|
||||
const firstTuesday = new Date(year, month, 1 + daysUntilTuesday);
|
||||
const secondTuesday = new Date(firstTuesday);
|
||||
secondTuesday.setDate(firstTuesday.getDate() + 7);
|
||||
return secondTuesday;
|
||||
}
|
||||
|
||||
function getNextPDGAUpdateDate() {
|
||||
const today = new Date();
|
||||
const currentMonth = today.getMonth();
|
||||
const currentYear = today.getFullYear();
|
||||
|
||||
const secondTuesday = secondTuesdayOf(currentYear, currentMonth);
|
||||
const firstDayOfMonth = new Date(currentYear, currentMonth, 1);
|
||||
const firstTuesday = new Date(firstDayOfMonth);
|
||||
|
||||
const daysUntilTuesday = (2 - firstDayOfMonth.getDay() + 7) % 7;
|
||||
firstTuesday.setDate(1 + daysUntilTuesday);
|
||||
|
||||
const secondTuesday = new Date(firstTuesday);
|
||||
secondTuesday.setDate(firstTuesday.getDate() + 7);
|
||||
|
||||
if (today <= secondTuesday) {
|
||||
return secondTuesday;
|
||||
} else {
|
||||
const nextMonth = currentMonth === 11 ? 0 : currentMonth + 1;
|
||||
const nextYear = currentMonth === 11 ? currentYear + 1 : currentYear;
|
||||
return secondTuesdayOf(nextYear, nextMonth);
|
||||
}
|
||||
}
|
||||
|
||||
function getPreviousPDGAUpdateDate() {
|
||||
const today = new Date();
|
||||
const year = today.getFullYear();
|
||||
const month = today.getMonth();
|
||||
const secondTuesday = secondTuesdayOf(year, month);
|
||||
if (today > secondTuesday) return secondTuesday;
|
||||
// Otherwise: last month's second Tuesday
|
||||
const prevMonth = month === 0 ? 11 : month - 1;
|
||||
const prevYear = month === 0 ? year - 1 : year;
|
||||
return secondTuesdayOf(prevYear, prevMonth);
|
||||
const firstDayNextMonth = new Date(nextYear, nextMonth, 1);
|
||||
const firstTuesdayNext = new Date(firstDayNextMonth);
|
||||
|
||||
const daysUntilTuesdayNext = (2 - firstDayNextMonth.getDay() + 7) % 7;
|
||||
firstTuesdayNext.setDate(1 + daysUntilTuesdayNext);
|
||||
|
||||
const secondTuesdayNext = new Date(firstTuesdayNext);
|
||||
secondTuesdayNext.setDate(firstTuesdayNext.getDate() + 7);
|
||||
|
||||
return secondTuesdayNext;
|
||||
}
|
||||
}
|
||||
|
||||
function calculateStandardDeviation(ratings) {
|
||||
@@ -83,15 +81,19 @@ function calculateStandardDeviation(ratings) {
|
||||
return Math.sqrt(variance);
|
||||
}
|
||||
|
||||
function calculatePredictedRating(roundRatings) {
|
||||
function calculatePredictedRating(roundRatings, context = {}) {
|
||||
const pdgaNumber = context.pdgaNumber ?? null;
|
||||
const callsite = context.callsite ?? 'unknown';
|
||||
const debugLog = [];
|
||||
debugLog.push('=== PDGA RATING CALCULATION (Following Official Rules) ===');
|
||||
|
||||
if (!roundRatings || roundRatings.length === 0) {
|
||||
debugLog.push('❌ No rounds provided for prediction');
|
||||
logger.warn({ pdgaNumber, callsite, reason: 'no_rounds' }, 'predicted rating computed as 0');
|
||||
return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null };
|
||||
}
|
||||
|
||||
logger.info({ pdgaNumber, callsite, inputRounds: roundRatings.length }, 'predicted rating calculation started');
|
||||
debugLog.push(`📊 Starting with ${roundRatings.length} total rounds`);
|
||||
|
||||
const nextUpdateDate = getNextPDGAUpdateDate();
|
||||
@@ -104,6 +106,7 @@ function calculatePredictedRating(roundRatings) {
|
||||
|
||||
if (allSortedRounds.length === 0) {
|
||||
debugLog.push('❌ No valid rounds after filtering for update date');
|
||||
logger.warn({ pdgaNumber, callsite, reason: 'no_rounds_after_date_filter', inputRounds: roundRatings.length, nextUpdateDate: nextUpdateDate.toISOString() }, 'predicted rating computed as 0');
|
||||
return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null };
|
||||
}
|
||||
|
||||
@@ -121,8 +124,9 @@ function calculatePredictedRating(roundRatings) {
|
||||
debugLog.push('🗓️ 12-MONTH FILTERING:');
|
||||
debugLog.push(`✅ Rounds in last 12 months: ${eligibleRounds.length}`);
|
||||
|
||||
let twentyFourMonthsBeforeUpdate = null;
|
||||
if (eligibleRounds.length < 8) {
|
||||
const twentyFourMonthsBeforeUpdate = new Date(nextUpdateDate);
|
||||
twentyFourMonthsBeforeUpdate = new Date(nextUpdateDate);
|
||||
twentyFourMonthsBeforeUpdate.setFullYear(twentyFourMonthsBeforeUpdate.getFullYear() - 2);
|
||||
|
||||
eligibleRounds = allSortedRounds.filter(r => r.date >= twentyFourMonthsBeforeUpdate);
|
||||
@@ -131,6 +135,7 @@ function calculatePredictedRating(roundRatings) {
|
||||
|
||||
if (eligibleRounds.length === 0) {
|
||||
debugLog.push('❌ No eligible rounds found');
|
||||
logger.warn({ pdgaNumber, callsite, reason: 'no_eligible_rounds', afterDateFilter: allSortedRounds.length, twelveMonthCutoff: twelveMonthsBeforeUpdate.toISOString(), twentyFourMonthCutoff: twentyFourMonthsBeforeUpdate ? twentyFourMonthsBeforeUpdate.toISOString() : null }, 'predicted rating computed as 0');
|
||||
return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null };
|
||||
}
|
||||
|
||||
@@ -239,7 +244,20 @@ function calculatePredictedRating(roundRatings) {
|
||||
debugLog.push(` Final Rating: ${finalRating}`);
|
||||
debugLog.push('=== END PDGA CALCULATION ===');
|
||||
|
||||
logger.info({
|
||||
pdgaNumber,
|
||||
callsite,
|
||||
finalRating,
|
||||
inputRounds: roundRatings.length,
|
||||
afterDateFilter: allSortedRounds.length,
|
||||
eligibleRounds: eligibleRounds.length,
|
||||
outlierExclusionApplied: workingRatings.length >= 7,
|
||||
doubleWeightingApplied: workingRatings.length >= 9,
|
||||
excludedRoundsCount,
|
||||
cutoffRating
|
||||
}, 'predicted rating calculated');
|
||||
|
||||
return { rating: finalRating, stdDev: Math.round(stdDev), debugLog, excludedRoundsCount, cutoffRating };
|
||||
}
|
||||
|
||||
module.exports = { parseDate, getNextPDGAUpdateDate, getPreviousPDGAUpdateDate, calculatePredictedRating, calculateStandardDeviation };
|
||||
module.exports = { parseDate, getNextPDGAUpdateDate, calculatePredictedRating, calculateStandardDeviation };
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
const { calculatePredictedRating, getNextPDGAUpdateDate } = require('./rating-calculator');
|
||||
const logger = require('../logger');
|
||||
|
||||
function calculateRequiredAverage(roundRatings, targetRating, numRounds) {
|
||||
function calculateRequiredAverage(roundRatings, targetRating, numRounds, pdgaNumber) {
|
||||
if (!Array.isArray(roundRatings) || roundRatings.length === 0) {
|
||||
const err = new Error('No round history');
|
||||
err.code = 'NO_ROUNDS';
|
||||
throw err;
|
||||
}
|
||||
|
||||
const currentPredicted = calculatePredictedRating(roundRatings).rating;
|
||||
const currentPredicted = calculatePredictedRating(roundRatings, { pdgaNumber, callsite: 'target-rating-calculator' }).rating;
|
||||
const nextUpdate = getNextPDGAUpdateDate();
|
||||
const syntheticDate = new Date(nextUpdate.getTime() - 24 * 60 * 60 * 1000);
|
||||
|
||||
|
||||
@@ -65,11 +65,6 @@ function renderSparkline(values, opts) {
|
||||
<span class="m-player-name"><%= player.name %></span>
|
||||
<span class="m-pdga-num">#<%= player.pdgaNumber %></span>
|
||||
</div>
|
||||
<button class="icon-btn refresh-icon m-refresh-icon" type="button"
|
||||
onclick="event.stopPropagation(); refreshPlayerData(<%= player.pdgaNumber %>)"
|
||||
title="Refresh rating + prediction" aria-label="Refresh rating and prediction">
|
||||
<i class="fas fa-sync-alt"></i>
|
||||
</button>
|
||||
<span class="m-chevron">▼</span>
|
||||
</div>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user