const { db } = require('../db'); const { getPlayerFromDB, getRoundHistoryFromDB, savePredictedRatingToDB, savePlayerToDB, getMonthlyHistory, getAllMonthlyHistoriesFromDB, getAllRatingHistoriesFromDB } = require('../models/player'); const { fetchPlayerDataHTTP, parsePlayerData } = require('../scrapers/player-http'); const { calculatePredictedRating, getPreviousPDGAUpdateDate } = require('./rating-calculator'); const logger = require('../logger'); function formatDisplayDate(dateStr) { return new Date(dateStr).toLocaleDateString('en-US', { day: '2-digit', month: 'short', year: 'numeric' }); } // Derives previous-month rating and the delta to it. Prefers PDGA's reported // rating_change (canonical), falls back to our own monthly snapshots when // rating_change is missing — common for players whose latest scrape failed. function deriveMonthlyDeltas(rating, rawRatingChange, monthlyHistory) { if (rating != null && rawRatingChange != null) { return { lastMonthRating: rating - rawRatingChange, ratingChange: rawRatingChange }; } if (rating != null && monthlyHistory && monthlyHistory.length >= 1) { // The "last month" snapshot depends on whether current_rating is already in // history. If equal, current is the most recent entry — last month is the one // before it. If not, current is newer than history — the latest entry IS last month. const lastIdx = monthlyHistory.length - 1; const lastMonth = (monthlyHistory[lastIdx] === rating) ? (monthlyHistory.length >= 2 ? monthlyHistory[lastIdx - 1] : null) : monthlyHistory[lastIdx]; if (lastMonth != null) { return { lastMonthRating: lastMonth, ratingChange: rating - lastMonth }; } } return { lastMonthRating: null, ratingChange: rawRatingChange }; } async function getPlayerDataFromDB(pdgaNumber, { includeMonthlyHistory = true } = {}) { try { const cachedPlayer = await getPlayerFromDB(pdgaNumber); if (cachedPlayer) { logger.debug(`Loading PDGA ${pdgaNumber} from DB (source of truth)`); let predictedRating = cachedPlayer.predicted_rating; let stdDev = cachedPlayer.std_dev; let excludedRoundsCount = cachedPlayer.excluded_rounds_count; let cutoffRating = cachedPlayer.cutoff_rating; let predictedCalculatedAtRaw = cachedPlayer.predicted_calculated_at; if (!predictedRating || predictedRating === 0) { predictedRating = await getPredictedRatingFromDB(pdgaNumber); const updatedPlayer = await getPlayerFromDB(pdgaNumber); stdDev = updatedPlayer?.std_dev; excludedRoundsCount = updatedPlayer?.excluded_rounds_count; cutoffRating = updatedPlayer?.cutoff_rating; predictedCalculatedAtRaw = updatedPlayer?.predicted_calculated_at; } // Staleness-check: invalidate cached predicted_rating if the PDGA cycle has // rolled over since it was calculated. Don't recompute — round_history may be // equally stale. UI will show "—" until the next manual refresh. const predictedCalculatedAt = predictedCalculatedAtRaw ? new Date(predictedCalculatedAtRaw) : null; const previousUpdate = getPreviousPDGAUpdateDate(); const hasPredicted = predictedRating !== null && predictedRating !== 0; const isStale = hasPredicted && ( predictedCalculatedAt === null || predictedCalculatedAt < previousUpdate ); if (isStale) { logger.debug(`PDGA ${pdgaNumber}: predicted rating stale (calculated ${predictedCalculatedAt?.toISOString() ?? 'unknown'}, cycle rolled ${previousUpdate.toISOString()})`); predictedRating = null; stdDev = null; excludedRoundsCount = null; cutoffRating = null; } const rating = cachedPlayer.current_rating; const rawRatingChange = cachedPlayer.rating_change; const resolvedPredicted = predictedRating > 0 ? predictedRating : null; const resolvedStdDev = stdDev > 0 ? stdDev : null; // Skip in bulk-fetch paths where caller supplies history via getAllMonthlyHistoriesFromDB const monthlyHistory = includeMonthlyHistory ? await getMonthlyHistory(cachedPlayer.pdga_number) : []; const { lastMonthRating, ratingChange } = deriveMonthlyDeltas(rating, rawRatingChange, monthlyHistory); return { pdgaNumber: cachedPlayer.pdga_number, name: cachedPlayer.name, rating, ratingChange, predictedRating: resolvedPredicted, stdDev: resolvedStdDev, excludedRoundsCount: (excludedRoundsCount != null && excludedRoundsCount >= 0) ? excludedRoundsCount : null, cutoffRating: (cutoffRating != null && cutoffRating > 0) ? cutoffRating : null, lastMonthRating, // gap between next predicted update and current rating (null when either is missing) deltaPredicted: (resolvedPredicted != null && rating != null) ? resolvedPredicted - rating : null, monthlyHistory }; } return null; } catch (err) { logger.error(`Database error for PDGA ${pdgaNumber}:`, err.message); return null; } } async function scrapePDGARating(pdgaNumber, retries = 3) { logger.info(`Refreshing PDGA ${pdgaNumber} from PDGA website`); for (let attempt = 1; attempt <= retries; attempt++) { try { logger.info(`Attempt ${attempt}/${retries} for PDGA ${pdgaNumber} (using HTTP)`); const html = await fetchPlayerDataHTTP(pdgaNumber); const result = parsePlayerData(html, pdgaNumber); try { await savePlayerToDB(result); logger.info(`Saved PDGA ${pdgaNumber} to database`); } catch (dbErr) { logger.error(`Failed to save PDGA ${pdgaNumber} to database:`, dbErr.message); } logger.info(`Successfully scraped PDGA ${pdgaNumber} on attempt ${attempt}`); return result; } catch (error) { logger.error(`Attempt ${attempt}/${retries} failed for PDGA ${pdgaNumber}:`, error.message); if (attempt === retries) { return { pdgaNumber, name: 'Error', rating: 0, ratingChange: null, predictedRating: null }; } let retryDelay = 2000 * attempt; if (error.rateLimitInfo) { const retryAfter = error.rateLimitInfo.headers['retry-after']; if (retryAfter) { retryDelay = Math.max(retryDelay, (parseInt(retryAfter) + 1) * 1000); logger.warn(`Using Retry-After header: waiting ${retryDelay/1000}s`); } } if (error.code === 'ECONNRESET') { retryDelay = Math.max(retryDelay, 10000); logger.warn(`Connection reset detected: waiting ${retryDelay/1000}s`); } await new Promise(resolve => setTimeout(resolve, retryDelay)); } } } async function getPredictedRatingFromDB(pdgaNumber) { try { const roundHistory = await getRoundHistoryFromDB(pdgaNumber); if (roundHistory.length > 0) { logger.debug(`Using ${roundHistory.length} cached rounds for PDGA ${pdgaNumber} prediction`); const roundRatings = roundHistory.map(round => ({ rating: round.rating, date: new Date(round.date), competition: round.competition_name || 'Unknown' })); const result = calculatePredictedRating(roundRatings); await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating); return result.rating; } return 0; } catch (err) { logger.error(`Error getting predicted rating from DB for ${pdgaNumber}:`, err.message); return 0; } } async function getAllRatingsFromDB(progressCallback = null) { try { const allPlayers = await new Promise((resolve, reject) => { db.all( 'SELECT pdga_number, name, current_rating, rating_change FROM players ORDER BY pdga_number', [], (err, rows) => { if (err) reject(err); else resolve(rows || []); } ); }); logger.info(`Loading ${allPlayers.length} players from database...`); // Fetch all monthly histories in one query so the per-player loop doesn't add N extra queries const monthlyHistoryMap = await getAllMonthlyHistoriesFromDB(12); const ratingHistoryMap = await getAllRatingHistoriesFromDB(); const ratings = []; const total = allPlayers.length; for (let i = 0; i < allPlayers.length; i++) { const player = allPlayers[i]; const pdgaNumber = player.pdga_number; if (progressCallback) { progressCallback({ current: i + 1, total, pdgaNumber, status: 'loading' }); } try { const playerData = await getPlayerDataFromDB(pdgaNumber, { includeMonthlyHistory: false }); if (playerData) { playerData.monthlyHistory = monthlyHistoryMap.get(pdgaNumber) ?? []; const rawHistory = ratingHistoryMap.get(pdgaNumber) ?? []; playerData.ratingHistory = rawHistory.map(row => ({ date: row.date, rating: row.rating, displayDate: formatDisplayDate(row.date) })); // Re-derive now that history is attached — bulk path skipped includeMonthlyHistory const derived = deriveMonthlyDeltas(playerData.rating, player.rating_change, playerData.monthlyHistory); playerData.lastMonthRating = derived.lastMonthRating; playerData.ratingChange = derived.ratingChange; ratings.push(playerData); } if (progressCallback) { progressCallback({ current: i + 1, total, pdgaNumber, status: 'completed', name: playerData ? playerData.name : player.name }); } } catch (error) { logger.error(`Failed to load PDGA ${pdgaNumber} from database:`, error.message); const errorRating = player.current_rating; const errorRatingChange = player.rating_change; const errorData = { pdgaNumber: parseInt(pdgaNumber), name: player.name || 'Database Error', rating: errorRating, ratingChange: errorRatingChange, predictedRating: null, stdDev: null, excludedRoundsCount: null, cutoffRating: null, lastMonthRating: (errorRating != null && errorRatingChange != null) ? errorRating - errorRatingChange : null, deltaPredicted: null, monthlyHistory: [], ratingHistory: [] }; ratings.push(errorData); if (progressCallback) { progressCallback({ current: i + 1, total, pdgaNumber, status: 'error', name: player.name || 'Database Error' }); } } } return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0)); } catch (error) { logger.error('Error loading players from database:', error); return []; } } async function refreshAllPlayersInDB(progressCallback = null) { try { const allPlayers = await new Promise((resolve, reject) => { db.all( 'SELECT pdga_number, name FROM players ORDER BY pdga_number', [], (err, rows) => { if (err) reject(err); else resolve(rows || []); } ); }); logger.info(`Refreshing ${allPlayers.length} players from database...`); const ratings = []; const total = allPlayers.length; for (let i = 0; i < allPlayers.length; i++) { const player = allPlayers[i]; const pdgaNumber = player.pdga_number; logger.info(`Refreshing PDGA ${pdgaNumber}... (${i + 1}/${total})`); if (progressCallback) { progressCallback({ current: i + 1, total, pdgaNumber, status: 'loading' }); } try { const playerData = await scrapePDGARating(pdgaNumber); ratings.push(playerData); if (progressCallback) { progressCallback({ current: i + 1, total, pdgaNumber, status: 'completed', name: playerData.name }); } await new Promise(resolve => setTimeout(resolve, 2000)); } catch (error) { logger.error(`Failed to refresh PDGA ${pdgaNumber}:`, error.message); const errorData = { pdgaNumber: parseInt(pdgaNumber), name: player.name || 'Error', rating: 0, ratingChange: null, predictedRating: null }; ratings.push(errorData); if (progressCallback) { progressCallback({ current: i + 1, total, pdgaNumber, status: 'error', name: player.name || 'Error' }); } } } return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0)); } catch (error) { logger.error('Error refreshing all players:', error); return []; } } /** * Aggregates KPI summary stats from an already-fetched player array. * All fields are derived from the player list — no extra DB queries. */ function computeKpis(players) { const active = players.filter(p => p.rating != null && p.rating > 0); const avg = active.length > 0 ? Math.round(active.reduce((sum, p) => sum + p.rating, 0) / active.length) : null; return { tracked: players.length, active: active.length, avg, climbing: players.filter(p => p.ratingChange != null && p.ratingChange > 0).length, slipping: players.filter(p => p.ratingChange != null && p.ratingChange < 0).length }; } module.exports = { getPlayerDataFromDB, scrapePDGARating, getPredictedRatingFromDB, getAllRatingsFromDB, refreshAllPlayersInDB, computeKpis, formatDisplayDate };