Samuel Enocsson
391 lines
13 KiB
JavaScript
391 lines
13 KiB
JavaScript
const { db } = require('../db');
|
|
const { getPlayerFromDB, getRoundHistoryFromDB, savePredictedRatingToDB, savePlayerToDB, getMonthlyHistory, getAllMonthlyHistoriesFromDB, getAllRatingHistoriesFromDB } = require('../models/player');
|
|
const { fetchPlayerDataHTTP, parsePlayerData } = require('../scrapers/player-http');
|
|
const { calculatePredictedRating, getPreviousPDGAUpdateDate } = require('./rating-calculator');
|
|
const logger = require('../logger');
|
|
|
|
function formatDisplayDate(dateStr) {
|
|
return new Date(dateStr).toLocaleDateString('en-US', {
|
|
day: '2-digit', month: 'short', year: 'numeric'
|
|
});
|
|
}
|
|
|
|
// Derives previous-month rating and the delta to it. Prefers PDGA's reported
|
|
// rating_change (canonical), falls back to our own monthly snapshots when
|
|
// rating_change is missing — common for players whose latest scrape failed.
|
|
function deriveMonthlyDeltas(rating, rawRatingChange, monthlyHistory) {
|
|
if (rating != null && rawRatingChange != null) {
|
|
return { lastMonthRating: rating - rawRatingChange, ratingChange: rawRatingChange };
|
|
}
|
|
if (rating != null && monthlyHistory && monthlyHistory.length >= 1) {
|
|
// The "last month" snapshot depends on whether current_rating is already in
|
|
// history. If equal, current is the most recent entry — last month is the one
|
|
// before it. If not, current is newer than history — the latest entry IS last month.
|
|
const lastIdx = monthlyHistory.length - 1;
|
|
const lastMonth = (monthlyHistory[lastIdx] === rating)
|
|
? (monthlyHistory.length >= 2 ? monthlyHistory[lastIdx - 1] : null)
|
|
: monthlyHistory[lastIdx];
|
|
if (lastMonth != null) {
|
|
return { lastMonthRating: lastMonth, ratingChange: rating - lastMonth };
|
|
}
|
|
}
|
|
return { lastMonthRating: null, ratingChange: rawRatingChange };
|
|
}
|
|
|
|
async function getPlayerDataFromDB(pdgaNumber, { includeMonthlyHistory = true } = {}) {
|
|
try {
|
|
const cachedPlayer = await getPlayerFromDB(pdgaNumber);
|
|
if (cachedPlayer) {
|
|
logger.debug(`Loading PDGA ${pdgaNumber} from DB (source of truth)`);
|
|
|
|
let predictedRating = cachedPlayer.predicted_rating;
|
|
let stdDev = cachedPlayer.std_dev;
|
|
let excludedRoundsCount = cachedPlayer.excluded_rounds_count;
|
|
let cutoffRating = cachedPlayer.cutoff_rating;
|
|
if (!predictedRating || predictedRating === 0) {
|
|
predictedRating = await getPredictedRatingFromDB(pdgaNumber);
|
|
const updatedPlayer = await getPlayerFromDB(pdgaNumber);
|
|
stdDev = updatedPlayer?.std_dev;
|
|
excludedRoundsCount = updatedPlayer?.excluded_rounds_count;
|
|
cutoffRating = updatedPlayer?.cutoff_rating;
|
|
}
|
|
|
|
// Staleness-check: invalidate cached predicted_rating if the PDGA cycle has
|
|
// rolled over since it was calculated. Don't recompute — round_history may be
|
|
// equally stale. UI will show "—" until the next manual refresh.
|
|
const predictedCalculatedAt = cachedPlayer.predicted_calculated_at
|
|
? new Date(cachedPlayer.predicted_calculated_at)
|
|
: null;
|
|
const previousUpdate = getPreviousPDGAUpdateDate();
|
|
const hasPredicted = predictedRating !== null && predictedRating !== 0;
|
|
const isStale = hasPredicted && (
|
|
predictedCalculatedAt === null || predictedCalculatedAt < previousUpdate
|
|
);
|
|
if (isStale) {
|
|
logger.debug(`PDGA ${pdgaNumber}: predicted rating stale (calculated ${predictedCalculatedAt?.toISOString() ?? 'unknown'}, cycle rolled ${previousUpdate.toISOString()})`);
|
|
predictedRating = null;
|
|
stdDev = null;
|
|
excludedRoundsCount = null;
|
|
cutoffRating = null;
|
|
}
|
|
|
|
const rating = cachedPlayer.current_rating;
|
|
const rawRatingChange = cachedPlayer.rating_change;
|
|
const resolvedPredicted = predictedRating > 0 ? predictedRating : null;
|
|
const resolvedStdDev = stdDev > 0 ? stdDev : null;
|
|
|
|
// Skip in bulk-fetch paths where caller supplies history via getAllMonthlyHistoriesFromDB
|
|
const monthlyHistory = includeMonthlyHistory
|
|
? await getMonthlyHistory(cachedPlayer.pdga_number)
|
|
: [];
|
|
|
|
const { lastMonthRating, ratingChange } = deriveMonthlyDeltas(rating, rawRatingChange, monthlyHistory);
|
|
|
|
return {
|
|
pdgaNumber: cachedPlayer.pdga_number,
|
|
name: cachedPlayer.name,
|
|
rating,
|
|
ratingChange,
|
|
predictedRating: resolvedPredicted,
|
|
stdDev: resolvedStdDev,
|
|
excludedRoundsCount: (excludedRoundsCount != null && excludedRoundsCount >= 0) ? excludedRoundsCount : null,
|
|
cutoffRating: (cutoffRating != null && cutoffRating > 0) ? cutoffRating : null,
|
|
lastMonthRating,
|
|
// gap between next predicted update and current rating (null when either is missing)
|
|
deltaPredicted: (resolvedPredicted != null && rating != null) ? resolvedPredicted - rating : null,
|
|
monthlyHistory
|
|
};
|
|
}
|
|
return null;
|
|
} catch (err) {
|
|
logger.error(`Database error for PDGA ${pdgaNumber}:`, err.message);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function scrapePDGARating(pdgaNumber, retries = 3) {
|
|
logger.info(`Refreshing PDGA ${pdgaNumber} from PDGA website`);
|
|
|
|
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
try {
|
|
logger.info(`Attempt ${attempt}/${retries} for PDGA ${pdgaNumber} (using HTTP)`);
|
|
|
|
const html = await fetchPlayerDataHTTP(pdgaNumber);
|
|
const result = parsePlayerData(html, pdgaNumber);
|
|
|
|
try {
|
|
await savePlayerToDB(result);
|
|
logger.info(`Saved PDGA ${pdgaNumber} to database`);
|
|
} catch (dbErr) {
|
|
logger.error(`Failed to save PDGA ${pdgaNumber} to database:`, dbErr.message);
|
|
}
|
|
|
|
logger.info(`Successfully scraped PDGA ${pdgaNumber} on attempt ${attempt}`);
|
|
return result;
|
|
|
|
} catch (error) {
|
|
logger.error(`Attempt ${attempt}/${retries} failed for PDGA ${pdgaNumber}:`, error.message);
|
|
|
|
if (attempt === retries) {
|
|
return {
|
|
pdgaNumber,
|
|
name: 'Error',
|
|
rating: 0,
|
|
ratingChange: null,
|
|
predictedRating: null
|
|
};
|
|
}
|
|
|
|
let retryDelay = 2000 * attempt;
|
|
|
|
if (error.rateLimitInfo) {
|
|
const retryAfter = error.rateLimitInfo.headers['retry-after'];
|
|
if (retryAfter) {
|
|
retryDelay = Math.max(retryDelay, (parseInt(retryAfter) + 1) * 1000);
|
|
logger.warn(`Using Retry-After header: waiting ${retryDelay/1000}s`);
|
|
}
|
|
}
|
|
|
|
if (error.code === 'ECONNRESET') {
|
|
retryDelay = Math.max(retryDelay, 10000);
|
|
logger.warn(`Connection reset detected: waiting ${retryDelay/1000}s`);
|
|
}
|
|
|
|
await new Promise(resolve => setTimeout(resolve, retryDelay));
|
|
}
|
|
}
|
|
}
|
|
|
|
async function getPredictedRatingFromDB(pdgaNumber) {
|
|
try {
|
|
const roundHistory = await getRoundHistoryFromDB(pdgaNumber);
|
|
if (roundHistory.length > 0) {
|
|
logger.debug(`Using ${roundHistory.length} cached rounds for PDGA ${pdgaNumber} prediction`);
|
|
|
|
const roundRatings = roundHistory.map(round => ({
|
|
rating: round.rating,
|
|
date: new Date(round.date),
|
|
competition: round.competition_name || 'Unknown'
|
|
}));
|
|
|
|
const result = calculatePredictedRating(roundRatings);
|
|
|
|
await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating);
|
|
|
|
return result.rating;
|
|
}
|
|
return 0;
|
|
} catch (err) {
|
|
logger.error(`Error getting predicted rating from DB for ${pdgaNumber}:`, err.message);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
async function getAllRatingsFromDB(progressCallback = null) {
|
|
try {
|
|
const allPlayers = await new Promise((resolve, reject) => {
|
|
db.all(
|
|
'SELECT pdga_number, name, current_rating, rating_change FROM players ORDER BY pdga_number',
|
|
[],
|
|
(err, rows) => {
|
|
if (err) reject(err);
|
|
else resolve(rows || []);
|
|
}
|
|
);
|
|
});
|
|
|
|
logger.info(`Loading ${allPlayers.length} players from database...`);
|
|
|
|
// Fetch all monthly histories in one query so the per-player loop doesn't add N extra queries
|
|
const monthlyHistoryMap = await getAllMonthlyHistoriesFromDB(12);
|
|
const ratingHistoryMap = await getAllRatingHistoriesFromDB();
|
|
|
|
const ratings = [];
|
|
const total = allPlayers.length;
|
|
|
|
for (let i = 0; i < allPlayers.length; i++) {
|
|
const player = allPlayers[i];
|
|
const pdgaNumber = player.pdga_number;
|
|
|
|
if (progressCallback) {
|
|
progressCallback({
|
|
current: i + 1,
|
|
total,
|
|
pdgaNumber,
|
|
status: 'loading'
|
|
});
|
|
}
|
|
|
|
try {
|
|
const playerData = await getPlayerDataFromDB(pdgaNumber, { includeMonthlyHistory: false });
|
|
|
|
if (playerData) {
|
|
playerData.monthlyHistory = monthlyHistoryMap.get(pdgaNumber) ?? [];
|
|
const rawHistory = ratingHistoryMap.get(pdgaNumber) ?? [];
|
|
playerData.ratingHistory = rawHistory.map(row => ({
|
|
date: row.date,
|
|
rating: row.rating,
|
|
displayDate: formatDisplayDate(row.date)
|
|
}));
|
|
// Re-derive now that history is attached — bulk path skipped includeMonthlyHistory
|
|
const derived = deriveMonthlyDeltas(playerData.rating, player.rating_change, playerData.monthlyHistory);
|
|
playerData.lastMonthRating = derived.lastMonthRating;
|
|
playerData.ratingChange = derived.ratingChange;
|
|
ratings.push(playerData);
|
|
}
|
|
|
|
if (progressCallback) {
|
|
progressCallback({
|
|
current: i + 1,
|
|
total,
|
|
pdgaNumber,
|
|
status: 'completed',
|
|
name: playerData ? playerData.name : player.name
|
|
});
|
|
}
|
|
} catch (error) {
|
|
logger.error(`Failed to load PDGA ${pdgaNumber} from database:`, error.message);
|
|
const errorRating = player.current_rating;
|
|
const errorRatingChange = player.rating_change;
|
|
const errorData = {
|
|
pdgaNumber: parseInt(pdgaNumber),
|
|
name: player.name || 'Database Error',
|
|
rating: errorRating,
|
|
ratingChange: errorRatingChange,
|
|
predictedRating: null,
|
|
stdDev: null,
|
|
excludedRoundsCount: null,
|
|
cutoffRating: null,
|
|
lastMonthRating: (errorRating != null && errorRatingChange != null) ? errorRating - errorRatingChange : null,
|
|
deltaPredicted: null,
|
|
monthlyHistory: [],
|
|
ratingHistory: []
|
|
};
|
|
ratings.push(errorData);
|
|
|
|
if (progressCallback) {
|
|
progressCallback({
|
|
current: i + 1,
|
|
total,
|
|
pdgaNumber,
|
|
status: 'error',
|
|
name: player.name || 'Database Error'
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0));
|
|
} catch (error) {
|
|
logger.error('Error loading players from database:', error);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
async function refreshAllPlayersInDB(progressCallback = null) {
|
|
try {
|
|
const allPlayers = await new Promise((resolve, reject) => {
|
|
db.all(
|
|
'SELECT pdga_number, name FROM players ORDER BY pdga_number',
|
|
[],
|
|
(err, rows) => {
|
|
if (err) reject(err);
|
|
else resolve(rows || []);
|
|
}
|
|
);
|
|
});
|
|
|
|
logger.info(`Refreshing ${allPlayers.length} players from database...`);
|
|
|
|
const ratings = [];
|
|
const total = allPlayers.length;
|
|
|
|
for (let i = 0; i < allPlayers.length; i++) {
|
|
const player = allPlayers[i];
|
|
const pdgaNumber = player.pdga_number;
|
|
|
|
logger.info(`Refreshing PDGA ${pdgaNumber}... (${i + 1}/${total})`);
|
|
|
|
if (progressCallback) {
|
|
progressCallback({
|
|
current: i + 1,
|
|
total,
|
|
pdgaNumber,
|
|
status: 'loading'
|
|
});
|
|
}
|
|
|
|
try {
|
|
const playerData = await scrapePDGARating(pdgaNumber);
|
|
ratings.push(playerData);
|
|
|
|
if (progressCallback) {
|
|
progressCallback({
|
|
current: i + 1,
|
|
total,
|
|
pdgaNumber,
|
|
status: 'completed',
|
|
name: playerData.name
|
|
});
|
|
}
|
|
|
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
} catch (error) {
|
|
logger.error(`Failed to refresh PDGA ${pdgaNumber}:`, error.message);
|
|
const errorData = {
|
|
pdgaNumber: parseInt(pdgaNumber),
|
|
name: player.name || 'Error',
|
|
rating: 0,
|
|
ratingChange: null,
|
|
predictedRating: null
|
|
};
|
|
ratings.push(errorData);
|
|
|
|
if (progressCallback) {
|
|
progressCallback({
|
|
current: i + 1,
|
|
total,
|
|
pdgaNumber,
|
|
status: 'error',
|
|
name: player.name || 'Error'
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0));
|
|
} catch (error) {
|
|
logger.error('Error refreshing all players:', error);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Aggregates KPI summary stats from an already-fetched player array.
|
|
* All fields are derived from the player list — no extra DB queries.
|
|
*/
|
|
function computeKpis(players) {
|
|
const active = players.filter(p => p.rating != null && p.rating > 0);
|
|
const avg = active.length > 0
|
|
? Math.round(active.reduce((sum, p) => sum + p.rating, 0) / active.length)
|
|
: null;
|
|
|
|
return {
|
|
tracked: players.length,
|
|
active: active.length,
|
|
avg,
|
|
climbing: players.filter(p => p.ratingChange != null && p.ratingChange > 0).length,
|
|
slipping: players.filter(p => p.ratingChange != null && p.ratingChange < 0).length
|
|
};
|
|
}
|
|
|
|
module.exports = {
|
|
getPlayerDataFromDB,
|
|
scrapePDGARating,
|
|
getPredictedRatingFromDB,
|
|
getAllRatingsFromDB,
|
|
refreshAllPlayersInDB,
|
|
computeKpis,
|
|
formatDisplayDate
|
|
};
|