shcizo/pdga-rating
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add Pino structured logging, release-please CI/CD and Docker pipeline

Browse Source 
Replace all console.log/error with Pino logger (info/warn/error/debug/fatal)
for structured JSON logging in production and pretty-print in development.
Remove redundant header dumps and consolidate rate-limit logging.

Add GitHub Actions workflow with release-please for automated semver releases
and Docker build/push to GHCR on new releases.
This commit is contained in:
Samuel Enocsson
2026-02-21 15:56:57 +01:00
parent 371a398446
commit 6ac32457a9
14 changed files with 498 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/db.js
+24 -23
View File
@@ -1,4 +1,5 @@
const sqlite3 = require('sqlite3').verbose();
const logger = require('./logger');
const dbPath = process.env.DB_PATH || './ratings.db';
const db = new sqlite3.Database(dbPath);
@@ -20,13 +21,13 @@ function initializeDatabase() {
db.get("PRAGMA table_info(players)", (err, info) => {
if (err) {
console.error('Error checking table schema:', err);
logger.error('Error checking table schema:', err);
return;
}
db.all("PRAGMA table_info(players)", (err, columns) => {
if (err) {
console.error('Error getting table info:', err);
logger.error('Error getting table info:', err);
return;
}
@@ -35,26 +36,26 @@ function initializeDatabase() {
const hasStdDev = columns.some(col => col.name === 'std_dev');
if (!hasLastRoundUpdate) {
console.log('Adding last_round_update column to players table...');
logger.info('Adding last_round_update column to players table...');
db.run(`ALTER TABLE players ADD COLUMN last_round_update DATETIME DEFAULT NULL`, (err) => {
if (err) console.error('Error adding last_round_update column:', err.message);
else console.log('Successfully added last_round_update column');
if (err) logger.error('Error adding last_round_update column:', err.message);
else logger.info('Successfully added last_round_update column');
});
}
if (!hasPredictedRating) {
console.log('Adding predicted_rating column to players table...');
logger.info('Adding predicted_rating column to players table...');
db.run(`ALTER TABLE players ADD COLUMN predicted_rating INTEGER DEFAULT NULL`, (err) => {
if (err) console.error('Error adding predicted_rating column:', err.message);
else console.log('Successfully added predicted_rating column');
if (err) logger.error('Error adding predicted_rating column:', err.message);
else logger.info('Successfully added predicted_rating column');
});
}
if (!hasStdDev) {
console.log('Adding std_dev column to players table...');
logger.info('Adding std_dev column to players table...');
db.run(`ALTER TABLE players ADD COLUMN std_dev INTEGER DEFAULT NULL`, (err) => {
if (err) console.error('Error adding std_dev column:', err.message);
else console.log('Successfully added std_dev column');
if (err) logger.error('Error adding std_dev column:', err.message);
else logger.info('Successfully added std_dev column');
});
}
});
@@ -111,7 +112,7 @@ function initializeDatabase() {
db.run(`ALTER TABLE layouts ADD COLUMN rating_count INTEGER DEFAULT 0`, () => {
db.run(`ALTER TABLE layouts ADD COLUMN last_calculated DATETIME`, () => {
db.run(`ALTER TABLE layouts ADD COLUMN last_played DATE`, () => {
console.log('Database initialized successfully');
logger.info('Database initialized successfully');
resolve();
});
});
@@ -136,47 +137,47 @@ async function checkAndPopulateDatabase() {
});
if (playerCount > 0) {
console.log(`✓ Database already has ${playerCount} players - skipping text file import`);
console.log('📝 Note: pdga-numbers.txt is only used when database is empty');
logger.info(`✓ Database already has ${playerCount} players - skipping text file import`);
logger.info('📝 Note: pdga-numbers.txt is only used when database is empty');
return;
}
console.log('=== Database is empty - populating from PDGA numbers file ===');
logger.info('=== Database is empty - populating from PDGA numbers file ===');
const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8')
.split('\n')
.map(num => num.trim())
.filter(num => num);
console.log(`Found ${pdgaNumbers.length} PDGA numbers in file`);
logger.info(`Found ${pdgaNumbers.length} PDGA numbers in file`);
if (pdgaNumbers.length === 0) {
console.log('⚠ No PDGA numbers found in file');
logger.info('⚠ No PDGA numbers found in file');
return;
}
console.log('Populating database with players from file...');
logger.info('Populating database with players from file...');
for (let i = 0; i < pdgaNumbers.length; i++) {
const pdgaNumber = pdgaNumbers[i];
console.log(`[${i + 1}/${pdgaNumbers.length}] Adding PDGA ${pdgaNumber}...`);
logger.info(`[${i + 1}/${pdgaNumbers.length}] Adding PDGA ${pdgaNumber}...`);
try {
const playerData = await scrapePDGARating(pdgaNumber);
console.log(` ✓ Added ${playerData.name}`);
logger.info(` ✓ Added ${playerData.name}`);
if (i < pdgaNumbers.length - 1) {
await new Promise(resolve => setTimeout(resolve, 2000));
}
} catch (error) {
console.error(` ✗ Failed to add PDGA ${pdgaNumber}:`, error.message);
logger.error(` ✗ Failed to add PDGA ${pdgaNumber}:`, error.message);
}
}
console.log('=== Database population complete ===');
logger.info('=== Database population complete ===');
} catch (error) {
console.error('Error during database population check:', error.message);
logger.error('Error during database population check:', error.message);
}
}
src/logger.js
+10
View File
@@ -0,0 +1,10 @@
const pino = require('pino');
const logger = pino({
level: process.env.LOG_LEVEL || 'info',
transport: process.env.NODE_ENV !== 'production'
? { target: 'pino-pretty' }
: undefined
});
module.exports = logger;
src/routes/courses.js
+27 -26
View File
@@ -4,6 +4,7 @@ const { db } = require('../db');
const { getAllCoursesFromDB, getLayoutsForCourse, updateLayoutRating } = require('../models/course');
const { launchBrowser } = require('../scrapers/browser');
const { layoutEventCache, scrapeCourseDirectory, scrapeCourseLayouts, scrapeEventResults } = require('../scrapers/course-puppeteer');
const logger = require('../logger');
// Request locking to prevent concurrent scrapes of the same resource
const activeScrapes = new Map();
@@ -33,7 +34,7 @@ router.get('/partials/course-layouts/:courseId', async (req, res) => {
const layouts = await getLayoutsForCourse(courseId);
res.render('../partials/course-layouts', { layouts, courseId });
} catch (error) {
console.error('Error loading course layouts:', error.message);
logger.error('Error loading course layouts:', error.message);
res.status(500).send('<div class="no-layouts">Error loading layouts</div>');
}
});
@@ -43,7 +44,7 @@ router.get('/api/courses', async (req, res) => {
const courses = await getAllCoursesFromDB();
res.json(courses);
} catch (error) {
console.error('Error fetching courses:', error.message);
logger.error('Error fetching courses:', error.message);
res.status(500).json({ error: 'Failed to fetch courses' });
}
});
@@ -54,7 +55,7 @@ router.get('/api/layouts/:courseId', async (req, res) => {
const layouts = await getLayoutsForCourse(courseId);
res.json(layouts);
} catch (error) {
console.error('Error fetching layouts:', error.message);
logger.error('Error fetching layouts:', error.message);
res.status(500).json({ error: 'Failed to fetch layouts' });
}
});
@@ -65,7 +66,7 @@ router.post('/api/scrape-courses', async (req, res) => {
let browser = null;
try {
console.log('Starting course directory scraping...');
logger.info('Starting course directory scraping...');
browser = await launchBrowser();
@@ -80,7 +81,7 @@ router.post('/api/scrape-courses', async (req, res) => {
message: `Successfully scraped ${courses.length} courses`
});
} catch (error) {
console.error('Error scraping courses:', error.message);
logger.error('Error scraping courses:', error.message);
if (browser) {
try { await browser.close(); } catch (e) {}
}
@@ -96,7 +97,7 @@ router.post('/api/scrape-layouts/:courseId', async (req, res) => {
const lockKey = `layout-${courseId}`;
if (activeScrapes.has(lockKey)) {
console.log(`⚠️ Scrape already in progress for course ${courseId}`);
logger.info(`⚠️ Scrape already in progress for course ${courseId}`);
return res.status(409).json({
error: 'Scrape already in progress for this course',
message: 'Please wait for the current scrape to complete'
@@ -118,19 +119,19 @@ router.post('/api/scrape-layouts/:courseId', async (req, res) => {
throw new Error('Course not found');
}
console.log(`Starting layout scraping for course: ${course.name}`);
logger.info(`Starting layout scraping for course: ${course.name}`);
browser = await launchBrowser();
const layouts = await scrapeCourseLayouts(browser, course.link, courseId);
console.log(`\n=== Starting event results scraping for ${course.name} ===`);
logger.info(`\n=== Starting event results scraping for ${course.name} ===`);
const courseIdInt = parseInt(courseId);
const layoutData = layoutEventCache.get(courseIdInt);
if (!layoutData || layoutData.length === 0) {
console.log('No event data found in cache, skipping event results scraping');
logger.info('No event data found in cache, skipping event results scraping');
await browser.close();
browser = null;
@@ -183,7 +184,7 @@ router.post('/api/scrape-layouts/:courseId', async (req, res) => {
await new Promise(resolve => setTimeout(resolve, 2000));
}
console.log(`\n=== Calculating final ratings for all layouts ===`);
logger.info(`\n=== Calculating final ratings for all layouts ===`);
let savedCount = 0;
for (const layoutKey in allLayoutRatings) {
@@ -194,10 +195,10 @@ router.post('/api/scrape-layouts/:courseId', async (req, res) => {
layoutDataResult.allRatings.reduce((sum, r) => sum + r, 0) / layoutDataResult.allRatings.length
);
console.log(`Layout: ${layoutDataResult.name} (Par ${layoutDataResult.par})`);
console.log(` Total ratings collected: ${layoutDataResult.allRatings.length}`);
console.log(` Mean rating: ${meanRating}`);
console.log(` Last played: ${layoutDataResult.latestDate || 'Unknown'}`);
logger.debug(`Layout: ${layoutDataResult.name} (Par ${layoutDataResult.par})`);
logger.debug(` Total ratings collected: ${layoutDataResult.allRatings.length}`);
logger.debug(` Mean rating: ${meanRating}`);
logger.debug(` Last played: ${layoutDataResult.latestDate || 'Unknown'}`);
try {
const changes = await updateLayoutRating(
@@ -209,11 +210,11 @@ router.post('/api/scrape-layouts/:courseId', async (req, res) => {
layoutDataResult.latestDate
);
if (changes > 0) {
console.log(` ✓ Updated in database`);
logger.info(` ✓ Updated in database`);
savedCount++;
}
} catch (err) {
console.error(` Error updating layout ${layoutDataResult.name}:`, err.message);
logger.error(` Error updating layout ${layoutDataResult.name}:`, err.message);
}
}
}
@@ -229,7 +230,7 @@ router.post('/api/scrape-layouts/:courseId', async (req, res) => {
message: `Successfully scraped ${layouts.length} layouts and processed ${Object.keys(eventGroups).length} events for ${course.name}`
};
} catch (error) {
console.error('Error scraping layouts:', error.message);
logger.error('Error scraping layouts:', error.message);
if (browser) {
try { await browser.close(); } catch (e) {}
}
@@ -249,7 +250,7 @@ router.post('/api/scrape-layouts/:courseId', async (req, res) => {
});
} finally {
activeScrapes.delete(lockKey);
console.log(`✓ Released lock for course ${courseId}`);
logger.info(`✓ Released lock for course ${courseId}`);
}
});
@@ -317,7 +318,7 @@ router.post('/api/scrape-event-results/:courseId', async (req, res) => {
await browser.close();
browser = null;
console.log(`\n=== Calculating final ratings for all layouts ===`);
logger.info(`\n=== Calculating final ratings for all layouts ===`);
let savedCount = 0;
for (const layoutKey in allLayoutRatings) {
@@ -328,10 +329,10 @@ router.post('/api/scrape-event-results/:courseId', async (req, res) => {
ld.allRatings.reduce((sum, r) => sum + r, 0) / ld.allRatings.length
);
console.log(`Layout: ${ld.name} (Par ${ld.par})`);
console.log(` Total ratings collected: ${ld.allRatings.length}`);
console.log(` Mean rating: ${meanRating}`);
console.log(` Last played: ${ld.latestDate || 'Unknown'}`);
logger.debug(`Layout: ${ld.name} (Par ${ld.par})`);
logger.debug(` Total ratings collected: ${ld.allRatings.length}`);
logger.debug(` Mean rating: ${meanRating}`);
logger.debug(` Last played: ${ld.latestDate || 'Unknown'}`);
try {
const changes = await updateLayoutRating(
@@ -343,11 +344,11 @@ router.post('/api/scrape-event-results/:courseId', async (req, res) => {
ld.latestDate
);
if (changes > 0) {
console.log(` ✓ Updated in database`);
logger.info(` ✓ Updated in database`);
savedCount++;
}
} catch (err) {
console.error(` Error updating layout ${ld.name}:`, err.message);
logger.error(` Error updating layout ${ld.name}:`, err.message);
}
}
}
@@ -360,7 +361,7 @@ router.post('/api/scrape-event-results/:courseId', async (req, res) => {
message: `Processed ${Object.keys(eventGroups).length} events, updated ${savedCount} layouts`
});
} catch (error) {
console.error('Error scraping event results:', error.message);
logger.error('Error scraping event results:', error.message);
if (browser) {
try { await browser.close(); } catch (e) {}
}
src/routes/players.js
+41 -40
View File
@@ -7,6 +7,7 @@ const { getOfficialRatingHistory, getOptimizedPlayerRounds } = require('../scrap
const { launchBrowser } = require('../scrapers/browser');
const { getPlayerDataFromDB, scrapePDGARating, getAllRatingsFromDB, refreshAllPlayersInDB, getPredictedRatingFromDB } = require('../services/player-service');
const { calculatePredictedRating } = require('../services/rating-calculator');
const logger = require('../logger');
router.get('/partials/ratings-table', async (req, res) => {
try {
@@ -28,7 +29,7 @@ router.get('/partials/player-history/:pdgaNumber', async (req, res) => {
try {
await saveRatingHistoryToDB(pdgaNumber, history);
} catch (dbErr) {
console.error('Failed to save rating history:', dbErr.message);
logger.error('Failed to save rating history:', dbErr.message);
}
}
@@ -40,7 +41,7 @@ router.get('/partials/player-history/:pdgaNumber', async (req, res) => {
res.render('../partials/player-history', { pdgaNumber, history: formattedHistory });
} catch (error) {
console.error('Error loading player history:', error.message);
logger.error('Error loading player history:', error.message);
res.status(500).send('<div class="loading-chart">Error loading rating history</div>');
}
});
@@ -92,14 +93,14 @@ router.post('/api/populate-database', (req, res) => {
res.write(`data: ${JSON.stringify(progress)}\n\n`);
};
console.log('=== Starting database population from database players ===');
logger.info('=== Starting database population from database players ===');
refreshAllPlayersInDB(progressCallback).then(ratings => {
console.log(`=== Database population complete: ${ratings.length} players refreshed ===`);
logger.info(`=== Database population complete: ${ratings.length} players refreshed ===`);
res.write(`data: ${JSON.stringify({ status: 'complete', ratings, message: `Successfully refreshed ${ratings.length} players` })}\n\n`);
res.end();
}).catch(error => {
console.error('Error populating database:', error);
logger.error('Error populating database:', error);
res.write(`data: ${JSON.stringify({ status: 'error', message: error.message })}\n\n`);
res.end();
});
@@ -155,7 +156,7 @@ router.get('/api/rating-history/:pdgaNumber', async (req, res) => {
const cachedHistory = await getRatingHistoryFromDB(pdgaNumber);
if (cachedHistory && cachedHistory.length > 0) {
console.log(`Using cached rating history from DB for PDGA ${pdgaNumber}`);
logger.info(`Using cached rating history from DB for PDGA ${pdgaNumber}`);
const formattedHistory = cachedHistory.map(row => ({
date: row.date,
rating: row.rating,
@@ -173,15 +174,15 @@ router.get('/api/rating-history/:pdgaNumber', async (req, res) => {
return;
}
console.log(`Fetching rating history for PDGA ${pdgaNumber}...`);
logger.info(`Fetching rating history for PDGA ${pdgaNumber}...`);
const html = await fetchRatingHistory(pdgaNumber);
const history = parseRatingHistory(html);
try {
await saveRatingHistoryToDB(pdgaNumber, history);
console.log(`Saved rating history for PDGA ${pdgaNumber} to database`);
logger.info(`Saved rating history for PDGA ${pdgaNumber} to database`);
} catch (dbErr) {
console.error(`Failed to save rating history to database:`, dbErr.message);
logger.error(`Failed to save rating history to database:`, dbErr.message);
}
res.json({
@@ -189,7 +190,7 @@ router.get('/api/rating-history/:pdgaNumber', async (req, res) => {
history
});
} catch (error) {
console.error('Error fetching rating history:', error.message);
logger.error('Error fetching rating history:', error.message);
res.status(500).json({ error: 'Failed to fetch rating history' });
}
});
@@ -198,19 +199,19 @@ router.post('/api/clear-cache', (req, res) => {
try {
db.run('UPDATE players SET last_updated = datetime("now", "-25 hours"), last_round_update = NULL', (err) => {
if (err) {
console.error('Error clearing database cache:', err);
logger.error('Error clearing database cache:', err);
res.status(500).json({ error: 'Failed to clear database cache' });
return;
}
console.log('Database cache cleared - all players will be refreshed on next request');
logger.info('Database cache cleared - all players will be refreshed on next request');
res.json({
success: true,
message: 'Cache cleared - database reset'
});
});
} catch (error) {
console.error('Error clearing cache:', error);
logger.error('Error clearing cache:', error);
res.status(500).json({ error: 'Failed to clear cache' });
}
});
@@ -218,7 +219,7 @@ router.post('/api/clear-cache', (req, res) => {
router.get('/api/search-player/:pdgaNumber', async (req, res) => {
try {
const { pdgaNumber } = req.params;
console.log(`Searching for player with PDGA number ${pdgaNumber}`);
logger.info(`Searching for player with PDGA number ${pdgaNumber}`);
const existingPlayer = await getPlayerFromDB(pdgaNumber);
if (existingPlayer) {
@@ -245,7 +246,7 @@ router.get('/api/search-player/:pdgaNumber', async (req, res) => {
player: playerData
});
} catch (error) {
console.error('Error searching for player:', error.message);
logger.error('Error searching for player:', error.message);
res.status(500).json({ error: 'Failed to search for player' });
}
});
@@ -258,7 +259,7 @@ router.post('/api/add-player', async (req, res) => {
return res.status(400).json({ error: 'PDGA number is required' });
}
console.log(`Adding player with PDGA number ${pdgaNumber}`);
logger.info(`Adding player with PDGA number ${pdgaNumber}`);
const existingPlayer = await getPlayerFromDB(pdgaNumber);
if (existingPlayer) {
@@ -281,14 +282,14 @@ router.post('/api/add-player', async (req, res) => {
await savePlayerToDB(playerData);
console.log(`Successfully added player: ${playerData.name} (#${pdgaNumber})`);
logger.info(`Successfully added player: ${playerData.name} (#${pdgaNumber})`);
res.json({
success: true,
player: playerData
});
} catch (error) {
console.error('Error adding player:', error.message);
logger.error('Error adding player:', error.message);
res.status(500).json({ error: 'Failed to add player' });
}
});
@@ -296,7 +297,7 @@ router.post('/api/add-player', async (req, res) => {
router.post('/api/refresh-player/:pdgaNumber', async (req, res) => {
try {
const { pdgaNumber } = req.params;
console.log(`Manually refreshing player data for PDGA ${pdgaNumber}`);
logger.info(`Manually refreshing player data for PDGA ${pdgaNumber}`);
const html = await fetchPlayerDataHTTP(pdgaNumber);
const playerData = parsePlayerData(html, pdgaNumber);
@@ -308,7 +309,7 @@ router.post('/api/refresh-player/:pdgaNumber', async (req, res) => {
player: playerData
});
} catch (error) {
console.error('Error refreshing player data:', error.message);
logger.error('Error refreshing player data:', error.message);
res.status(500).json({ error: 'Failed to refresh player data' });
}
});
@@ -316,31 +317,31 @@ router.post('/api/refresh-player/:pdgaNumber', async (req, res) => {
router.post('/api/refresh-rating-history/:pdgaNumber', async (req, res) => {
try {
const { pdgaNumber } = req.params;
console.log(`=== Manually refreshing rating history for PDGA ${pdgaNumber} ===`);
logger.info(`=== Manually refreshing rating history for PDGA ${pdgaNumber} ===`);
const startTime = Date.now();
const html = await fetchRatingHistory(pdgaNumber);
const fetchTime = Date.now() - startTime;
console.log(`HTML fetch completed in ${fetchTime}ms, received ${html.length} bytes`);
logger.info(`HTML fetch completed in ${fetchTime}ms, received ${html.length} bytes`);
const parseStartTime = Date.now();
const history = parseRatingHistory(html);
const parseTime = Date.now() - parseStartTime;
console.log(`Parsing completed in ${parseTime}ms, found ${history.length} history entries`);
logger.info(`Parsing completed in ${parseTime}ms, found ${history.length} history entries`);
if (history.length > 0) {
console.log('Sample history entries:', history.slice(0, 3));
logger.debug({ entries: history.slice(0, 3) }, 'Sample history entries');
} else {
console.log('No history entries found. HTML sample:', html.substring(0, 500));
logger.debug({ htmlSample: html.substring(0, 500) }, 'No history entries found');
}
const dbStartTime = Date.now();
await saveRatingHistoryToDB(pdgaNumber, history);
const dbTime = Date.now() - dbStartTime;
console.log(`Database save completed in ${dbTime}ms`);
logger.info(`Database save completed in ${dbTime}ms`);
const formattedHistory = history.map(entry => ({
date: entry.date,
@@ -348,16 +349,16 @@ router.post('/api/refresh-rating-history/:pdgaNumber', async (req, res) => {
displayDate: entry.displayDate
}));
console.log(`=== Rating history refresh completed for PDGA ${pdgaNumber} ===`);
logger.info(`=== Rating history refresh completed for PDGA ${pdgaNumber} ===`);
res.json({
success: true,
history: formattedHistory
});
} catch (error) {
console.error(`=== Error refreshing rating history for PDGA ${req.params.pdgaNumber} ===`);
console.error('Error type:', error.constructor.name);
console.error('Error message:', error.message);
logger.error(`=== Error refreshing rating history for PDGA ${req.params.pdgaNumber} ===`);
logger.error('Error type:', error.constructor.name);
logger.error('Error message:', error.message);
res.status(500).json({
error: 'Failed to refresh rating history',
@@ -392,7 +393,7 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
const isIncremental = !!sinceDate;
console.log(`${isIncremental ? 'Incrementally updating' : 'Fully refreshing'} round history for PDGA ${pdgaNumber}${sinceDate ? ` since ${sinceDate.toDateString()}` : ''}`);
logger.info(`${isIncremental ? 'Incrementally updating' : 'Fully refreshing'} round history for PDGA ${pdgaNumber}${sinceDate ? ` since ${sinceDate.toDateString()}` : ''}`);
browser = await launchBrowser();
@@ -403,13 +404,13 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
await saveRatingHistoryToDB(pdgaNumber, officialHistory);
}
} catch (historyError) {
console.error('Failed to fetch official history:', historyError.message);
logger.error('Failed to fetch official history:', historyError.message);
officialHistory = [];
}
let allRounds = [];
try {
console.log(`Using optimized approach: /details + new tournaments only for PDGA ${pdgaNumber}...`);
logger.info(`Using optimized approach: /details + new tournaments only for PDGA ${pdgaNumber}...`);
allRounds = await getOptimizedPlayerRounds(browser, pdgaNumber);
if (allRounds.length > 0) {
@@ -420,14 +421,14 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
}));
await saveRoundHistoryToDB(pdgaNumber, roundsForDB, false);
console.log(`✓ Saved ${allRounds.length} rounds using optimized approach`);
logger.info(`✓ Saved ${allRounds.length} rounds using optimized approach`);
await updateLastRoundUpdateDate(pdgaNumber);
} else {
console.log('ℹ No rounds found');
logger.info('ℹ No rounds found');
}
} catch (detailsError) {
console.error('Failed to fetch rounds using optimized approach:', detailsError.message);
logger.error('Failed to fetch rounds using optimized approach:', detailsError.message);
allRounds = [];
}
@@ -460,15 +461,15 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
message: `Used /details (${officialCount} rounds) + new tournaments (${newCount} rounds)`
});
} catch (error) {
console.error(`=== Error refreshing round history for PDGA ${pdgaNumber} ===`);
console.error('Error type:', error.constructor.name);
console.error('Error message:', error.message);
logger.error(`=== Error refreshing round history for PDGA ${pdgaNumber} ===`);
logger.error('Error type:', error.constructor.name);
logger.error('Error message:', error.message);
if (browser) {
try {
await browser.close();
} catch (closeError) {
console.error('Error closing browser:', closeError.message);
logger.error('Error closing browser:', closeError.message);
}
}
src/scrapers/course-puppeteer.js
+19 -18
View File
@@ -1,4 +1,5 @@
const { saveCourseToDB, saveLayoutToDB } = require('../models/course');
const logger = require('../logger');
// In-memory cache for layout-division-event mapping
const layoutEventCache = new Map();
@@ -8,7 +9,7 @@ function getLayoutEventCache() {
}
async function scrapeCourseDirectory(browser) {
console.log('=== Scraping Swedish courses from PDGA course directory ===');
logger.info('Scraping Swedish courses from PDGA course directory');
const page = await browser.newPage();
const allCourses = [];
let pageNumber = 0;
@@ -18,7 +19,7 @@ async function scrapeCourseDirectory(browser) {
while (hasMorePages) {
const url = `https://www.pdga.com/course-directory/advanced?title=&field_course_location_country=SE&field_course_location_locality=&field_course_location_administrative_area=All&field_course_location_postal_code=&field_course_type_value=All&rating_value=All&field_course_holes_value=18-100&field_course_total_length_value=All&field_course_target_type_value=All&field_course_tee_type_value=All&field_location_type_value=All&field_course_camping_value=All&field_course_facilities_value=All&field_course_fees_value=All&field_course_handicap_value=All&field_course_private_value=All&field_course_signage_value=All&field_cart_friendly_value=All&page=${pageNumber}`;
console.log(`Scraping page ${pageNumber}...`);
logger.info(`Scraping page ${pageNumber}...`);
await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 });
await page.waitForTimeout(1000);
@@ -46,34 +47,34 @@ async function scrapeCourseDirectory(browser) {
});
if (courses.length === 0) {
console.log(`No courses found on page ${pageNumber}, stopping pagination`);
logger.info(`No courses found on page ${pageNumber}, stopping pagination`);
hasMorePages = false;
} else {
console.log(`Found ${courses.length} courses on page ${pageNumber}`);
logger.info(`Found ${courses.length} courses on page ${pageNumber}`);
allCourses.push(...courses);
for (const course of courses) {
try {
await saveCourseToDB(course);
console.log(`✓ Saved course: ${course.name} (${course.city})`);
logger.info(`Saved course: ${course.name} (${course.city})`);
} catch (err) {
console.error(`Error saving course ${course.name}:`, err.message);
logger.error(`Error saving course ${course.name}: ${err.message}`);
}
}
pageNumber++;
if (hasMorePages) {
console.log('Waiting 2s before next page...');
logger.info('Waiting 2s before next page...');
await new Promise(resolve => setTimeout(resolve, 2000));
}
}
}
console.log(`✓ Total courses scraped: ${allCourses.length} across ${pageNumber} pages`);
logger.info(`Total courses scraped: ${allCourses.length} across ${pageNumber} pages`);
} catch (error) {
console.error('Error scraping course directory:', error.message);
logger.error('Error scraping course directory: ' + error.message);
} finally {
await page.close();
}
@@ -82,7 +83,7 @@ async function scrapeCourseDirectory(browser) {
}
async function scrapeCourseLayouts(browser, courseLink, courseId) {
console.log(`\n=== Scraping layouts from: ${courseLink} ===`);
logger.info(`Scraping layouts from: ${courseLink}`);
const page = await browser.newPage();
const layouts = [];
@@ -114,10 +115,10 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) {
});
if (layoutsTabClicked) {
console.log('✓ Layouts tab found and clicked');
logger.info('Layouts tab found and clicked');
await page.waitForTimeout(3000);
} else {
console.warn('⚠️ Layouts tab not found - may be on a single-layout course page');
logger.warn('Layouts tab not found - may be on a single-layout course page');
}
const extractedLayouts = await page.evaluate(() => {
@@ -198,7 +199,7 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) {
const courseIdInt = typeof courseId === 'string' ? parseInt(courseId) : courseId;
layoutEventCache.set(courseIdInt, layouts);
console.log(`✓ Successfully parsed ${layouts.length} layouts from course page`);
logger.info(`Successfully parsed ${layouts.length} layouts from course page`);
const uniqueLayouts = [];
const seen = new Set();
@@ -212,20 +213,20 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) {
}
if (uniqueLayouts.length < layouts.length) {
console.log(`ℹ️ Deduplicated to ${uniqueLayouts.length} unique layouts`);
logger.info(`Deduplicated to ${uniqueLayouts.length} unique layouts`);
}
for (const layout of uniqueLayouts) {
try {
await saveLayoutToDB(courseId, layout);
console.log(` ✓ Saved layout: ${layout.name} (Par ${layout.par})`);
logger.info(`Saved layout: ${layout.name} (Par ${layout.par})`);
} catch (err) {
console.error(` ✗ Error saving layout ${layout.name}:`, err.message);
logger.error(`Error saving layout ${layout.name}: ${err.message}`);
}
}
} catch (error) {
console.error('Error scraping course layouts:', error.message);
logger.error('Error scraping course layouts: ' + error.message);
} finally {
await page.close();
}
@@ -332,7 +333,7 @@ async function scrapeEventResults(browser, eventUrl, layoutsWithDivisions) {
}
} catch (error) {
console.error('Error scraping event results:', error.message);
logger.error('Error scraping event results: ' + error.message);
} finally {
await page.close();
}
src/scrapers/player-http.js
+26 -40
View File
@@ -1,4 +1,5 @@
const https = require('https');
const logger = require('../logger');
async function fetchPlayerDataHTTP(pdgaNumber) {
return new Promise((resolve, reject) => {
@@ -28,21 +29,14 @@ async function fetchPlayerDataHTTP(pdgaNumber) {
headers: res.headers
};
console.log(`PDGA Response Status for #${pdgaNumber}: ${res.statusCode}`);
console.log('Response Headers:', JSON.stringify(res.headers, null, 2));
logger.info(`PDGA Response Status for #${pdgaNumber}: ${res.statusCode}`);
if (res.headers['retry-after']) {
console.log(`Retry-After header: ${res.headers['retry-after']}`);
}
if (res.headers['x-ratelimit-limit']) {
console.log(`Rate Limit: ${res.headers['x-ratelimit-limit']}`);
}
if (res.headers['x-ratelimit-remaining']) {
console.log(`Rate Limit Remaining: ${res.headers['x-ratelimit-remaining']}`);
}
if (res.headers['x-ratelimit-reset']) {
console.log(`Rate Limit Reset: ${res.headers['x-ratelimit-reset']}`);
}
logger.debug({
retryAfter: res.headers['retry-after'],
rateLimit: res.headers['x-ratelimit-limit'],
rateLimitRemaining: res.headers['x-ratelimit-remaining'],
rateLimitReset: res.headers['x-ratelimit-reset']
}, `Rate limit details for #${pdgaNumber}`);
const error = new Error(`HTTP ${res.statusCode}`);
error.rateLimitInfo = rateLimitInfo;
@@ -52,10 +46,7 @@ async function fetchPlayerDataHTTP(pdgaNumber) {
});
req.on('error', (error) => {
console.log(`Request error for PDGA #${pdgaNumber}:`, error.code, error.message);
if (error.code === 'ECONNRESET') {
console.log('Connection reset - likely rate limited by PDGA');
}
logger.error(`Request error for PDGA #${pdgaNumber}: ${error.code} ${error.message}`);
reject(error);
});
@@ -88,7 +79,7 @@ function parsePlayerData(html, pdgaNumber) {
predictedRating: null
};
} catch (error) {
console.error(`Error parsing data for PDGA ${pdgaNumber}:`, error.message);
logger.error(`Error parsing data for PDGA ${pdgaNumber}: ${error.message}`);
return {
pdgaNumber,
name: 'Error',
@@ -112,7 +103,7 @@ async function fetchRatingHistory(pdgaNumber) {
timeout: 30000
};
console.log(`Fetching rating history for PDGA #${pdgaNumber} from: https://www.pdga.com/player/${pdgaNumber}/history`);
logger.info(`Fetching rating history for PDGA #${pdgaNumber} from: https://www.pdga.com/player/${pdgaNumber}/history`);
const req = https.request(options, (res) => {
let data = '';
@@ -122,25 +113,20 @@ async function fetchRatingHistory(pdgaNumber) {
res.on('end', () => {
if (res.statusCode === 200) {
console.log(`Rating history request successful for PDGA #${pdgaNumber}`);
logger.info(`Rating history request successful for PDGA #${pdgaNumber}`);
resolve(data);
} else {
console.log(`Rating History Error for PDGA #${pdgaNumber}:`);
console.log(`Status: ${res.statusCode}`);
console.log('Response Headers:', JSON.stringify(res.headers, null, 2));
logger.error(`Rating History Error for PDGA #${pdgaNumber}:`);
logger.error(`Status: ${res.statusCode}`);
if (res.headers['retry-after']) {
console.log(`Retry-After: ${res.headers['retry-after']} seconds`);
}
if (res.headers['x-ratelimit-limit']) {
console.log(`Rate Limit: ${res.headers['x-ratelimit-limit']}`);
}
if (res.headers['x-ratelimit-remaining']) {
console.log(`Rate Limit Remaining: ${res.headers['x-ratelimit-remaining']}`);
}
logger.debug({
retryAfter: res.headers['retry-after'],
rateLimit: res.headers['x-ratelimit-limit'],
rateLimitRemaining: res.headers['x-ratelimit-remaining']
}, `Rate limit details for history #${pdgaNumber}`);
if (data.length > 0) {
console.log(`Partial response received (${data.length} bytes):`, data.substring(0, 200));
logger.debug(`Partial response received (${data.length} bytes): ${data.substring(0, 200)}`);
}
const error = new Error(`HTTP ${res.statusCode} for rating history`);
@@ -152,28 +138,28 @@ async function fetchRatingHistory(pdgaNumber) {
});
req.on('error', (error) => {
console.log(`Rating history request error for PDGA #${pdgaNumber}:`, {
logger.error({
code: error.code,
message: error.message,
errno: error.errno,
syscall: error.syscall
});
}, `Rating history request error for PDGA #${pdgaNumber}`);
if (error.code === 'ECONNRESET') {
console.log('Connection reset on rating history - likely rate limited by PDGA');
logger.debug('Connection reset on rating history - likely rate limited by PDGA');
}
if (error.code === 'ECONNREFUSED') {
console.log('Connection refused - PDGA server may be blocking requests');
logger.debug('Connection refused - PDGA server may be blocking requests');
}
if (error.code === 'ETIMEDOUT') {
console.log('Request timed out - server may be overloaded');
logger.debug('Request timed out - server may be overloaded');
}
reject(error);
});
req.on('timeout', () => {
console.log(`Rating history request timeout for PDGA #${pdgaNumber} after 30s`);
logger.info(`Rating history request timeout for PDGA #${pdgaNumber} after 30s`);
req.destroy();
reject(new Error('Request timeout'));
});
src/scrapers/player-puppeteer.js
+20 -19
View File
@@ -1,4 +1,5 @@
const { parseDate } = require('../services/rating-calculator');
const logger = require('../logger');
async function getOfficialRatingHistory(browser, pdgaNumber) {
const page = await browser.newPage();
@@ -47,7 +48,7 @@ async function getOfficialRatingHistory(browser, pdgaNumber) {
});
} catch (error) {
console.error('Error fetching official rating history:', error.message);
logger.error('Error fetching official rating history: ' + error.message);
} finally {
await page.close();
}
@@ -123,7 +124,7 @@ async function getPlayerTournamentDetails(browser, pdgaNumber) {
}
}
} catch (e) {
console.log(`Date parsing failed for "${round.dateText}": ${e.message}`);
logger.info(`Date parsing failed for "${round.dateText}": ${e.message}`);
}
}
return {
@@ -137,7 +138,7 @@ async function getPlayerTournamentDetails(browser, pdgaNumber) {
tournamentRounds = fixedRounds;
} catch (error) {
console.error('Error fetching tournament details:', error.message);
logger.error('Error fetching tournament details: ' + error.message);
} finally {
await page.close();
}
@@ -153,7 +154,7 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) {
const url = `https://www.pdga.com/player/${pdgaNumber}`;
await page.goto(url, { waitUntil: 'networkidle2' });
console.log(`Looking for tournaments after ${afterDate.toDateString()}...`);
logger.info(`Looking for tournaments after ${afterDate.toDateString()}...`);
const newTournamentUrls = await page.evaluate((afterTimestamp) => {
const afterDate = new Date(afterTimestamp);
@@ -192,11 +193,11 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) {
return urls;
}, afterDate.getTime());
console.log(`Found ${newTournamentUrls.length} new tournaments after ${afterDate.toDateString()}`);
logger.info(`Found ${newTournamentUrls.length} new tournaments after ${afterDate.toDateString()}`);
for (const tournamentData of newTournamentUrls) {
try {
console.log(`Scraping new tournament: ${tournamentData.name} (${tournamentData.date})`);
logger.info(`Scraping new tournament: ${tournamentData.name} (${tournamentData.date})`);
await page.goto(tournamentData.url, { waitUntil: 'domcontentloaded', timeout: 30000 });
await page.waitForTimeout(500);
@@ -238,16 +239,16 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) {
});
});
console.log(`✓ Found ${roundRatings.length} round ratings for ${tournamentData.name}`);
logger.info(`Found ${roundRatings.length} round ratings for ${tournamentData.name}`);
}
} catch (error) {
console.error(`Error scraping tournament ${tournamentData.name}:`, error.message);
logger.error(`Error scraping tournament ${tournamentData.name}: ${error.message}`);
}
}
} catch (error) {
console.error(`Error getting new tournament rounds for PDGA ${pdgaNumber}:`, error);
logger.error(`Error getting new tournament rounds for PDGA ${pdgaNumber}: ${error.message}`);
} finally {
await page.close();
}
@@ -256,30 +257,30 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) {
}
async function getOptimizedPlayerRounds(browser, pdgaNumber) {
console.log(`=== Optimized Round Collection for PDGA ${pdgaNumber} ===`);
logger.info(`Optimized Round Collection for PDGA ${pdgaNumber}`);
try {
console.log('Step 1: Getting official rating rounds from /details page...');
logger.info('Getting official rating rounds from /details page...');
const officialRounds = await getPlayerTournamentDetails(browser, pdgaNumber);
if (officialRounds.length === 0) {
console.log('No official rounds found in details page');
logger.info('No official rounds found in details page');
return [];
}
console.log(`✓ Found ${officialRounds.length} official rating rounds`);
logger.info(`Found ${officialRounds.length} official rating rounds`);
const sortedRounds = officialRounds.sort((a, b) => b.date - a.date);
const latestOfficialDate = sortedRounds[0].date;
console.log(`Latest official round: ${latestOfficialDate.toDateString()}`);
logger.info(`Latest official round: ${latestOfficialDate.toDateString()}`);
console.log('Step 2: Looking for NEW tournaments since latest official round...');
logger.info('Looking for new tournaments since latest official round...');
const newRounds = await getNewTournamentRounds(browser, pdgaNumber, latestOfficialDate);
if (newRounds.length > 0) {
console.log(`✓ Found ${newRounds.length} new round ratings`);
logger.info(`Found ${newRounds.length} new round ratings`);
} else {
console.log('ℹ No new tournaments found since latest official round');
logger.info('No new tournaments found since latest official round');
}
const allRounds = [
@@ -299,12 +300,12 @@ async function getOptimizedPlayerRounds(browser, pdgaNumber) {
allRounds.sort((a, b) => a.date - b.date);
console.log(`=== Summary: ${officialRounds.length} official + ${newRounds.length} new = ${allRounds.length} total rounds ===`);
logger.info(`Summary: ${officialRounds.length} official + ${newRounds.length} new = ${allRounds.length} total rounds`);
return allRounds;
} catch (error) {
console.error('Error in optimized round collection:', error.message);
logger.error('Error in optimized round collection: ' + error.message);
return [];
}
}
src/services/player-service.js
+20 -19
View File
@@ -2,12 +2,13 @@ const { db } = require('../db');
const { getPlayerFromDB, getRoundHistoryFromDB, savePredictedRatingToDB, savePlayerToDB } = require('../models/player');
const { fetchPlayerDataHTTP, parsePlayerData } = require('../scrapers/player-http');
const { calculatePredictedRating } = require('./rating-calculator');
const logger = require('../logger');
async function getPlayerDataFromDB(pdgaNumber) {
try {
const cachedPlayer = await getPlayerFromDB(pdgaNumber);
if (cachedPlayer) {
console.log(`Loading PDGA ${pdgaNumber} from DB (source of truth)`);
logger.debug(`Loading PDGA ${pdgaNumber} from DB (source of truth)`);
let predictedRating = cachedPlayer.predicted_rating;
let stdDev = cachedPlayer.std_dev;
@@ -28,33 +29,33 @@ async function getPlayerDataFromDB(pdgaNumber) {
}
return null;
} catch (err) {
console.error(`Database error for PDGA ${pdgaNumber}:`, err.message);
logger.error(`Database error for PDGA ${pdgaNumber}:`, err.message);
return null;
}
}
async function scrapePDGARating(pdgaNumber, retries = 3) {
console.log(`=== Refreshing PDGA ${pdgaNumber} from PDGA website ===`);
logger.info(`Refreshing PDGA ${pdgaNumber} from PDGA website`);
for (let attempt = 1; attempt <= retries; attempt++) {
try {
console.log(`Attempt ${attempt}/${retries} for PDGA ${pdgaNumber} (using HTTP)`);
logger.info(`Attempt ${attempt}/${retries} for PDGA ${pdgaNumber} (using HTTP)`);
const html = await fetchPlayerDataHTTP(pdgaNumber);
const result = parsePlayerData(html, pdgaNumber);
try {
await savePlayerToDB(result);
console.log(`Saved PDGA ${pdgaNumber} to database`);
logger.info(`Saved PDGA ${pdgaNumber} to database`);
} catch (dbErr) {
console.error(`Failed to save PDGA ${pdgaNumber} to database:`, dbErr.message);
logger.error(`Failed to save PDGA ${pdgaNumber} to database:`, dbErr.message);
}
console.log(`Successfully scraped PDGA ${pdgaNumber} on attempt ${attempt}`);
logger.info(`Successfully scraped PDGA ${pdgaNumber} on attempt ${attempt}`);
return result;
} catch (error) {
console.error(`Attempt ${attempt}/${retries} failed for PDGA ${pdgaNumber}:`, error.message);
logger.error(`Attempt ${attempt}/${retries} failed for PDGA ${pdgaNumber}:`, error.message);
if (attempt === retries) {
return {
@@ -72,13 +73,13 @@ async function scrapePDGARating(pdgaNumber, retries = 3) {
const retryAfter = error.rateLimitInfo.headers['retry-after'];
if (retryAfter) {
retryDelay = Math.max(retryDelay, (parseInt(retryAfter) + 1) * 1000);
console.log(`Using Retry-After header: waiting ${retryDelay/1000}s`);
logger.warn(`Using Retry-After header: waiting ${retryDelay/1000}s`);
}
}
if (error.code === 'ECONNRESET') {
retryDelay = Math.max(retryDelay, 10000);
console.log(`Connection reset detected: waiting ${retryDelay/1000}s`);
logger.warn(`Connection reset detected: waiting ${retryDelay/1000}s`);
}
await new Promise(resolve => setTimeout(resolve, retryDelay));
@@ -90,7 +91,7 @@ async function getPredictedRatingFromDB(pdgaNumber) {
try {
const roundHistory = await getRoundHistoryFromDB(pdgaNumber);
if (roundHistory.length > 0) {
console.log(`Using ${roundHistory.length} cached rounds for PDGA ${pdgaNumber} prediction`);
logger.debug(`Using ${roundHistory.length} cached rounds for PDGA ${pdgaNumber} prediction`);
const roundRatings = roundHistory.map(round => ({
rating: round.rating,
@@ -106,7 +107,7 @@ async function getPredictedRatingFromDB(pdgaNumber) {
}
return 0;
} catch (err) {
console.error(`Error getting predicted rating from DB for ${pdgaNumber}:`, err.message);
logger.error(`Error getting predicted rating from DB for ${pdgaNumber}:`, err.message);
return 0;
}
}
@@ -124,7 +125,7 @@ async function getAllRatingsFromDB(progressCallback = null) {
);
});
console.log(`Loading ${allPlayers.length} players from database...`);
logger.info(`Loading ${allPlayers.length} players from database...`);
const ratings = [];
const total = allPlayers.length;
@@ -159,7 +160,7 @@ async function getAllRatingsFromDB(progressCallback = null) {
});
}
} catch (error) {
console.error(`Failed to load PDGA ${pdgaNumber} from database:`, error.message);
logger.error(`Failed to load PDGA ${pdgaNumber} from database:`, error.message);
const errorData = {
pdgaNumber: parseInt(pdgaNumber),
name: player.name || 'Database Error',
@@ -183,7 +184,7 @@ async function getAllRatingsFromDB(progressCallback = null) {
return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0));
} catch (error) {
console.error('Error loading players from database:', error);
logger.error('Error loading players from database:', error);
return [];
}
}
@@ -201,7 +202,7 @@ async function refreshAllPlayersInDB(progressCallback = null) {
);
});
console.log(`Refreshing ${allPlayers.length} players from database...`);
logger.info(`Refreshing ${allPlayers.length} players from database...`);
const ratings = [];
const total = allPlayers.length;
@@ -210,7 +211,7 @@ async function refreshAllPlayersInDB(progressCallback = null) {
const player = allPlayers[i];
const pdgaNumber = player.pdga_number;
console.log(`Refreshing PDGA ${pdgaNumber}... (${i + 1}/${total})`);
logger.info(`Refreshing PDGA ${pdgaNumber}... (${i + 1}/${total})`);
if (progressCallback) {
progressCallback({
@@ -237,7 +238,7 @@ async function refreshAllPlayersInDB(progressCallback = null) {
await new Promise(resolve => setTimeout(resolve, 2000));
} catch (error) {
console.error(`Failed to refresh PDGA ${pdgaNumber}:`, error.message);
logger.error(`Failed to refresh PDGA ${pdgaNumber}:`, error.message);
const errorData = {
pdgaNumber: parseInt(pdgaNumber),
name: player.name || 'Error',
@@ -261,7 +262,7 @@ async function refreshAllPlayersInDB(progressCallback = null) {
return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0));
} catch (error) {
console.error('Error refreshing all players:', error);
logger.error('Error refreshing all players:', error);
return [];
}
}