diff --git a/courses.html b/courses.html index 6108615..3ecd07b 100644 --- a/courses.html +++ b/courses.html @@ -87,6 +87,29 @@ background-color: #6c757d; cursor: not-allowed; } + .search-container { + margin-bottom: 20px; + text-align: center; + } + .search-input { + width: 100%; + max-width: 400px; + padding: 10px 15px; + font-size: 16px; + border: 2px solid #ddd; + border-radius: 4px; + outline: none; + transition: border-color 0.2s; + } + .search-input:focus { + border-color: #007bff; + } + .search-results-info { + text-align: center; + margin: 10px 0; + color: #666; + font-size: 14px; + } table { width: 100%; border-collapse: collapse; @@ -251,13 +274,21 @@ Courses +
+ +
+
+
-
@@ -265,6 +296,8 @@ diff --git a/index.html b/index.html index 5da8d2b..cda76bf 100644 --- a/index.html +++ b/index.html @@ -311,6 +311,123 @@ .debug-close:hover { color: #333; } + .add-player-section { + background-color: #f8f9fa; + border: 2px solid #007bff; + border-radius: 8px; + padding: 20px; + margin-bottom: 30px; + text-align: center; + } + .add-player-section h3 { + margin-top: 0; + margin-bottom: 15px; + color: #333; + font-size: 18px; + } + .add-player-form { + display: flex; + justify-content: center; + align-items: center; + gap: 10px; + flex-wrap: wrap; + } + .pdga-input { + padding: 10px 15px; + font-size: 16px; + border: 2px solid #ddd; + border-radius: 4px; + outline: none; + width: 250px; + transition: border-color 0.2s; + } + .pdga-input:focus { + border-color: #007bff; + } + .btn-add { + background-color: #28a745; + } + .btn-add:hover { + background-color: #218838; + } + .modal { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba(0, 0, 0, 0.5); + z-index: 10001; + display: none; + justify-content: center; + align-items: center; + } + .modal-content { + background: white; + border-radius: 8px; + padding: 0; + max-width: 500px; + width: 90%; + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3); + position: relative; + } + .modal-header { + font-weight: bold; + font-size: 20px; + padding: 20px; + color: #333; + border-bottom: 2px solid #007bff; + } + .modal-body { + padding: 20px; + font-size: 16px; + color: #495057; + line-height: 1.6; + } + .modal-footer { + padding: 15px 20px; + border-top: 1px solid #e9ecef; + display: flex; + justify-content: flex-end; + gap: 10px; + } + .modal-close { + position: absolute; + top: 10px; + right: 15px; + font-size: 28px; + color: #999; + cursor: pointer; + background: none; + border: none; + line-height: 1; + } + .modal-close:hover { + color: #333; + } + .btn-cancel { + background-color: #6c757d; + } + .btn-cancel:hover { + background-color: #5a6268; + } + .btn-confirm { + background-color: #28a745; + } + .btn-confirm:hover { + background-color: #218838; + } + @media (max-width: 768px) { + .add-player-section { + padding: 15px; + } + .add-player-form { + flex-direction: column; + } + .pdga-input { + width: 100%; + } + } @@ -320,6 +437,23 @@ Player Ratings Courses + + +
+

Add Yourself to Tracked Players

+
+ + +
+
Load All @@ -343,6 +477,19 @@
+ + + diff --git a/server.js b/server.js index a508984..02b4efe 100644 --- a/server.js +++ b/server.js @@ -9,6 +9,7 @@ const app = express(); const PORT = 3000; app.use(express.static('public')); +app.use(express.json()); // Initialize SQLite database const dbPath = process.env.DB_PATH || './ratings.db'; @@ -43,16 +44,17 @@ function initializeDatabase() { console.error('Error checking table schema:', err); return; } - - // Check if column exists by querying table info + + // Check if columns exist by querying table info db.all("PRAGMA table_info(players)", (err, columns) => { if (err) { console.error('Error getting table info:', err); return; } - + const hasLastRoundUpdate = columns.some(col => col.name === 'last_round_update'); - + const hasPredictedRating = columns.some(col => col.name === 'predicted_rating'); + if (!hasLastRoundUpdate) { console.log('Adding last_round_update column to players table...'); db.run(` @@ -65,6 +67,19 @@ function initializeDatabase() { } }); } + + if (!hasPredictedRating) { + console.log('Adding predicted_rating column to players table...'); + db.run(` + ALTER TABLE players ADD COLUMN predicted_rating INTEGER DEFAULT NULL + `, (err) => { + if (err) { + console.error('Error adding predicted_rating column:', err.message); + } else { + console.log('Successfully added predicted_rating column'); + } + }); + } }); }); @@ -140,57 +155,58 @@ function initializeDatabase() { }); } -// Check and populate database from PDGA numbers file at startup +// Check and populate database from PDGA numbers file at startup (only if DB is empty) async function checkAndPopulateDatabase() { try { - console.log('=== Checking database population against PDGA numbers file ==='); - + // Check if database has any players + const playerCount = await new Promise((resolve, reject) => { + db.get('SELECT COUNT(*) as count FROM players', [], (err, row) => { + if (err) reject(err); + else resolve(row.count); + }); + }); + + if (playerCount > 0) { + console.log(`✓ Database already has ${playerCount} players - skipping text file import`); + console.log('📝 Note: pdga-numbers.txt is only used when database is empty'); + return; + } + + console.log('=== Database is empty - populating from PDGA numbers file ==='); + const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8') .split('\n') .map(num => num.trim()) .filter(num => num); - + console.log(`Found ${pdgaNumbers.length} PDGA numbers in file`); - - const missingPlayers = []; - - // Check which players are missing from database - for (const pdgaNumber of pdgaNumbers) { - const player = await getPlayerFromDB(pdgaNumber); - if (!player) { - missingPlayers.push(pdgaNumber); - } - } - - if (missingPlayers.length === 0) { - console.log('✓ All players from PDGA numbers file are already in database'); + + if (pdgaNumbers.length === 0) { + console.log('⚠ No PDGA numbers found in file'); return; } - - console.log(`Found ${missingPlayers.length} missing players: [${missingPlayers.join(', ')}]`); - console.log('=== Starting automatic population of missing players ==='); - - // Populate missing players - for (let i = 0; i < missingPlayers.length; i++) { - const pdgaNumber = missingPlayers[i]; - console.log(`[${i + 1}/${missingPlayers.length}] Scraping missing player PDGA ${pdgaNumber}...`); - + + console.log('Populating database with players from file...'); + + for (let i = 0; i < pdgaNumbers.length; i++) { + const pdgaNumber = pdgaNumbers[i]; + console.log(`[${i + 1}/${pdgaNumbers.length}] Adding PDGA ${pdgaNumber}...`); + try { const playerData = await scrapePDGARating(pdgaNumber); - console.log(`✓ Added PDGA ${pdgaNumber}: ${playerData.name}`); - + console.log(` ✓ Added ${playerData.name}`); + // Delay between requests to be respectful to PDGA - if (i < missingPlayers.length - 1) { - console.log('Waiting 2s before next request...'); + if (i < pdgaNumbers.length - 1) { await new Promise(resolve => setTimeout(resolve, 2000)); } } catch (error) { - console.error(`✗ Failed to add PDGA ${pdgaNumber}: ${error.message}`); + console.error(` ✗ Failed to add PDGA ${pdgaNumber}:`, error.message); } } - + console.log('=== Database population complete ==='); - + } catch (error) { console.error('Error during database population check:', error.message); } @@ -514,8 +530,13 @@ async function getPlayerDataFromDB(pdgaNumber) { const cachedPlayer = await getPlayerFromDB(pdgaNumber); if (cachedPlayer) { console.log(`Loading PDGA ${pdgaNumber} from DB (source of truth)`); - const predictedRating = await getPredictedRatingFromDB(pdgaNumber); - + + // Use stored predicted_rating if available, otherwise calculate it from round history + let predictedRating = cachedPlayer.predicted_rating; + if (!predictedRating || predictedRating === 0) { + predictedRating = await getPredictedRatingFromDB(pdgaNumber); + } + return { pdgaNumber: cachedPlayer.pdga_number, name: cachedPlayer.name, @@ -621,15 +642,19 @@ async function getPredictedRatingFromDB(pdgaNumber) { const roundHistory = await getRoundHistoryFromDB(pdgaNumber); if (roundHistory.length > 0) { console.log(`Using ${roundHistory.length} cached rounds for PDGA ${pdgaNumber} prediction`); - + // Convert to the format expected by calculatePredictedRating const roundRatings = roundHistory.map(round => ({ rating: round.rating, date: new Date(round.date), competition: round.competition_name || 'Unknown' })); - + const result = calculatePredictedRating(roundRatings); + + // Save the calculated prediction to database + await savePredictedRatingToDB(pdgaNumber, result.rating); + return result.rating; } return 0; @@ -639,6 +664,19 @@ async function getPredictedRatingFromDB(pdgaNumber) { } } +function savePredictedRatingToDB(pdgaNumber, predictedRating) { + return new Promise((resolve, reject) => { + db.run( + 'UPDATE players SET predicted_rating = ? WHERE pdga_number = ?', + [predictedRating, pdgaNumber], + function(err) { + if (err) reject(err); + else resolve(); + } + ); + }); +} + async function getOfficialRatingHistory(browser, pdgaNumber) { const page = await browser.newPage(); let ratingHistory = []; @@ -1913,18 +1951,27 @@ async function scrapeEventResults(browser, eventUrl, layoutsWithDivisions) { async function getAllRatingsFromDB(progressCallback = null) { try { - const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8') - .split('\n') - .map(num => num.trim()) - .filter(num => num); - + // Get all players from database instead of text file + const allPlayers = await new Promise((resolve, reject) => { + db.all( + 'SELECT pdga_number, name, current_rating, rating_change FROM players ORDER BY pdga_number', + [], + (err, rows) => { + if (err) reject(err); + else resolve(rows || []); + } + ); + }); + + console.log(`Loading ${allPlayers.length} players from database...`); + const ratings = []; - const total = pdgaNumbers.length; - - for (let i = 0; i < pdgaNumbers.length; i++) { - const pdgaNumber = pdgaNumbers[i]; - console.log(`Loading PDGA ${pdgaNumber} from database... (${i + 1}/${total})`); - + const total = allPlayers.length; + + for (let i = 0; i < allPlayers.length; i++) { + const player = allPlayers[i]; + const pdgaNumber = player.pdga_number; + if (progressCallback) { progressCallback({ current: i + 1, @@ -1933,53 +1980,50 @@ async function getAllRatingsFromDB(progressCallback = null) { status: 'loading' }); } - + try { - // Load from database only (source of truth) + // Load full player data from database const playerData = await getPlayerDataFromDB(pdgaNumber); - + if (playerData) { ratings.push(playerData); - } else { - console.log(`PDGA ${pdgaNumber} not found in DB - skipping (page load)`); - // Skip players not in DB for page loads } - + if (progressCallback) { progressCallback({ current: i + 1, total, pdgaNumber, - status: playerData ? 'completed' : 'skipped', - name: playerData ? playerData.name : 'Not in DB' + status: 'completed', + name: playerData ? playerData.name : player.name }); } } catch (error) { console.error(`Failed to load PDGA ${pdgaNumber} from database:`, error.message); const errorData = { pdgaNumber: parseInt(pdgaNumber), - name: 'Database Error', - rating: null, - ratingChange: null, + name: player.name || 'Database Error', + rating: player.current_rating, + ratingChange: player.rating_change, predictedRating: null }; ratings.push(errorData); - + if (progressCallback) { progressCallback({ current: i + 1, total, pdgaNumber, status: 'error', - name: 'Database Error' + name: player.name || 'Database Error' }); } } } - + return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0)); } catch (error) { - console.error('Error reading PDGA numbers:', error); + console.error('Error loading players from database:', error); return []; } } @@ -2040,12 +2084,12 @@ app.post('/api/populate-database', (req, res) => { res.write(`data: ${JSON.stringify(progress)}\n\n`); }; - console.log('=== Starting database population from PDGA numbers file ==='); - - // Use the scraping function to populate database - getAllRatingsWithScraping(progressCallback).then(ratings => { - console.log(`=== Database population complete: ${ratings.length} players added ===`); - res.write(`data: ${JSON.stringify({ status: 'complete', ratings, message: `Successfully populated database with ${ratings.length} players` })}\n\n`); + console.log('=== Starting database population from database players ==='); + + // Populate database by refreshing all players in database + refreshAllPlayersInDB(progressCallback).then(ratings => { + console.log(`=== Database population complete: ${ratings.length} players refreshed ===`); + res.write(`data: ${JSON.stringify({ status: 'complete', ratings, message: `Successfully refreshed ${ratings.length} players` })}\n\n`); res.end(); }).catch(error => { console.error('Error populating database:', error); @@ -2054,25 +2098,19 @@ app.post('/api/populate-database', (req, res) => { }); }); -// Simple endpoint to check if database needs population +// Simple endpoint to check database status app.get('/api/database-status', async (req, res) => { try { - const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8') - .split('\n') - .map(num => num.trim()) - .filter(num => num); - - let playersInDB = 0; - for (const pdgaNumber of pdgaNumbers) { - const player = await getPlayerFromDB(pdgaNumber); - if (player) playersInDB++; - } - + const playerCount = await new Promise((resolve, reject) => { + db.get('SELECT COUNT(*) as count FROM players', [], (err, row) => { + if (err) reject(err); + else resolve(row.count); + }); + }); + res.json({ - totalExpected: pdgaNumbers.length, - playersInDB: playersInDB, - needsPopulation: playersInDB === 0, - populationProgress: Math.round((playersInDB / pdgaNumbers.length) * 100) + playersInDB: playerCount, + needsPopulation: playerCount === 0 }); } catch (error) { res.status(500).json({ error: 'Failed to check database status' }); @@ -2092,8 +2130,8 @@ app.get('/api/load-all-players', (req, res) => { res.write(`data: ${JSON.stringify(progress)}\n\n`); }; - // Use the original scraping function for bulk loading - getAllRatingsWithScraping(progressCallback).then(ratings => { + // Refresh all players currently in database + refreshAllPlayersInDB(progressCallback).then(ratings => { res.write(`data: ${JSON.stringify({ status: 'complete', ratings })}\n\n`); res.end(); }).catch(error => { @@ -2176,6 +2214,87 @@ async function getAllRatingsWithScraping(progressCallback = null) { } } +// Refresh all players currently in database +async function refreshAllPlayersInDB(progressCallback = null) { + try { + // Get all players from database + const allPlayers = await new Promise((resolve, reject) => { + db.all( + 'SELECT pdga_number, name FROM players ORDER BY pdga_number', + [], + (err, rows) => { + if (err) reject(err); + else resolve(rows || []); + } + ); + }); + + console.log(`Refreshing ${allPlayers.length} players from database...`); + + const ratings = []; + const total = allPlayers.length; + + for (let i = 0; i < allPlayers.length; i++) { + const player = allPlayers[i]; + const pdgaNumber = player.pdga_number; + + console.log(`Refreshing PDGA ${pdgaNumber}... (${i + 1}/${total})`); + + if (progressCallback) { + progressCallback({ + current: i + 1, + total, + pdgaNumber, + status: 'loading' + }); + } + + try { + const playerData = await scrapePDGARating(pdgaNumber); + ratings.push(playerData); + + if (progressCallback) { + progressCallback({ + current: i + 1, + total, + pdgaNumber, + status: 'completed', + name: playerData.name + }); + } + + // Delay between PDGA scraping requests to be respectful + await new Promise(resolve => setTimeout(resolve, 2000)); + } catch (error) { + console.error(`Failed to refresh PDGA ${pdgaNumber}:`, error.message); + const errorData = { + pdgaNumber: parseInt(pdgaNumber), + name: player.name || 'Error', + rating: 0, + ratingChange: null, + predictedRating: null + }; + ratings.push(errorData); + + if (progressCallback) { + progressCallback({ + current: i + 1, + total, + pdgaNumber, + status: 'error', + name: player.name || 'Error' + }); + } + } + } + + return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0)); + } catch (error) { + console.error('Error refreshing all players:', error); + return []; + } +} + async function fetchRatingHistory(pdgaNumber) { return new Promise((resolve, reject) => { const options = { @@ -2380,18 +2499,105 @@ app.post('/api/clear-cache', (req, res) => { }); // Individual player refresh endpoints +// Search for a player (check if exists in DB and fetch from PDGA) +app.get('/api/search-player/:pdgaNumber', async (req, res) => { + try { + const { pdgaNumber } = req.params; + console.log(`Searching for player with PDGA number ${pdgaNumber}`); + + // Check if player already exists in database + const existingPlayer = await getPlayerFromDB(pdgaNumber); + if (existingPlayer) { + return res.json({ + alreadyExists: true, + player: { + pdgaNumber: existingPlayer.pdga_number, + name: existingPlayer.name, + rating: existingPlayer.current_rating, + ratingChange: existingPlayer.rating_change + } + }); + } + + // Fetch player data from PDGA + const html = await fetchPlayerDataHTTP(pdgaNumber); + const playerData = parsePlayerData(html, pdgaNumber); + + // Check if player was found (name shouldn't be 'Unknown') + if (playerData.name === 'Unknown' || !playerData.name) { + return res.status(404).json({ error: 'Player not found' }); + } + + res.json({ + alreadyExists: false, + player: playerData + }); + } catch (error) { + console.error('Error searching for player:', error.message); + res.status(500).json({ error: 'Failed to search for player' }); + } +}); + +// Add a new player to the database +app.post('/api/add-player', async (req, res) => { + try { + const { pdgaNumber } = req.body; + + if (!pdgaNumber) { + return res.status(400).json({ error: 'PDGA number is required' }); + } + + console.log(`Adding player with PDGA number ${pdgaNumber}`); + + // Check if player already exists + const existingPlayer = await getPlayerFromDB(pdgaNumber); + if (existingPlayer) { + return res.status(409).json({ + error: 'Player already exists', + player: { + pdgaNumber: existingPlayer.pdga_number, + name: existingPlayer.name, + rating: existingPlayer.current_rating + } + }); + } + + // Fetch player data from PDGA + const html = await fetchPlayerDataHTTP(pdgaNumber); + const playerData = parsePlayerData(html, pdgaNumber); + + // Verify player was found + if (playerData.name === 'Unknown' || !playerData.name) { + return res.status(404).json({ error: 'Player not found' }); + } + + // Save to database + await savePlayerToDB(playerData); + + console.log(`Successfully added player: ${playerData.name} (#${pdgaNumber})`); + + res.json({ + success: true, + player: playerData + }); + } catch (error) { + console.error('Error adding player:', error.message); + res.status(500).json({ error: 'Failed to add player' }); + } +}); + app.post('/api/refresh-player/:pdgaNumber', async (req, res) => { try { const { pdgaNumber } = req.params; console.log(`Manually refreshing player data for PDGA ${pdgaNumber}`); - + // Force refresh by bypassing cache const html = await fetchPlayerDataHTTP(pdgaNumber); const playerData = parsePlayerData(html, pdgaNumber); - + // Save to database await savePlayerToDB(playerData); - + res.json({ success: true, player: playerData @@ -2473,10 +2679,25 @@ app.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => { // Check when we last updated rounds for this player const lastRoundUpdate = await getLastRoundUpdateDate(pdgaNumber); const sinceDate = lastRoundUpdate ? new Date(lastRoundUpdate) : null; + + // Rate limit: Only allow refresh once every 24 hours + if (sinceDate) { + const hoursSinceUpdate = (Date.now() - sinceDate.getTime()) / (1000 * 60 * 60); + if (hoursSinceUpdate < 24) { + const hoursRemaining = Math.ceil(24 - hoursSinceUpdate); + return res.status(429).json({ + error: 'Rate limit exceeded', + message: `Prediction can only be refreshed once every 24 hours. Please try again in ${hoursRemaining} hour(s).`, + lastUpdate: sinceDate.toISOString(), + hoursRemaining: hoursRemaining + }); + } + } + const isIncremental = !!sinceDate; - + console.log(`${isIncremental ? 'Incrementally updating' : 'Fully refreshing'} round history for PDGA ${pdgaNumber}${sinceDate ? ` since ${sinceDate.toDateString()}` : ''}`); - + try { browser = await puppeteer.launch({ headless: "new", @@ -2548,13 +2769,16 @@ app.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => { date: new Date(round.date), competition: round.competition_name })); - + const result = calculatePredictedRating(roundsForPrediction); - + + // Save the predicted rating to database for persistence + await savePredictedRatingToDB(pdgaNumber, result.rating); + // Count official vs new rounds const officialCount = allRounds.filter(r => r.source === 'official').length; const newCount = allRounds.filter(r => r.source === 'new').length; - + res.json({ success: true, predictedRating: result.rating, @@ -3041,72 +3265,6 @@ app.post('/api/scrape-event-results/:courseId', async (req, res) => { } }); -app.post('/api/scrape-all-layouts', async (req, res) => { - // Increase timeout for bulk scraping operations - req.setTimeout(1800000); // 30 minutes for bulk operations - res.setTimeout(1800000); - - let browser = null; - try { - console.log('Starting bulk layout scraping for all courses...'); - - const courses = await getAllCoursesFromDB(); - console.log(`Found ${courses.length} courses to scrape`); - - browser = await puppeteer.launch({ - headless: "new", - args: [ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', - '--disable-accelerated-2d-canvas', - '--no-first-run', - '--no-zygote', - '--disable-gpu' - ] - }); - - let totalLayouts = 0; - for (let i = 0; i < courses.length; i++) { - const course = courses[i]; - console.log(`[${i + 1}/${courses.length}] Scraping layouts for: ${course.name}`); - - try { - const layouts = await scrapeCourseLayouts(browser, course.link, course.id); - totalLayouts += layouts.length; - - // Delay between requests to be respectful - if (i < courses.length - 1) { - console.log('Waiting 2s before next request...'); - await new Promise(resolve => setTimeout(resolve, 2000)); - } - } catch (error) { - console.error(`Error scraping layouts for ${course.name}:`, error.message); - } - } - - await browser.close(); - browser = null; - - res.json({ - success: true, - coursesProcessed: courses.length, - totalLayouts: totalLayouts, - message: `Successfully scraped layouts for ${courses.length} courses (${totalLayouts} total layouts)` - }); - } catch (error) { - console.error('Error scraping all layouts:', error.message); - if (browser) { - try { - await browser.close(); - } catch (closeError) { - console.error('Error closing browser:', closeError.message); - } - } - res.status(500).json({ error: 'Failed to scrape all layouts' }); - } -}); - app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => { let browser = null; try {