diff --git a/server.js b/server.js index a48665e..0a14d8d 100644 --- a/server.js +++ b/server.js @@ -533,7 +533,7 @@ async function getOfficialRatingHistory(browser, pdgaNumber) { try { const url = `https://www.pdga.com/player/${pdgaNumber}/history`; await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 }); - await page.waitForTimeout(2000); + await page.waitForTimeout(1000); // Reduced delay // Extract the rating history data ratingHistory = await page.evaluate(() => { @@ -592,7 +592,7 @@ async function getPlayerTournamentDetails(browser, pdgaNumber) { try { const url = `https://www.pdga.com/player/${pdgaNumber}/details`; await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 }); - await page.waitForTimeout(2000); + await page.waitForTimeout(1000); // Reduced delay // Extract individual tournament rounds with actual dates and ratings tournamentRounds = await page.evaluate(() => { @@ -693,6 +693,201 @@ async function getPlayerTournamentDetails(browser, pdgaNumber) { return tournamentRounds; } +// Get the most recent tournament date from /details page (official rating rounds) +async function getLatestOfficialRoundDate(browser, pdgaNumber) { + try { + const detailsRounds = await getPlayerTournamentDetails(browser, pdgaNumber); + if (detailsRounds.length === 0) { + return null; + } + + // Find the most recent date from details page + const sortedRounds = detailsRounds.sort((a, b) => b.date - a.date); + const latestDate = sortedRounds[0].date; + + console.log(`Latest official round date for PDGA ${pdgaNumber}: ${latestDate.toDateString()}`); + return latestDate; + } catch (error) { + console.error('Error getting latest official round date:', error.message); + return null; + } +} + +// Get NEW tournament rounds (played after the latest official round) +async function getNewTournamentRounds(browser, pdgaNumber, afterDate) { + const page = await browser.newPage(); + let newRounds = []; + + try { + const url = `https://www.pdga.com/player/${pdgaNumber}`; + await page.goto(url, { waitUntil: 'networkidle2' }); + + console.log(`Looking for tournaments after ${afterDate.toDateString()}...`); + + // Get tournament URLs that are newer than afterDate + const newTournamentUrls = await page.evaluate((afterTimestamp) => { + const afterDate = new Date(afterTimestamp); + const tables = document.querySelectorAll('table[id*="player-results"]'); + const urls = []; + + tables.forEach(table => { + const rows = table.querySelectorAll('tbody tr'); + rows.forEach(row => { + const dateCell = row.querySelector('.dates'); + const tournamentCell = row.querySelector('.tournament a'); + + if (dateCell && tournamentCell) { + const dateText = dateCell.innerText.trim(); + const dateMatch = dateText.match(/\d{1,2}-[A-Za-z]{3}-\d{4}/); + + if (dateMatch) { + const dateStr = dateMatch[0]; + const date = new Date(dateStr); + + // Only include tournaments AFTER the latest official round + if (date > afterDate) { + const href = tournamentCell.getAttribute('href'); + if (href) { + urls.push({ + url: `https://www.pdga.com${href}`, + date: dateStr, + name: tournamentCell.innerText.trim() + }); + } + } + } + } + }); + }); + + return urls; + }, afterDate.getTime()); + + console.log(`Found ${newTournamentUrls.length} new tournaments after ${afterDate.toDateString()}`); + + // Scrape individual round ratings from new tournaments + for (const tournamentData of newTournamentUrls) { + try { + console.log(`Scraping new tournament: ${tournamentData.name} (${tournamentData.date})`); + + await page.goto(tournamentData.url, { waitUntil: 'domcontentloaded', timeout: 30000 }); + await page.waitForTimeout(500); // Reduced from 2s to 0.5s since we're only scraping a few tournaments + + const roundRatings = await page.evaluate((pdgaNum) => { + const rows = document.querySelectorAll('tr'); + + for (const row of rows) { + const cells = row.querySelectorAll('td'); + const hasPlayerNumber = Array.from(cells).some(cell => + cell.innerText && cell.innerText.includes(pdgaNum.toString()) + ); + + if (hasPlayerNumber) { + const roundRatingCells = row.querySelectorAll('td.round-rating'); + const ratings = []; + + roundRatingCells.forEach(cell => { + const rating = parseInt(cell.innerText.trim()); + if (!isNaN(rating) && rating > 0) { + ratings.push(rating); + } + }); + + return ratings; + } + } + + return []; + }, pdgaNumber); + + if (roundRatings.length > 0) { + const parsedDate = parseDate(tournamentData.date); + roundRatings.forEach(rating => { + newRounds.push({ + rating, + date: parsedDate, + competition: tournamentData.name + }); + }); + + console.log(`✓ Found ${roundRatings.length} round ratings for ${tournamentData.name}`); + } + + } catch (error) { + console.error(`Error scraping tournament ${tournamentData.name}:`, error.message); + } + } + + } catch (error) { + console.error(`Error getting new tournament rounds for PDGA ${pdgaNumber}:`, error); + } finally { + await page.close(); + } + + return newRounds; +} + +// Optimized function: Get /details rounds + new tournaments only +async function getOptimizedPlayerRounds(browser, pdgaNumber) { + console.log(`=== Optimized Round Collection for PDGA ${pdgaNumber} ===`); + + try { + // Step 1: Get all official rating rounds from /details page + console.log('Step 1: Getting official rating rounds from /details page...'); + const officialRounds = await getPlayerTournamentDetails(browser, pdgaNumber); + + if (officialRounds.length === 0) { + console.log('No official rounds found in details page'); + return []; + } + + console.log(`✓ Found ${officialRounds.length} official rating rounds`); + + // Step 2: Find the most recent official round date + const sortedRounds = officialRounds.sort((a, b) => b.date - a.date); + const latestOfficialDate = sortedRounds[0].date; + console.log(`Latest official round: ${latestOfficialDate.toDateString()}`); + + // Step 3: Get NEW tournament rounds (after latest official round) + console.log('Step 2: Looking for NEW tournaments since latest official round...'); + const newRounds = await getNewTournamentRounds(browser, pdgaNumber, latestOfficialDate); + + if (newRounds.length > 0) { + console.log(`✓ Found ${newRounds.length} new round ratings`); + } else { + console.log('ℹ No new tournaments found since latest official round'); + } + + // Step 4: Combine official rounds + new rounds + const allRounds = [ + ...officialRounds.map(round => ({ + rating: round.rating, + date: round.date, + competition: round.competition, + source: 'official' // From /details page + })), + ...newRounds.map(round => ({ + rating: round.rating, + date: round.date, + competition: round.competition, + source: 'new' // From individual tournaments + })) + ]; + + // Sort by date (oldest first) + allRounds.sort((a, b) => a.date - b.date); + + console.log(`=== Summary: ${officialRounds.length} official + ${newRounds.length} new = ${allRounds.length} total rounds ===`); + + return allRounds; + + } catch (error) { + console.error('Error in optimized round collection:', error.message); + return []; + } +} + +// Legacy function - keep for backward compatibility but mark as deprecated async function getPlayerCompetitionRatings(browser, pdgaNumber, sinceDate = null) { const page = await browser.newPage(); let allRatings = []; @@ -773,8 +968,8 @@ async function getPlayerCompetitionRatings(browser, pdgaNumber, sinceDate = null throw navError; // Re-throw to be caught by outer try-catch } - console.log(`Waiting 3s before scraping tournament data...`); - await page.waitForTimeout(3000); // Longer delay between requests + console.log(`Waiting 1s before scraping tournament data...`); + await page.waitForTimeout(1000); // Reduced delay for optimized approach console.log(`Starting page evaluation for PDGA ${pdgaNumber}...`); let roundRatings; @@ -1714,42 +1909,59 @@ app.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => { officialHistory = []; } - // Step 2: Get tournament round details - let tournamentRounds = []; + // Step 2: Get optimized round collection (details + new tournaments only) + let allRounds = []; try { - tournamentRounds = await getPlayerTournamentDetails(browser, pdgaNumber); - if (tournamentRounds.length > 0) { - await saveRoundHistoryToDB(pdgaNumber, tournamentRounds, false); + console.log(`Using optimized approach: /details + new tournaments only for PDGA ${pdgaNumber}...`); + allRounds = await getOptimizedPlayerRounds(browser, pdgaNumber); + + if (allRounds.length > 0) { + // Convert to the format expected by saveRoundHistoryToDB + const roundsForDB = allRounds.map(round => ({ + rating: round.rating, + date: round.date, + competition: round.competition + })); + + // Save all rounds (replacing existing data with the complete optimized set) + await saveRoundHistoryToDB(pdgaNumber, roundsForDB, false); // false = replace all + console.log(`✓ Saved ${allRounds.length} rounds using optimized approach`); + + // Update timestamp to mark when we last did a full collection + await updateLastRoundUpdateDate(pdgaNumber); + } else { + console.log('ℹ No rounds found'); } } catch (detailsError) { - console.error('Failed to fetch tournament details:', detailsError.message); - tournamentRounds = []; + console.error('Failed to fetch rounds using optimized approach:', detailsError.message); + allRounds = []; } await browser.close(); browser = null; - // Update timestamp and calculate prediction - if (tournamentRounds.length > 0) { - await updateLastRoundUpdateDate(pdgaNumber); - } - - const allRounds = await getRoundHistoryFromDB(pdgaNumber); - const allRoundsForPrediction = allRounds.map(round => ({ + // Calculate prediction from optimized round collection + const dbRounds = await getRoundHistoryFromDB(pdgaNumber); + const roundsForPrediction = dbRounds.map(round => ({ rating: round.rating, date: new Date(round.date), competition: round.competition_name })); - const predictedRating = calculatePredictedRating(allRoundsForPrediction); + const predictedRating = calculatePredictedRating(roundsForPrediction); + + // Count official vs new rounds + const officialCount = allRounds.filter(r => r.source === 'official').length; + const newCount = allRounds.filter(r => r.source === 'new').length; res.json({ success: true, predictedRating, - totalRounds: allRoundsForPrediction.length, - officialRounds: officialHistory.length, - newRounds: tournamentRounds.length, - wasIncremental: isIncremental + totalRounds: roundsForPrediction.length, + officialRounds: officialCount, + newRounds: newCount, + approach: 'optimized', + message: `Used /details (${officialCount} rounds) + new tournaments (${newCount} rounds)` }); } catch (error) { console.error(`=== Error refreshing round history for PDGA ${pdgaNumber} ===`);