Add real-time progress bar for rating loading

- Implement Server-Sent Events for live progress updates
- Add animated progress bar with percentage display
- Show real-time status: current player being loaded
- Display player names as they complete loading
- Handle errors gracefully with progress continuation
- Replace HTTP-only approach for better reliability
- Enhanced user experience with visual feedback

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Samuel Enocsson
2025-08-12 11:51:51 +02:00
parent 2528bc4b5d
commit 2d55651f43
2 changed files with 324 additions and 142 deletions
+128 -67
View File
@@ -29,6 +29,30 @@
font-size: 18px; font-size: 18px;
color: #666; color: #666;
} }
.progress-container {
width: 100%;
background-color: #f0f0f0;
border-radius: 10px;
padding: 3px;
margin: 20px 0;
}
.progress-bar {
width: 0%;
height: 30px;
background-color: #007bff;
border-radius: 8px;
text-align: center;
line-height: 30px;
color: white;
font-weight: bold;
transition: width 0.3s ease;
}
.progress-text {
text-align: center;
margin: 10px 0;
font-size: 16px;
color: #666;
}
table { table {
width: 100%; width: 100%;
border-collapse: collapse; border-collapse: collapse;
@@ -99,86 +123,123 @@
<body> <body>
<div class="container"> <div class="container">
<h1>PDGA Player Ratings</h1> <h1>PDGA Player Ratings</h1>
<div id="loading" class="loading">Loading ratings...</div> <div id="loading" class="loading" style="display: none;">Loading ratings...</div>
<div id="progress-section" style="display: none;">
<div class="progress-container">
<div id="progress-bar" class="progress-bar">0%</div>
</div>
<div id="progress-text" class="progress-text">Preparing to load ratings...</div>
</div>
<div id="ratings-table"></div> <div id="ratings-table"></div>
</div> </div>
<script> <script>
async function fetchRatings() { function fetchRatingsWithProgress() {
const loadingDiv = document.getElementById('loading'); const progressSection = document.getElementById('progress-section');
const progressBar = document.getElementById('progress-bar');
const progressText = document.getElementById('progress-text');
const tableDiv = document.getElementById('ratings-table'); const tableDiv = document.getElementById('ratings-table');
loadingDiv.style.display = 'block'; progressSection.style.display = 'block';
tableDiv.innerHTML = ''; tableDiv.innerHTML = '';
try { const eventSource = new EventSource('/api/ratings/progress');
const response = await fetch('/api/ratings');
const ratings = await response.json(); eventSource.onmessage = function(event) {
const data = JSON.parse(event.data);
loadingDiv.style.display = 'none'; if (data.status === 'loading') {
const percentage = Math.round((data.current / data.total) * 100);
if (ratings.length === 0) { progressBar.style.width = `${percentage}%`;
tableDiv.innerHTML = '<p>No ratings found.</p>'; progressBar.textContent = `${percentage}%`;
return; progressText.textContent = `Loading player ${data.current}/${data.total}: PDGA #${data.pdgaNumber}`;
} else if (data.status === 'completed') {
const percentage = Math.round((data.current / data.total) * 100);
progressBar.style.width = `${percentage}%`;
progressBar.textContent = `${percentage}%`;
progressText.textContent = `Loaded ${data.name} (${data.current}/${data.total})`;
} else if (data.status === 'error') {
const percentage = Math.round((data.current / data.total) * 100);
progressBar.style.width = `${percentage}%`;
progressBar.textContent = `${percentage}%`;
progressText.textContent = `Error loading PDGA #${data.pdgaNumber} (${data.current}/${data.total})`;
} else if (data.status === 'complete') {
progressSection.style.display = 'none';
displayRatings(data.ratings);
eventSource.close();
} else if (data.status === 'error') {
progressSection.style.display = 'none';
tableDiv.innerHTML = '<p>Error loading ratings. Please try again.</p>';
eventSource.close();
} }
};
let tableHTML = `
<table> eventSource.onerror = function() {
<thead> progressSection.style.display = 'none';
<tr> tableDiv.innerHTML = '<p>Connection error. Please refresh the page.</p>';
<th>Rank</th> eventSource.close();
<th>Player Name</th> };
<th>PDGA #</th> }
<th>Current Rating</th>
<th>Rating Change</th> function displayRatings(ratings) {
<th>Predicted Rating</th> const tableDiv = document.getElementById('ratings-table');
<th>Difference</th>
</tr> if (ratings.length === 0) {
</thead> tableDiv.innerHTML = '<p>No ratings found.</p>';
<tbody> return;
`; }
ratings.forEach((player, index) => { let tableHTML = `
const difference = player.predictedRating && player.rating ? <table>
player.predictedRating - player.rating : 0; <thead>
const diffText = difference > 0 ? `+${difference}` : difference.toString(); <tr>
const diffClass = difference > 0 ? 'positive' : difference < 0 ? 'negative' : 'neutral'; <th>Rank</th>
<th>Player Name</th>
const ratingChangeText = player.ratingChange ? <th>PDGA #</th>
(player.ratingChange > 0 ? `+${player.ratingChange}` : player.ratingChange.toString()) : 'N/A'; <th>Current Rating</th>
const ratingChangeClass = player.ratingChange > 0 ? 'positive' : <th>Rating Change</th>
player.ratingChange < 0 ? 'negative' : 'neutral'; <th>Predicted Rating</th>
<th>Difference</th>
tableHTML += `
<tr id="row-${player.pdgaNumber}">
<td>${index + 1}</td>
<td><a href="https://www.pdga.com/player/${player.pdgaNumber}" target="_blank">${player.name}</a></td>
<td class="pdga-number">#${player.pdgaNumber}</td>
<td class="rating">${player.rating || 'N/A'}</td>
<td class="rating-change ${ratingChangeClass}">${ratingChangeText}</td>
<td class="predicted-rating" id="predicted-${player.pdgaNumber}">
${player.predictedRating || 'N/A'}
</td>
<td class="difference ${diffClass}" id="diff-${player.pdgaNumber}">
${difference ? diffText :
`<button class="calc-btn" onclick="calculatePredictedRating(${player.pdgaNumber})">Predict Rating</button>`}
</td>
</tr> </tr>
`; </thead>
}); <tbody>
`;
ratings.forEach((player, index) => {
const difference = player.predictedRating && player.rating ?
player.predictedRating - player.rating : 0;
const diffText = difference > 0 ? `+${difference}` : difference.toString();
const diffClass = difference > 0 ? 'positive' : difference < 0 ? 'negative' : 'neutral';
const ratingChangeText = player.ratingChange ?
(player.ratingChange > 0 ? `+${player.ratingChange}` : player.ratingChange.toString()) : 'N/A';
const ratingChangeClass = player.ratingChange > 0 ? 'positive' :
player.ratingChange < 0 ? 'negative' : 'neutral';
tableHTML += ` tableHTML += `
</tbody> <tr id="row-${player.pdgaNumber}">
</table> <td>${index + 1}</td>
<td><a href="https://www.pdga.com/player/${player.pdgaNumber}" target="_blank">${player.name}</a></td>
<td class="pdga-number">#${player.pdgaNumber}</td>
<td class="rating">${player.rating || 'N/A'}</td>
<td class="rating-change ${ratingChangeClass}">${ratingChangeText}</td>
<td class="predicted-rating" id="predicted-${player.pdgaNumber}">
${player.predictedRating || 'N/A'}
</td>
<td class="difference ${diffClass}" id="diff-${player.pdgaNumber}">
${difference ? diffText :
`<button class="calc-btn" onclick="calculatePredictedRating(${player.pdgaNumber})">Predict Rating</button>`}
</td>
</tr>
`; `;
});
tableDiv.innerHTML = tableHTML;
tableHTML += `
} catch (error) { </tbody>
loadingDiv.style.display = 'none'; </table>
tableDiv.innerHTML = '<p>Error loading ratings. Please try again.</p>'; `;
console.error('Error:', error);
} tableDiv.innerHTML = tableHTML;
} }
async function calculatePredictedRating(pdgaNumber) { async function calculatePredictedRating(pdgaNumber) {
@@ -219,7 +280,7 @@
} }
} }
fetchRatings(); fetchRatingsWithProgress();
</script> </script>
</body> </body>
</html> </html>
+196 -75
View File
@@ -1,5 +1,6 @@
const express = require('express'); const express = require('express');
const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer');
const https = require('https');
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
@@ -11,7 +12,82 @@ app.use(express.static('public'));
const cache = new Map(); const cache = new Map();
const CACHE_DURATION = 24 * 60 * 60 * 1000; const CACHE_DURATION = 24 * 60 * 60 * 1000;
async function scrapePDGARating(pdgaNumber) { async function fetchPlayerDataHTTP(pdgaNumber) {
return new Promise((resolve, reject) => {
const options = {
hostname: 'www.pdga.com',
port: 443,
path: `/player/${pdgaNumber}`,
method: 'GET',
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
},
timeout: 30000
};
const req = https.request(options, (res) => {
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
if (res.statusCode === 200) {
resolve(data);
} else {
reject(new Error(`HTTP ${res.statusCode}`));
}
});
});
req.on('error', (error) => {
reject(error);
});
req.on('timeout', () => {
req.destroy();
reject(new Error('Request timeout'));
});
req.setTimeout(30000);
req.end();
});
}
function parsePlayerData(html, pdgaNumber) {
try {
// Extract player name from title
const nameMatch = html.match(/<title>([^<]+?)\s*\|\s*Professional Disc Golf Association/i);
const name = nameMatch ? nameMatch[1].trim() : 'Unknown';
// Extract current rating - account for HTML tags between "Current Rating:" and the number
const ratingMatch = html.match(/Current Rating:[^>]*>\s*(\d+)/i);
const rating = ratingMatch ? parseInt(ratingMatch[1]) : 0;
// Extract rating change - look for the +/- number in the rating context
const changeMatch = html.match(/Current Rating:[\s\S]*?([+\-]\d+)[\s\S]*?\(as of/i);
const ratingChange = changeMatch ? parseInt(changeMatch[1]) : null;
return {
pdgaNumber,
name: name.replace(/\s*#\d+$/, ''),
rating,
ratingChange,
predictedRating: null
};
} catch (error) {
console.error(`Error parsing data for PDGA ${pdgaNumber}:`, error.message);
return {
pdgaNumber,
name: 'Error',
rating: 0,
ratingChange: null,
predictedRating: null
};
}
}
async function scrapePDGARating(pdgaNumber, retries = 3) {
const cacheKey = `player-${pdgaNumber}`; const cacheKey = `player-${pdgaNumber}`;
const cached = cache.get(cacheKey); const cached = cache.get(cacheKey);
@@ -20,74 +96,37 @@ async function scrapePDGARating(pdgaNumber) {
return cached.data; return cached.data;
} }
const browser = await puppeteer.launch({ for (let attempt = 1; attempt <= retries; attempt++) {
headless: "new", try {
args: [ console.log(`Attempt ${attempt}/${retries} for PDGA ${pdgaNumber} (using HTTP)`);
'--no-sandbox',
'--disable-setuid-sandbox', const html = await fetchPlayerDataHTTP(pdgaNumber);
'--disable-dev-shm-usage', const result = parsePlayerData(html, pdgaNumber);
'--disable-accelerated-2d-canvas',
'--no-first-run', cache.set(cacheKey, {
'--no-zygote', data: result,
'--disable-gpu' timestamp: Date.now()
] });
});
const page = await browser.newPage(); console.log(`Successfully scraped PDGA ${pdgaNumber} on attempt ${attempt}`);
return result;
try {
const url = `https://www.pdga.com/player/${pdgaNumber}`; } catch (error) {
await page.goto(url, { waitUntil: 'networkidle2' }); console.error(`Attempt ${attempt}/${retries} failed for PDGA ${pdgaNumber}:`, error.message);
const playerName = await page.$eval('h1', el => { if (attempt === retries) {
const text = el.innerText.trim(); return {
return text.replace(/\s*#\d+$/, ''); pdgaNumber,
}); name: 'Error',
rating: 0,
const ratingData = await page.evaluate(() => { ratingChange: null,
const elements = document.querySelectorAll('li'); predictedRating: null
for (const el of elements) { };
const text = el.innerText || el.textContent;
if (text.includes('Current Rating:')) {
const ratingMatch = text.match(/Current Rating:\s*(\d+)/);
// Look for rating change pattern: "Current Rating: 911 +6 (as of..."
const changeMatch = text.match(/Current Rating:\s*\d+\s+([+\-]\d+)\s+\(as of/);
return {
rating: ratingMatch ? ratingMatch[1] : null,
change: changeMatch ? changeMatch[1] : null
};
}
} }
return { rating: null, change: null };
}); // Wait before retry
await new Promise(resolve => setTimeout(resolve, 2000 * attempt));
await browser.close(); }
const result = {
pdgaNumber,
name: playerName,
rating: ratingData.rating ? parseInt(ratingData.rating) : 0,
ratingChange: ratingData.change ? parseInt(ratingData.change) : null,
predictedRating: null
};
cache.set(cacheKey, {
data: result,
timestamp: Date.now()
});
return result;
} catch (error) {
console.error(`Error scraping PDGA ${pdgaNumber}:`, error);
await browser.close();
return {
pdgaNumber,
name: 'Error',
rating: 0,
ratingChange: null,
predictedRating: null
};
} }
} }
@@ -276,7 +315,7 @@ function calculateStandardDeviation(ratings) {
return Math.sqrt(variance); return Math.sqrt(variance);
} }
async function getAllRatings() { async function getAllRatings(progressCallback = null) {
try { try {
const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8') const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8')
.split('\n') .split('\n')
@@ -284,11 +323,58 @@ async function getAllRatings() {
.filter(num => num); .filter(num => num);
const ratings = []; const ratings = [];
const total = pdgaNumbers.length;
for (const pdgaNumber of pdgaNumbers) { for (let i = 0; i < pdgaNumbers.length; i++) {
console.log(`Scraping PDGA ${pdgaNumber}...`); const pdgaNumber = pdgaNumbers[i];
const playerData = await scrapePDGARating(pdgaNumber); console.log(`Scraping PDGA ${pdgaNumber}... (${i + 1}/${total})`);
ratings.push(playerData);
if (progressCallback) {
progressCallback({
current: i + 1,
total,
pdgaNumber,
status: 'loading'
});
}
try {
const playerData = await scrapePDGARating(pdgaNumber);
ratings.push(playerData);
if (progressCallback) {
progressCallback({
current: i + 1,
total,
pdgaNumber,
status: 'completed',
name: playerData.name
});
}
// Longer delay to avoid overwhelming the server
await new Promise(resolve => setTimeout(resolve, 1000));
} catch (error) {
console.error(`Failed to scrape PDGA ${pdgaNumber}:`, error.message);
const errorData = {
pdgaNumber,
name: 'Error',
rating: 0,
ratingChange: null,
predictedRating: null
};
ratings.push(errorData);
if (progressCallback) {
progressCallback({
current: i + 1,
total,
pdgaNumber,
status: 'error',
name: 'Error'
});
}
}
} }
return ratings.sort((a, b) => b.rating - a.rating); return ratings.sort((a, b) => b.rating - a.rating);
@@ -311,10 +397,37 @@ app.get('/api/ratings', async (req, res) => {
} }
}); });
app.get('/api/ratings/progress', (req, res) => {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Headers': 'Cache-Control'
});
const progressCallback = (progress) => {
res.write(`data: ${JSON.stringify(progress)}\n\n`);
};
getAllRatings(progressCallback).then(ratings => {
res.write(`data: ${JSON.stringify({ status: 'complete', ratings })}\n\n`);
res.end();
}).catch(error => {
res.write(`data: ${JSON.stringify({ status: 'error', error: error.message })}\n\n`);
res.end();
});
req.on('close', () => {
res.end();
});
});
app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => { app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => {
let browser = null;
try { try {
const { pdgaNumber } = req.params; const { pdgaNumber } = req.params;
const browser = await puppeteer.launch({ browser = await puppeteer.launch({
headless: "new", headless: "new",
args: [ args: [
'--no-sandbox', '--no-sandbox',
@@ -331,13 +444,21 @@ app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => {
const predictedRating = await getPredictedRating(browser, pdgaNumber); const predictedRating = await getPredictedRating(browser, pdgaNumber);
await browser.close(); await browser.close();
browser = null;
res.json({ res.json({
pdgaNumber: parseInt(pdgaNumber), pdgaNumber: parseInt(pdgaNumber),
predictedRating predictedRating
}); });
} catch (error) { } catch (error) {
console.error('Error calculating predicted rating:', error); console.error('Error calculating predicted rating:', error.message || error);
if (browser) {
try {
await browser.close();
} catch (closeError) {
console.error('Error closing browser:', closeError.message);
}
}
res.status(500).json({ error: 'Failed to calculate predicted rating' }); res.status(500).json({ error: 'Failed to calculate predicted rating' });
} }
}); });