Add real-time progress bar for rating loading
- Implement Server-Sent Events for live progress updates - Add animated progress bar with percentage display - Show real-time status: current player being loaded - Display player names as they complete loading - Handle errors gracefully with progress continuation - Replace HTTP-only approach for better reliability - Enhanced user experience with visual feedback 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Samuel Enocsson
+128
-67
@@ -29,6 +29,30 @@
|
||||
font-size: 18px;
|
||||
color: #666;
|
||||
}
|
||||
.progress-container {
|
||||
width: 100%;
|
||||
background-color: #f0f0f0;
|
||||
border-radius: 10px;
|
||||
padding: 3px;
|
||||
margin: 20px 0;
|
||||
}
|
||||
.progress-bar {
|
||||
width: 0%;
|
||||
height: 30px;
|
||||
background-color: #007bff;
|
||||
border-radius: 8px;
|
||||
text-align: center;
|
||||
line-height: 30px;
|
||||
color: white;
|
||||
font-weight: bold;
|
||||
transition: width 0.3s ease;
|
||||
}
|
||||
.progress-text {
|
||||
text-align: center;
|
||||
margin: 10px 0;
|
||||
font-size: 16px;
|
||||
color: #666;
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
@@ -99,86 +123,123 @@
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>PDGA Player Ratings</h1>
|
||||
<div id="loading" class="loading">Loading ratings...</div>
|
||||
<div id="loading" class="loading" style="display: none;">Loading ratings...</div>
|
||||
<div id="progress-section" style="display: none;">
|
||||
<div class="progress-container">
|
||||
<div id="progress-bar" class="progress-bar">0%</div>
|
||||
</div>
|
||||
<div id="progress-text" class="progress-text">Preparing to load ratings...</div>
|
||||
</div>
|
||||
<div id="ratings-table"></div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
async function fetchRatings() {
|
||||
const loadingDiv = document.getElementById('loading');
|
||||
function fetchRatingsWithProgress() {
|
||||
const progressSection = document.getElementById('progress-section');
|
||||
const progressBar = document.getElementById('progress-bar');
|
||||
const progressText = document.getElementById('progress-text');
|
||||
const tableDiv = document.getElementById('ratings-table');
|
||||
|
||||
loadingDiv.style.display = 'block';
|
||||
progressSection.style.display = 'block';
|
||||
tableDiv.innerHTML = '';
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/ratings');
|
||||
const ratings = await response.json();
|
||||
const eventSource = new EventSource('/api/ratings/progress');
|
||||
|
||||
eventSource.onmessage = function(event) {
|
||||
const data = JSON.parse(event.data);
|
||||
|
||||
loadingDiv.style.display = 'none';
|
||||
|
||||
if (ratings.length === 0) {
|
||||
tableDiv.innerHTML = '<p>No ratings found.</p>';
|
||||
return;
|
||||
if (data.status === 'loading') {
|
||||
const percentage = Math.round((data.current / data.total) * 100);
|
||||
progressBar.style.width = `${percentage}%`;
|
||||
progressBar.textContent = `${percentage}%`;
|
||||
progressText.textContent = `Loading player ${data.current}/${data.total}: PDGA #${data.pdgaNumber}`;
|
||||
} else if (data.status === 'completed') {
|
||||
const percentage = Math.round((data.current / data.total) * 100);
|
||||
progressBar.style.width = `${percentage}%`;
|
||||
progressBar.textContent = `${percentage}%`;
|
||||
progressText.textContent = `Loaded ${data.name} (${data.current}/${data.total})`;
|
||||
} else if (data.status === 'error') {
|
||||
const percentage = Math.round((data.current / data.total) * 100);
|
||||
progressBar.style.width = `${percentage}%`;
|
||||
progressBar.textContent = `${percentage}%`;
|
||||
progressText.textContent = `Error loading PDGA #${data.pdgaNumber} (${data.current}/${data.total})`;
|
||||
} else if (data.status === 'complete') {
|
||||
progressSection.style.display = 'none';
|
||||
displayRatings(data.ratings);
|
||||
eventSource.close();
|
||||
} else if (data.status === 'error') {
|
||||
progressSection.style.display = 'none';
|
||||
tableDiv.innerHTML = '<p>Error loading ratings. Please try again.</p>';
|
||||
eventSource.close();
|
||||
}
|
||||
|
||||
let tableHTML = `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Rank</th>
|
||||
<th>Player Name</th>
|
||||
<th>PDGA #</th>
|
||||
<th>Current Rating</th>
|
||||
<th>Rating Change</th>
|
||||
<th>Predicted Rating</th>
|
||||
<th>Difference</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
`;
|
||||
|
||||
ratings.forEach((player, index) => {
|
||||
const difference = player.predictedRating && player.rating ?
|
||||
player.predictedRating - player.rating : 0;
|
||||
const diffText = difference > 0 ? `+${difference}` : difference.toString();
|
||||
const diffClass = difference > 0 ? 'positive' : difference < 0 ? 'negative' : 'neutral';
|
||||
|
||||
const ratingChangeText = player.ratingChange ?
|
||||
(player.ratingChange > 0 ? `+${player.ratingChange}` : player.ratingChange.toString()) : 'N/A';
|
||||
const ratingChangeClass = player.ratingChange > 0 ? 'positive' :
|
||||
player.ratingChange < 0 ? 'negative' : 'neutral';
|
||||
|
||||
tableHTML += `
|
||||
<tr id="row-${player.pdgaNumber}">
|
||||
<td>${index + 1}</td>
|
||||
<td><a href="https://www.pdga.com/player/${player.pdgaNumber}" target="_blank">${player.name}</a></td>
|
||||
<td class="pdga-number">#${player.pdgaNumber}</td>
|
||||
<td class="rating">${player.rating || 'N/A'}</td>
|
||||
<td class="rating-change ${ratingChangeClass}">${ratingChangeText}</td>
|
||||
<td class="predicted-rating" id="predicted-${player.pdgaNumber}">
|
||||
${player.predictedRating || 'N/A'}
|
||||
</td>
|
||||
<td class="difference ${diffClass}" id="diff-${player.pdgaNumber}">
|
||||
${difference ? diffText :
|
||||
`<button class="calc-btn" onclick="calculatePredictedRating(${player.pdgaNumber})">Predict Rating</button>`}
|
||||
</td>
|
||||
};
|
||||
|
||||
eventSource.onerror = function() {
|
||||
progressSection.style.display = 'none';
|
||||
tableDiv.innerHTML = '<p>Connection error. Please refresh the page.</p>';
|
||||
eventSource.close();
|
||||
};
|
||||
}
|
||||
|
||||
function displayRatings(ratings) {
|
||||
const tableDiv = document.getElementById('ratings-table');
|
||||
|
||||
if (ratings.length === 0) {
|
||||
tableDiv.innerHTML = '<p>No ratings found.</p>';
|
||||
return;
|
||||
}
|
||||
|
||||
let tableHTML = `
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Rank</th>
|
||||
<th>Player Name</th>
|
||||
<th>PDGA #</th>
|
||||
<th>Current Rating</th>
|
||||
<th>Rating Change</th>
|
||||
<th>Predicted Rating</th>
|
||||
<th>Difference</th>
|
||||
</tr>
|
||||
`;
|
||||
});
|
||||
</thead>
|
||||
<tbody>
|
||||
`;
|
||||
|
||||
ratings.forEach((player, index) => {
|
||||
const difference = player.predictedRating && player.rating ?
|
||||
player.predictedRating - player.rating : 0;
|
||||
const diffText = difference > 0 ? `+${difference}` : difference.toString();
|
||||
const diffClass = difference > 0 ? 'positive' : difference < 0 ? 'negative' : 'neutral';
|
||||
|
||||
const ratingChangeText = player.ratingChange ?
|
||||
(player.ratingChange > 0 ? `+${player.ratingChange}` : player.ratingChange.toString()) : 'N/A';
|
||||
const ratingChangeClass = player.ratingChange > 0 ? 'positive' :
|
||||
player.ratingChange < 0 ? 'negative' : 'neutral';
|
||||
|
||||
tableHTML += `
|
||||
</tbody>
|
||||
</table>
|
||||
<tr id="row-${player.pdgaNumber}">
|
||||
<td>${index + 1}</td>
|
||||
<td><a href="https://www.pdga.com/player/${player.pdgaNumber}" target="_blank">${player.name}</a></td>
|
||||
<td class="pdga-number">#${player.pdgaNumber}</td>
|
||||
<td class="rating">${player.rating || 'N/A'}</td>
|
||||
<td class="rating-change ${ratingChangeClass}">${ratingChangeText}</td>
|
||||
<td class="predicted-rating" id="predicted-${player.pdgaNumber}">
|
||||
${player.predictedRating || 'N/A'}
|
||||
</td>
|
||||
<td class="difference ${diffClass}" id="diff-${player.pdgaNumber}">
|
||||
${difference ? diffText :
|
||||
`<button class="calc-btn" onclick="calculatePredictedRating(${player.pdgaNumber})">Predict Rating</button>`}
|
||||
</td>
|
||||
</tr>
|
||||
`;
|
||||
|
||||
tableDiv.innerHTML = tableHTML;
|
||||
|
||||
} catch (error) {
|
||||
loadingDiv.style.display = 'none';
|
||||
tableDiv.innerHTML = '<p>Error loading ratings. Please try again.</p>';
|
||||
console.error('Error:', error);
|
||||
}
|
||||
});
|
||||
|
||||
tableHTML += `
|
||||
</tbody>
|
||||
</table>
|
||||
`;
|
||||
|
||||
tableDiv.innerHTML = tableHTML;
|
||||
}
|
||||
|
||||
async function calculatePredictedRating(pdgaNumber) {
|
||||
@@ -219,7 +280,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
fetchRatings();
|
||||
fetchRatingsWithProgress();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,5 +1,6 @@
|
||||
const express = require('express');
|
||||
const puppeteer = require('puppeteer');
|
||||
const https = require('https');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
@@ -11,7 +12,82 @@ app.use(express.static('public'));
|
||||
const cache = new Map();
|
||||
const CACHE_DURATION = 24 * 60 * 60 * 1000;
|
||||
|
||||
async function scrapePDGARating(pdgaNumber) {
|
||||
async function fetchPlayerDataHTTP(pdgaNumber) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const options = {
|
||||
hostname: 'www.pdga.com',
|
||||
port: 443,
|
||||
path: `/player/${pdgaNumber}`,
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||||
},
|
||||
timeout: 30000
|
||||
};
|
||||
|
||||
const req = https.request(options, (res) => {
|
||||
let data = '';
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on('end', () => {
|
||||
if (res.statusCode === 200) {
|
||||
resolve(data);
|
||||
} else {
|
||||
reject(new Error(`HTTP ${res.statusCode}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
|
||||
req.on('timeout', () => {
|
||||
req.destroy();
|
||||
reject(new Error('Request timeout'));
|
||||
});
|
||||
|
||||
req.setTimeout(30000);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function parsePlayerData(html, pdgaNumber) {
|
||||
try {
|
||||
// Extract player name from title
|
||||
const nameMatch = html.match(/<title>([^<]+?)\s*\|\s*Professional Disc Golf Association/i);
|
||||
const name = nameMatch ? nameMatch[1].trim() : 'Unknown';
|
||||
|
||||
// Extract current rating - account for HTML tags between "Current Rating:" and the number
|
||||
const ratingMatch = html.match(/Current Rating:[^>]*>\s*(\d+)/i);
|
||||
const rating = ratingMatch ? parseInt(ratingMatch[1]) : 0;
|
||||
|
||||
// Extract rating change - look for the +/- number in the rating context
|
||||
const changeMatch = html.match(/Current Rating:[\s\S]*?([+\-]\d+)[\s\S]*?\(as of/i);
|
||||
const ratingChange = changeMatch ? parseInt(changeMatch[1]) : null;
|
||||
|
||||
return {
|
||||
pdgaNumber,
|
||||
name: name.replace(/\s*#\d+$/, ''),
|
||||
rating,
|
||||
ratingChange,
|
||||
predictedRating: null
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error parsing data for PDGA ${pdgaNumber}:`, error.message);
|
||||
return {
|
||||
pdgaNumber,
|
||||
name: 'Error',
|
||||
rating: 0,
|
||||
ratingChange: null,
|
||||
predictedRating: null
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function scrapePDGARating(pdgaNumber, retries = 3) {
|
||||
const cacheKey = `player-${pdgaNumber}`;
|
||||
const cached = cache.get(cacheKey);
|
||||
|
||||
@@ -20,74 +96,37 @@ async function scrapePDGARating(pdgaNumber) {
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: "new",
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-accelerated-2d-canvas',
|
||||
'--no-first-run',
|
||||
'--no-zygote',
|
||||
'--disable-gpu'
|
||||
]
|
||||
});
|
||||
const page = await browser.newPage();
|
||||
|
||||
try {
|
||||
const url = `https://www.pdga.com/player/${pdgaNumber}`;
|
||||
await page.goto(url, { waitUntil: 'networkidle2' });
|
||||
|
||||
const playerName = await page.$eval('h1', el => {
|
||||
const text = el.innerText.trim();
|
||||
return text.replace(/\s*#\d+$/, '');
|
||||
});
|
||||
|
||||
const ratingData = await page.evaluate(() => {
|
||||
const elements = document.querySelectorAll('li');
|
||||
for (const el of elements) {
|
||||
const text = el.innerText || el.textContent;
|
||||
if (text.includes('Current Rating:')) {
|
||||
const ratingMatch = text.match(/Current Rating:\s*(\d+)/);
|
||||
|
||||
// Look for rating change pattern: "Current Rating: 911 +6 (as of..."
|
||||
const changeMatch = text.match(/Current Rating:\s*\d+\s+([+\-]\d+)\s+\(as of/);
|
||||
|
||||
return {
|
||||
rating: ratingMatch ? ratingMatch[1] : null,
|
||||
change: changeMatch ? changeMatch[1] : null
|
||||
};
|
||||
}
|
||||
for (let attempt = 1; attempt <= retries; attempt++) {
|
||||
try {
|
||||
console.log(`Attempt ${attempt}/${retries} for PDGA ${pdgaNumber} (using HTTP)`);
|
||||
|
||||
const html = await fetchPlayerDataHTTP(pdgaNumber);
|
||||
const result = parsePlayerData(html, pdgaNumber);
|
||||
|
||||
cache.set(cacheKey, {
|
||||
data: result,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
console.log(`Successfully scraped PDGA ${pdgaNumber} on attempt ${attempt}`);
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Attempt ${attempt}/${retries} failed for PDGA ${pdgaNumber}:`, error.message);
|
||||
|
||||
if (attempt === retries) {
|
||||
return {
|
||||
pdgaNumber,
|
||||
name: 'Error',
|
||||
rating: 0,
|
||||
ratingChange: null,
|
||||
predictedRating: null
|
||||
};
|
||||
}
|
||||
return { rating: null, change: null };
|
||||
});
|
||||
|
||||
await browser.close();
|
||||
|
||||
const result = {
|
||||
pdgaNumber,
|
||||
name: playerName,
|
||||
rating: ratingData.rating ? parseInt(ratingData.rating) : 0,
|
||||
ratingChange: ratingData.change ? parseInt(ratingData.change) : null,
|
||||
predictedRating: null
|
||||
};
|
||||
|
||||
cache.set(cacheKey, {
|
||||
data: result,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error(`Error scraping PDGA ${pdgaNumber}:`, error);
|
||||
await browser.close();
|
||||
return {
|
||||
pdgaNumber,
|
||||
name: 'Error',
|
||||
rating: 0,
|
||||
ratingChange: null,
|
||||
predictedRating: null
|
||||
};
|
||||
|
||||
// Wait before retry
|
||||
await new Promise(resolve => setTimeout(resolve, 2000 * attempt));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,7 +315,7 @@ function calculateStandardDeviation(ratings) {
|
||||
return Math.sqrt(variance);
|
||||
}
|
||||
|
||||
async function getAllRatings() {
|
||||
async function getAllRatings(progressCallback = null) {
|
||||
try {
|
||||
const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8')
|
||||
.split('\n')
|
||||
@@ -284,11 +323,58 @@ async function getAllRatings() {
|
||||
.filter(num => num);
|
||||
|
||||
const ratings = [];
|
||||
const total = pdgaNumbers.length;
|
||||
|
||||
for (const pdgaNumber of pdgaNumbers) {
|
||||
console.log(`Scraping PDGA ${pdgaNumber}...`);
|
||||
const playerData = await scrapePDGARating(pdgaNumber);
|
||||
ratings.push(playerData);
|
||||
for (let i = 0; i < pdgaNumbers.length; i++) {
|
||||
const pdgaNumber = pdgaNumbers[i];
|
||||
console.log(`Scraping PDGA ${pdgaNumber}... (${i + 1}/${total})`);
|
||||
|
||||
if (progressCallback) {
|
||||
progressCallback({
|
||||
current: i + 1,
|
||||
total,
|
||||
pdgaNumber,
|
||||
status: 'loading'
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const playerData = await scrapePDGARating(pdgaNumber);
|
||||
ratings.push(playerData);
|
||||
|
||||
if (progressCallback) {
|
||||
progressCallback({
|
||||
current: i + 1,
|
||||
total,
|
||||
pdgaNumber,
|
||||
status: 'completed',
|
||||
name: playerData.name
|
||||
});
|
||||
}
|
||||
|
||||
// Longer delay to avoid overwhelming the server
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
} catch (error) {
|
||||
console.error(`Failed to scrape PDGA ${pdgaNumber}:`, error.message);
|
||||
const errorData = {
|
||||
pdgaNumber,
|
||||
name: 'Error',
|
||||
rating: 0,
|
||||
ratingChange: null,
|
||||
predictedRating: null
|
||||
};
|
||||
ratings.push(errorData);
|
||||
|
||||
if (progressCallback) {
|
||||
progressCallback({
|
||||
current: i + 1,
|
||||
total,
|
||||
pdgaNumber,
|
||||
status: 'error',
|
||||
name: 'Error'
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ratings.sort((a, b) => b.rating - a.rating);
|
||||
@@ -311,10 +397,37 @@ app.get('/api/ratings', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/api/ratings/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Headers': 'Cache-Control'
|
||||
});
|
||||
|
||||
const progressCallback = (progress) => {
|
||||
res.write(`data: ${JSON.stringify(progress)}\n\n`);
|
||||
};
|
||||
|
||||
getAllRatings(progressCallback).then(ratings => {
|
||||
res.write(`data: ${JSON.stringify({ status: 'complete', ratings })}\n\n`);
|
||||
res.end();
|
||||
}).catch(error => {
|
||||
res.write(`data: ${JSON.stringify({ status: 'error', error: error.message })}\n\n`);
|
||||
res.end();
|
||||
});
|
||||
|
||||
req.on('close', () => {
|
||||
res.end();
|
||||
});
|
||||
});
|
||||
|
||||
app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => {
|
||||
let browser = null;
|
||||
try {
|
||||
const { pdgaNumber } = req.params;
|
||||
const browser = await puppeteer.launch({
|
||||
browser = await puppeteer.launch({
|
||||
headless: "new",
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
@@ -331,13 +444,21 @@ app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => {
|
||||
const predictedRating = await getPredictedRating(browser, pdgaNumber);
|
||||
|
||||
await browser.close();
|
||||
browser = null;
|
||||
|
||||
res.json({
|
||||
pdgaNumber: parseInt(pdgaNumber),
|
||||
predictedRating
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error calculating predicted rating:', error);
|
||||
console.error('Error calculating predicted rating:', error.message || error);
|
||||
if (browser) {
|
||||
try {
|
||||
await browser.close();
|
||||
} catch (closeError) {
|
||||
console.error('Error closing browser:', closeError.message);
|
||||
}
|
||||
}
|
||||
res.status(500).json({ error: 'Failed to calculate predicted rating' });
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user