diff --git a/index.html b/index.html index ffe2c88..26e270d 100644 --- a/index.html +++ b/index.html @@ -29,6 +29,30 @@ font-size: 18px; color: #666; } + .progress-container { + width: 100%; + background-color: #f0f0f0; + border-radius: 10px; + padding: 3px; + margin: 20px 0; + } + .progress-bar { + width: 0%; + height: 30px; + background-color: #007bff; + border-radius: 8px; + text-align: center; + line-height: 30px; + color: white; + font-weight: bold; + transition: width 0.3s ease; + } + .progress-text { + text-align: center; + margin: 10px 0; + font-size: 16px; + color: #666; + } table { width: 100%; border-collapse: collapse; @@ -99,86 +123,123 @@

PDGA Player Ratings

-
Loading ratings...
+ +
\ No newline at end of file diff --git a/server.js b/server.js index 0a0c638..2f5fd67 100644 --- a/server.js +++ b/server.js @@ -1,5 +1,6 @@ const express = require('express'); const puppeteer = require('puppeteer'); +const https = require('https'); const fs = require('fs'); const path = require('path'); @@ -11,7 +12,82 @@ app.use(express.static('public')); const cache = new Map(); const CACHE_DURATION = 24 * 60 * 60 * 1000; -async function scrapePDGARating(pdgaNumber) { +async function fetchPlayerDataHTTP(pdgaNumber) { + return new Promise((resolve, reject) => { + const options = { + hostname: 'www.pdga.com', + port: 443, + path: `/player/${pdgaNumber}`, + method: 'GET', + headers: { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + }, + timeout: 30000 + }; + + const req = https.request(options, (res) => { + let data = ''; + res.on('data', (chunk) => { + data += chunk; + }); + + res.on('end', () => { + if (res.statusCode === 200) { + resolve(data); + } else { + reject(new Error(`HTTP ${res.statusCode}`)); + } + }); + }); + + req.on('error', (error) => { + reject(error); + }); + + req.on('timeout', () => { + req.destroy(); + reject(new Error('Request timeout')); + }); + + req.setTimeout(30000); + req.end(); + }); +} + +function parsePlayerData(html, pdgaNumber) { + try { + // Extract player name from title + const nameMatch = html.match(/([^<]+?)\s*\|\s*Professional Disc Golf Association/i); + const name = nameMatch ? nameMatch[1].trim() : 'Unknown'; + + // Extract current rating - account for HTML tags between "Current Rating:" and the number + const ratingMatch = html.match(/Current Rating:[^>]*>\s*(\d+)/i); + const rating = ratingMatch ? parseInt(ratingMatch[1]) : 0; + + // Extract rating change - look for the +/- number in the rating context + const changeMatch = html.match(/Current Rating:[\s\S]*?([+\-]\d+)[\s\S]*?\(as of/i); + const ratingChange = changeMatch ? parseInt(changeMatch[1]) : null; + + return { + pdgaNumber, + name: name.replace(/\s*#\d+$/, ''), + rating, + ratingChange, + predictedRating: null + }; + } catch (error) { + console.error(`Error parsing data for PDGA ${pdgaNumber}:`, error.message); + return { + pdgaNumber, + name: 'Error', + rating: 0, + ratingChange: null, + predictedRating: null + }; + } +} + +async function scrapePDGARating(pdgaNumber, retries = 3) { const cacheKey = `player-${pdgaNumber}`; const cached = cache.get(cacheKey); @@ -20,74 +96,37 @@ async function scrapePDGARating(pdgaNumber) { return cached.data; } - const browser = await puppeteer.launch({ - headless: "new", - args: [ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', - '--disable-accelerated-2d-canvas', - '--no-first-run', - '--no-zygote', - '--disable-gpu' - ] - }); - const page = await browser.newPage(); - - try { - const url = `https://www.pdga.com/player/${pdgaNumber}`; - await page.goto(url, { waitUntil: 'networkidle2' }); - - const playerName = await page.$eval('h1', el => { - const text = el.innerText.trim(); - return text.replace(/\s*#\d+$/, ''); - }); - - const ratingData = await page.evaluate(() => { - const elements = document.querySelectorAll('li'); - for (const el of elements) { - const text = el.innerText || el.textContent; - if (text.includes('Current Rating:')) { - const ratingMatch = text.match(/Current Rating:\s*(\d+)/); - - // Look for rating change pattern: "Current Rating: 911 +6 (as of..." - const changeMatch = text.match(/Current Rating:\s*\d+\s+([+\-]\d+)\s+\(as of/); - - return { - rating: ratingMatch ? ratingMatch[1] : null, - change: changeMatch ? changeMatch[1] : null - }; - } + for (let attempt = 1; attempt <= retries; attempt++) { + try { + console.log(`Attempt ${attempt}/${retries} for PDGA ${pdgaNumber} (using HTTP)`); + + const html = await fetchPlayerDataHTTP(pdgaNumber); + const result = parsePlayerData(html, pdgaNumber); + + cache.set(cacheKey, { + data: result, + timestamp: Date.now() + }); + + console.log(`Successfully scraped PDGA ${pdgaNumber} on attempt ${attempt}`); + return result; + + } catch (error) { + console.error(`Attempt ${attempt}/${retries} failed for PDGA ${pdgaNumber}:`, error.message); + + if (attempt === retries) { + return { + pdgaNumber, + name: 'Error', + rating: 0, + ratingChange: null, + predictedRating: null + }; } - return { rating: null, change: null }; - }); - - await browser.close(); - - const result = { - pdgaNumber, - name: playerName, - rating: ratingData.rating ? parseInt(ratingData.rating) : 0, - ratingChange: ratingData.change ? parseInt(ratingData.change) : null, - predictedRating: null - }; - - cache.set(cacheKey, { - data: result, - timestamp: Date.now() - }); - - return result; - } catch (error) { - console.error(`Error scraping PDGA ${pdgaNumber}:`, error); - await browser.close(); - return { - pdgaNumber, - name: 'Error', - rating: 0, - ratingChange: null, - predictedRating: null - }; + + // Wait before retry + await new Promise(resolve => setTimeout(resolve, 2000 * attempt)); + } } } @@ -276,7 +315,7 @@ function calculateStandardDeviation(ratings) { return Math.sqrt(variance); } -async function getAllRatings() { +async function getAllRatings(progressCallback = null) { try { const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8') .split('\n') @@ -284,11 +323,58 @@ async function getAllRatings() { .filter(num => num); const ratings = []; + const total = pdgaNumbers.length; - for (const pdgaNumber of pdgaNumbers) { - console.log(`Scraping PDGA ${pdgaNumber}...`); - const playerData = await scrapePDGARating(pdgaNumber); - ratings.push(playerData); + for (let i = 0; i < pdgaNumbers.length; i++) { + const pdgaNumber = pdgaNumbers[i]; + console.log(`Scraping PDGA ${pdgaNumber}... (${i + 1}/${total})`); + + if (progressCallback) { + progressCallback({ + current: i + 1, + total, + pdgaNumber, + status: 'loading' + }); + } + + try { + const playerData = await scrapePDGARating(pdgaNumber); + ratings.push(playerData); + + if (progressCallback) { + progressCallback({ + current: i + 1, + total, + pdgaNumber, + status: 'completed', + name: playerData.name + }); + } + + // Longer delay to avoid overwhelming the server + await new Promise(resolve => setTimeout(resolve, 1000)); + } catch (error) { + console.error(`Failed to scrape PDGA ${pdgaNumber}:`, error.message); + const errorData = { + pdgaNumber, + name: 'Error', + rating: 0, + ratingChange: null, + predictedRating: null + }; + ratings.push(errorData); + + if (progressCallback) { + progressCallback({ + current: i + 1, + total, + pdgaNumber, + status: 'error', + name: 'Error' + }); + } + } } return ratings.sort((a, b) => b.rating - a.rating); @@ -311,10 +397,37 @@ app.get('/api/ratings', async (req, res) => { } }); +app.get('/api/ratings/progress', (req, res) => { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Headers': 'Cache-Control' + }); + + const progressCallback = (progress) => { + res.write(`data: ${JSON.stringify(progress)}\n\n`); + }; + + getAllRatings(progressCallback).then(ratings => { + res.write(`data: ${JSON.stringify({ status: 'complete', ratings })}\n\n`); + res.end(); + }).catch(error => { + res.write(`data: ${JSON.stringify({ status: 'error', error: error.message })}\n\n`); + res.end(); + }); + + req.on('close', () => { + res.end(); + }); +}); + app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => { + let browser = null; try { const { pdgaNumber } = req.params; - const browser = await puppeteer.launch({ + browser = await puppeteer.launch({ headless: "new", args: [ '--no-sandbox', @@ -331,13 +444,21 @@ app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => { const predictedRating = await getPredictedRating(browser, pdgaNumber); await browser.close(); + browser = null; res.json({ pdgaNumber: parseInt(pdgaNumber), predictedRating }); } catch (error) { - console.error('Error calculating predicted rating:', error); + console.error('Error calculating predicted rating:', error.message || error); + if (browser) { + try { + await browser.close(); + } catch (closeError) { + console.error('Error closing browser:', closeError.message); + } + } res.status(500).json({ error: 'Failed to calculate predicted rating' }); } });