Initial commit: PDGA rating scraper and predictor

- Web scraping app for PDGA player ratings - Current rating extraction from player pages - Tournament round rating scraping for predictions - Statistical rating prediction algorithm - Interactive table with on-demand calculations - Caching for performance optimization 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-12 11:13:13 +02:00
commit deb162dc13
7 changed files with 2716 additions and 0 deletions
@@ -0,0 +1,4 @@
+node_modules/
+.env
+.cache/
+*.log
@@ -0,0 +1,32 @@
+# PDGA Ratings Scraper
+
+A web application that scrapes PDGA player ratings and displays them in a sorted table.
+
+## Setup
+
+1. Install dependencies:
+```bash
+npm install
+```
+
+2. Add PDGA numbers to `pdga-numbers.txt` (one per line)
+
+3. Start the server:
+```bash
+npm start
+```
+
+4. Open http://localhost:3000 in your browser
+
+## Usage
+
+- Edit `pdga-numbers.txt` to add/remove players
+- Click "Refresh Ratings" to scrape current ratings
+- Players are automatically sorted by rating (highest first)
+
+## Features
+
+- Web scraping of PDGA player ratings
+- Sortable table display
+- Easy player management via text file
+- Real-time rating updates
@@ -0,0 +1,232 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>PDGA Ratings</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 40px;
+            background-color: #f5f5f5;
+        }
+        .container {
+            max-width: 800px;
+            margin: 0 auto;
+            background: white;
+            padding: 30px;
+            border-radius: 8px;
+            box-shadow: 0 2px 10px rgba(0,0,0,0.1);
+        }
+        h1 {
+            color: #333;
+            text-align: center;
+            margin-bottom: 30px;
+        }
+        .loading {
+            text-align: center;
+            padding: 20px;
+            font-size: 18px;
+            color: #666;
+        }
+        .refresh-btn {
+            background-color: #007bff;
+            color: white;
+            padding: 10px 20px;
+            border: none;
+            border-radius: 4px;
+            cursor: pointer;
+            font-size: 16px;
+            margin-bottom: 20px;
+        }
+        .refresh-btn:hover {
+            background-color: #0056b3;
+        }
+        table {
+            width: 100%;
+            border-collapse: collapse;
+            margin-top: 20px;
+        }
+        th, td {
+            padding: 12px;
+            text-align: left;
+            border-bottom: 1px solid #ddd;
+        }
+        th {
+            background-color: #f8f9fa;
+            font-weight: bold;
+            color: #495057;
+        }
+        tr:hover {
+            background-color: #f5f5f5;
+        }
+        .rating {
+            font-weight: bold;
+            color: #007bff;
+        }
+        .pdga-number {
+            color: #6c757d;
+            font-size: 14px;
+        }
+        .difference {
+            font-weight: bold;
+        }
+        .positive {
+            color: #28a745;
+        }
+        .negative {
+            color: #dc3545;
+        }
+        .neutral {
+            color: #6c757d;
+        }
+        .calc-btn {
+            background-color: #17a2b8;
+            color: white;
+            border: none;
+            padding: 5px 10px;
+            border-radius: 3px;
+            cursor: pointer;
+            font-size: 12px;
+        }
+        .calc-btn:hover {
+            background-color: #138496;
+        }
+        .calc-btn:disabled {
+            background-color: #6c757d;
+            cursor: not-allowed;
+        }
+        .rating-change {
+            font-weight: bold;
+            font-size: 14px;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>PDGA Player Ratings</h1>
+        <button class="refresh-btn" onclick="fetchRatings()">Refresh Ratings</button>
+        <div id="loading" class="loading">Loading ratings...</div>
+        <div id="ratings-table"></div>
+    </div>
+
+    <script>
+        async function fetchRatings() {
+            const loadingDiv = document.getElementById('loading');
+            const tableDiv = document.getElementById('ratings-table');
+            
+            loadingDiv.style.display = 'block';
+            tableDiv.innerHTML = '';
+            
+            try {
+                const response = await fetch('/api/ratings');
+                const ratings = await response.json();
+                
+                loadingDiv.style.display = 'none';
+                
+                if (ratings.length === 0) {
+                    tableDiv.innerHTML = '<p>No ratings found.</p>';
+                    return;
+                }
+                
+                let tableHTML = `
+                    <table>
+                        <thead>
+                            <tr>
+                                <th>Rank</th>
+                                <th>Player Name</th>
+                                <th>PDGA #</th>
+                                <th>Current Rating</th>
+                                <th>Rating Change</th>
+                                <th>Predicted Rating</th>
+                                <th>Difference</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                `;
+                
+                ratings.forEach((player, index) => {
+                    const difference = player.predictedRating && player.rating ? 
+                        player.predictedRating - player.rating : 0;
+                    const diffText = difference > 0 ? `+${difference}` : difference.toString();
+                    const diffClass = difference > 0 ? 'positive' : difference < 0 ? 'negative' : 'neutral';
+                    
+                    const ratingChangeText = player.ratingChange ? 
+                        (player.ratingChange > 0 ? `+${player.ratingChange}` : player.ratingChange.toString()) : 'N/A';
+                    const ratingChangeClass = player.ratingChange > 0 ? 'positive' : 
+                                            player.ratingChange < 0 ? 'negative' : 'neutral';
+                    
+                    tableHTML += `
+                        <tr id="row-${player.pdgaNumber}">
+                            <td>${index + 1}</td>
+                            <td>${player.name}</td>
+                            <td class="pdga-number">#${player.pdgaNumber}</td>
+                            <td class="rating">${player.rating || 'N/A'}</td>
+                            <td class="rating-change ${ratingChangeClass}">${ratingChangeText}</td>
+                            <td class="predicted-rating" id="predicted-${player.pdgaNumber}">
+                                ${player.predictedRating || 'N/A'}
+                            </td>
+                            <td class="difference ${diffClass}" id="diff-${player.pdgaNumber}">
+                                ${difference ? diffText : 
+                                    `<button class="calc-btn" onclick="calculatePredictedRating(${player.pdgaNumber})">Calculate Approx Rating</button>`}
+                            </td>
+                        </tr>
+                    `;
+                });
+                
+                tableHTML += `
+                        </tbody>
+                    </table>
+                `;
+                
+                tableDiv.innerHTML = tableHTML;
+                
+            } catch (error) {
+                loadingDiv.style.display = 'none';
+                tableDiv.innerHTML = '<p>Error loading ratings. Please try again.</p>';
+                console.error('Error:', error);
+            }
+        }
+        
+        async function calculatePredictedRating(pdgaNumber) {
+            const button = document.querySelector(`#diff-${pdgaNumber} .calc-btn`);
+            const predictedCell = document.getElementById(`predicted-${pdgaNumber}`);
+            const diffCell = document.getElementById(`diff-${pdgaNumber}`);
+            
+            button.disabled = true;
+            button.textContent = 'Calculating...';
+            
+            try {
+                const response = await fetch(`/api/predicted-rating/${pdgaNumber}`, {
+                    method: 'POST'
+                });
+                
+                const data = await response.json();
+                
+                if (data.predictedRating) {
+                    predictedCell.textContent = data.predictedRating;
+                    
+                    const currentRating = parseInt(document.querySelector(`#row-${pdgaNumber} .rating`).textContent);
+                    const difference = data.predictedRating - currentRating;
+                    const diffText = difference > 0 ? `+${difference}` : difference.toString();
+                    const diffClass = difference > 0 ? 'positive' : difference < 0 ? 'negative' : 'neutral';
+                    
+                    diffCell.className = `difference ${diffClass}`;
+                    diffCell.textContent = diffText;
+                } else {
+                    predictedCell.textContent = 'Error';
+                    diffCell.textContent = 'Error';
+                }
+                
+            } catch (error) {
+                console.error('Error calculating predicted rating:', error);
+                predictedCell.textContent = 'Error';
+                button.disabled = false;
+                button.textContent = 'Calculate Approx Rating';
+            }
+        }
+        
+        fetchRatings();
+    </script>
+</body>
+</html>
@@ -0,0 +1,18 @@
+{
+  "name": "pdga-ratings",
+  "version": "1.0.0",
+  "description": "PDGA rating scraper and display",
+  "main": "server.js",
+  "scripts": {
+    "start": "node server.js",
+    "dev": "nodemon server.js"
+  },
+  "dependencies": {
+    "express": "^4.18.2",
+    "puppeteer": "^21.0.0",
+    "fs": "^0.0.1-security"
+  },
+  "devDependencies": {
+    "nodemon": "^3.0.1"
+  }
+}
@@ -0,0 +1,8 @@
+60954
+178737
+251092
+221990
+176006
+201220
+288059
+242229
@@ -0,0 +1,340 @@
+const express = require('express');
+const puppeteer = require('puppeteer');
+const fs = require('fs');
+const path = require('path');
+
+const app = express();
+const PORT = 3000;
+
+app.use(express.static('public'));
+
+const cache = new Map();
+const CACHE_DURATION = 24 * 60 * 60 * 1000;
+
+async function scrapePDGARating(pdgaNumber) {
+  const cacheKey = `player-${pdgaNumber}`;
+  const cached = cache.get(cacheKey);
+  
+  if (cached && Date.now() - cached.timestamp < CACHE_DURATION) {
+    console.log(`Using cached data for PDGA ${pdgaNumber}`);
+    return cached.data;
+  }
+  
+  const browser = await puppeteer.launch({ headless: true });
+  const page = await browser.newPage();
+  
+  try {
+    const url = `https://www.pdga.com/player/${pdgaNumber}`;
+    await page.goto(url, { waitUntil: 'networkidle2' });
+    
+    const playerName = await page.$eval('h1', el => {
+      const text = el.innerText.trim();
+      return text.replace(/\s*#\d+$/, '');
+    });
+    
+    const ratingData = await page.evaluate(() => {
+      const elements = document.querySelectorAll('li');
+      for (const el of elements) {
+        const text = el.innerText || el.textContent;
+        if (text.includes('Current Rating:')) {
+          console.log('Found rating text:', text);
+          const ratingMatch = text.match(/Current Rating:\s*(\d+)/);
+          
+          // Try different patterns for rating change
+          const changePatterns = [
+            /\[(\+\d+)\]/,
+            /\[(\-\d+)\]/,
+            /(\+\d+)/,
+            /(\-\d+)/
+          ];
+          
+          let change = null;
+          for (const pattern of changePatterns) {
+            const match = text.match(pattern);
+            if (match) {
+              change = match[1];
+              break;
+            }
+          }
+          
+          return {
+            rating: ratingMatch ? ratingMatch[1] : null,
+            change: change
+          };
+        }
+      }
+      return { rating: null, change: null };
+    });
+    
+    await browser.close();
+    
+    const result = {
+      pdgaNumber,
+      name: playerName,
+      rating: ratingData.rating ? parseInt(ratingData.rating) : 0,
+      ratingChange: ratingData.change ? parseInt(ratingData.change) : null,
+      predictedRating: null
+    };
+    
+    cache.set(cacheKey, {
+      data: result,
+      timestamp: Date.now()
+    });
+    
+    return result;
+  } catch (error) {
+    console.error(`Error scraping PDGA ${pdgaNumber}:`, error);
+    await browser.close();
+    return {
+      pdgaNumber,
+      name: 'Error',
+      rating: 0,
+      ratingChange: null,
+      predictedRating: null
+    };
+  }
+}
+
+async function getPredictedRating(browser, pdgaNumber) {
+  try {
+    const roundRatings = await getPlayerCompetitionRatings(browser, pdgaNumber);
+    return calculatePredictedRating(roundRatings);
+  } catch (error) {
+    console.error(`Error getting predicted rating for ${pdgaNumber}:`, error);
+    return 0;
+  }
+}
+
+async function getPlayerCompetitionRatings(browser, pdgaNumber) {
+  const page = await browser.newPage();
+  let allRatings = [];
+  
+  try {
+    const url = `https://www.pdga.com/player/${pdgaNumber}`;
+    await page.goto(url, { waitUntil: 'networkidle2' });
+    
+    const tournamentUrls = await page.evaluate(() => {
+      const tables = document.querySelectorAll('table[id*="player-results"]');
+      const urls = [];
+      
+      tables.forEach(table => {
+        const rows = table.querySelectorAll('tbody tr');
+        rows.forEach(row => {
+          const dateCell = row.querySelector('.dates');
+          const tournamentCell = row.querySelector('.tournament a');
+          
+          if (dateCell && tournamentCell) {
+            const dateText = dateCell.innerText.trim();
+            const dateMatch = dateText.match(/\d{1,2}-[A-Za-z]{3}-\d{4}/);
+            
+            if (dateMatch) {
+              const dateStr = dateMatch[0];
+              const date = new Date(dateStr);
+              const twoYearsAgo = new Date();
+              twoYearsAgo.setFullYear(twoYearsAgo.getFullYear() - 2);
+              
+              if (date > twoYearsAgo) {
+                const href = tournamentCell.getAttribute('href');
+                if (href) {
+                  urls.push({
+                    url: `https://www.pdga.com${href}`,
+                    date: dateStr
+                  });
+                }
+              }
+            }
+          }
+        });
+      });
+      
+      return urls.slice(0, 15);
+    });
+    
+    console.log(`Found ${tournamentUrls.length} recent tournaments for PDGA ${pdgaNumber}`);
+    
+    for (const tournamentData of tournamentUrls) {
+      try {
+        await page.goto(tournamentData.url, { waitUntil: 'networkidle2' });
+        await page.waitForTimeout(2000);
+        
+        const roundRatings = await page.evaluate((pdgaNum) => {
+          const rows = document.querySelectorAll('tr');
+          
+          for (const row of rows) {
+            const cells = row.querySelectorAll('td');
+            const hasPlayerNumber = Array.from(cells).some(cell => 
+              cell.innerText && cell.innerText.includes(pdgaNum.toString())
+            );
+            
+            if (hasPlayerNumber) {
+              const roundRatingCells = row.querySelectorAll('td.round-rating');
+              const ratings = [];
+              
+              roundRatingCells.forEach(cell => {
+                const rating = parseInt(cell.innerText.trim());
+                if (!isNaN(rating) && rating > 0) {
+                  ratings.push(rating);
+                }
+              });
+              
+              return ratings;
+            }
+          }
+          
+          return [];
+        }, pdgaNumber);
+        
+        if (roundRatings.length > 0) {
+          const parsedDate = parseDate(tournamentData.date);
+          roundRatings.forEach(rating => {
+            allRatings.push({
+              rating,
+              date: parsedDate
+            });
+          });
+          console.log(`Found ${roundRatings.length} round ratings for ${tournamentData.url}`);
+        }
+        
+      } catch (error) {
+        console.error(`Error scraping tournament ${tournamentData.url}:`, error);
+      }
+    }
+    
+  } catch (error) {
+    console.error(`Error getting competition ratings for PDGA ${pdgaNumber}:`, error);
+  } finally {
+    await page.close();
+  }
+  
+  const oneYearAgo = new Date();
+  oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
+  
+  const recentRatings = allRatings.filter(r => r.date > oneYearAgo);
+  return recentRatings.length > 8 ? recentRatings : allRatings;
+}
+
+function parseDate(dateStr) {
+  const formats = [
+    /^(\d{1,2})-([A-Za-z]{3})-(\d{4})$/,
+    /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/
+  ];
+  
+  for (const format of formats) {
+    const match = dateStr.match(format);
+    if (match) {
+      if (format === formats[0]) {
+        const monthMap = {
+          'Jan': 0, 'Feb': 1, 'Mar': 2, 'Apr': 3, 'May': 4, 'Jun': 5,
+          'Jul': 6, 'Aug': 7, 'Sep': 8, 'Oct': 9, 'Nov': 10, 'Dec': 11
+        };
+        const day = parseInt(match[1]);
+        const month = monthMap[match[2]];
+        const year = parseInt(match[3]);
+        return new Date(year, month, day);
+      }
+    }
+  }
+  
+  return new Date(dateStr);
+}
+
+function calculatePredictedRating(roundRatings) {
+  if (!roundRatings || roundRatings.length === 0) return 0;
+  
+  const ratings = roundRatings
+    .sort((a, b) => b.date - a.date)
+    .map(r => r.rating);
+  
+  const weightedRatings = [];
+  const oneFourth = ratings.length > 9 ? Math.round(ratings.length * 0.25) : -1;
+  
+  for (let i = 0; i < ratings.length; i++) {
+    const rating = ratings[i];
+    weightedRatings.push(rating);
+    
+    if (i < oneFourth) {
+      weightedRatings.push(rating);
+    }
+  }
+  
+  const validRatings = weightedRatings.filter(r => r > 0);
+  if (validRatings.length === 0) return 0;
+  
+  const mean = validRatings.reduce((sum, r) => sum + r, 0) / validRatings.length;
+  const stdDev = calculateStandardDeviation(ratings);
+  const deviation = Math.min(stdDev * 2.5, 100);
+  
+  const filteredRatings = validRatings.filter(rating => Math.abs(mean - rating) < deviation);
+  
+  if (filteredRatings.length === 0) return Math.round(mean);
+  
+  return Math.round(filteredRatings.reduce((sum, r) => sum + r, 0) / filteredRatings.length);
+}
+
+function calculateStandardDeviation(ratings) {
+  if (!ratings || ratings.length === 0) return 0;
+  
+  const mean = ratings.reduce((sum, r) => sum + r, 0) / ratings.length;
+  const variance = ratings.reduce((sum, r) => sum + Math.pow(r - mean, 2), 0) / ratings.length;
+  
+  return Math.sqrt(variance);
+}
+
+async function getAllRatings() {
+  try {
+    const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8')
+      .split('\n')
+      .map(num => num.trim())
+      .filter(num => num);
+    
+    const ratings = [];
+    
+    for (const pdgaNumber of pdgaNumbers) {
+      console.log(`Scraping PDGA ${pdgaNumber}...`);
+      const playerData = await scrapePDGARating(pdgaNumber);
+      ratings.push(playerData);
+    }
+    
+    return ratings.sort((a, b) => b.rating - a.rating);
+  } catch (error) {
+    console.error('Error reading PDGA numbers:', error);
+    return [];
+  }
+}
+
+app.get('/', (req, res) => {
+  res.sendFile(path.join(__dirname, 'index.html'));
+});
+
+app.get('/api/ratings', async (req, res) => {
+  try {
+    const ratings = await getAllRatings();
+    res.json(ratings);
+  } catch (error) {
+    res.status(500).json({ error: 'Failed to fetch ratings' });
+  }
+});
+
+app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => {
+  try {
+    const { pdgaNumber } = req.params;
+    const browser = await puppeteer.launch({ headless: true });
+    
+    console.log(`Calculating predicted rating for PDGA ${pdgaNumber}...`);
+    const predictedRating = await getPredictedRating(browser, pdgaNumber);
+    
+    await browser.close();
+    
+    res.json({ 
+      pdgaNumber: parseInt(pdgaNumber),
+      predictedRating 
+    });
+  } catch (error) {
+    console.error('Error calculating predicted rating:', error);
+    res.status(500).json({ error: 'Failed to calculate predicted rating' });
+  }
+});
+
+app.listen(PORT, () => {
+  console.log(`PDGA Ratings app running on http://localhost:${PORT}`);
+});