fix: skip recent-events tournament when extracted date predates afterDate (#24)

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "pdga-ratings",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "pdga-ratings",
-      "version": "1.3.0",
+      "version": "1.4.0",
       "dependencies": {
         "ejs": "^4.0.1",
         "express": "^4.18.2",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pdga-ratings",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "PDGA rating scraper and display",
   "main": "server.js",
   "scripts": {

public/css/players.css

@@ -94,82 +94,6 @@
     box-shadow: var(--shadow-lg);
 }
 
-/* ── Debug Icon ───────────────────────────────── */
-
-.debug-icon:hover {
-    opacity: 1 !important;
-    color: var(--accent) !important;
-}
-
-/* ── Debug Modal ──────────────────────────────── */
-
-.debug-modal {
-    position: fixed;
-    top: 0;
-    left: 0;
-    width: 100%;
-    height: 100%;
-    background: rgba(15, 23, 42, 0.5);
-    backdrop-filter: blur(4px);
-    z-index: 10001;
-    display: none;
-    justify-content: center;
-    align-items: center;
-}
-
-.debug-content {
-    background: var(--surface-1);
-    border-radius: var(--radius-lg);
-    padding: 24px;
-    max-width: 640px;
-    width: 90%;
-    max-height: 80vh;
-    overflow-y: auto;
-    box-shadow: var(--shadow-overlay);
-    position: relative;
-}
-
-.debug-header {
-    font-weight: 700;
-    font-size: 16px;
-    margin-bottom: 16px;
-    color: var(--text-primary);
-    padding-bottom: 12px;
-    border-bottom: 1px solid var(--border);
-}
-
-.debug-log {
-    font-family: var(--font-mono);
-    background: var(--surface-2);
-    border: 1px solid var(--border);
-    border-radius: var(--radius-md);
-    padding: 16px;
-    font-size: 12px;
-    line-height: 1.6;
-    white-space: pre-line;
-    color: var(--text-primary);
-}
-
-.debug-close {
-    position: absolute;
-    top: 12px;
-    right: 16px;
-    font-size: 22px;
-    color: var(--text-muted);
-    cursor: pointer;
-    background: none;
-    border: none;
-    padding: 4px;
-    border-radius: var(--radius-sm);
-    transition: color var(--transition), background var(--transition);
-    line-height: 1;
-}
-
-.debug-close:hover {
-    color: var(--text-primary);
-    background: var(--surface-3);
-}
-
 /* ── Add Player Modal ─────────────────────────── */
 
 .modal {

public/js/players.js

@@ -1,4 +1,3 @@
-const cachedDebugInfo = {};
 let pendingPlayerData = null;
 let openPdgaNumber = null;
 
@@ -186,10 +185,6 @@ async function refreshRoundHistory(pdgaNumber) {
         }
 
         if (data.success) {
-            if (data.debugLog) {
-                cachedDebugInfo[pdgaNumber] = data.debugLog;
-            }
-
             const predictedCell = document.getElementById(`predicted-${pdgaNumber}`);
             if (predictedCell) {
                 const predictedValue = predictedCell.querySelector('.predicted-value');
@@ -234,43 +229,6 @@ async function refreshRatingHistory(pdgaNumber) {
     }
 }
 
-async function showDebugInfo(pdgaNumber) {
-    const modal = document.getElementById('debug-modal');
-    const header = document.getElementById('debug-header');
-    const log = document.getElementById('debug-log');
-
-    const playerNameElement = document.querySelector(`#row-${pdgaNumber} .player-name a`);
-    const playerName = playerNameElement ? playerNameElement.textContent : `PDGA #${pdgaNumber}`;
-
-    header.textContent = `Prediction Calculation Details - ${playerName}`;
-    log.textContent = 'Loading calculation details...';
-    modal.style.display = 'flex';
-
-    try {
-        if (cachedDebugInfo[pdgaNumber]) {
-            log.textContent = cachedDebugInfo[pdgaNumber].join('\n');
-            return;
-        }
-
-        const response = await fetch(`/api/refresh-round-history/${pdgaNumber}`, { method: 'POST' });
-        const data = await response.json();
-
-        if (data.success && data.debugLog) {
-            cachedDebugInfo[pdgaNumber] = data.debugLog;
-            log.textContent = data.debugLog.join('\n');
-        } else {
-            log.textContent = 'No debug information available. Try refreshing the prediction first.';
-        }
-    } catch (error) {
-        console.error('Error fetching debug info:', error);
-        log.textContent = 'Error loading debug information. Please try again.';
-    }
-}
-
-function closeDebugModal(event) {
-    document.getElementById('debug-modal').style.display = 'none';
-}
-
 async function searchAndAddPlayer(event) {
     if (event) event.preventDefault();
     const input = document.getElementById('pdga-number-input');
@@ -712,7 +670,6 @@ async function refreshHistoryThenCalculate(pdgaNumber) {
             return;
         }
 
-        if (data.debugLog) cachedDebugInfo[pdgaNumber] = data.debugLog;
         const predictedCell = document.getElementById('predicted-' + pdgaNumber);
         if (predictedCell) {
             const predictedValue = predictedCell.querySelector('.predicted-value');

src/db.js

@@ -34,6 +34,8 @@ function initializeDatabase() {
           const hasLastRoundUpdate = columns.some(col => col.name === 'last_round_update');
           const hasPredictedRating = columns.some(col => col.name === 'predicted_rating');
           const hasStdDev = columns.some(col => col.name === 'std_dev');
+          const hasExcludedRoundsCount = columns.some(col => col.name === 'excluded_rounds_count');
+          const hasCutoffRating = columns.some(col => col.name === 'cutoff_rating');
 
           if (!hasLastRoundUpdate) {
             logger.info('Adding last_round_update column to players table...');
@@ -58,6 +60,22 @@ function initializeDatabase() {
               else logger.info('Successfully added std_dev column');
             });
           }
+
+          if (!hasExcludedRoundsCount) {
+            logger.info('Adding excluded_rounds_count column to players table...');
+            db.run(`ALTER TABLE players ADD COLUMN excluded_rounds_count INTEGER DEFAULT NULL`, (err) => {
+              if (err) logger.error('Error adding excluded_rounds_count column:', err.message);
+              else logger.info('Successfully added excluded_rounds_count column');
+            });
+          }
+
+          if (!hasCutoffRating) {
+            logger.info('Adding cutoff_rating column to players table...');
+            db.run(`ALTER TABLE players ADD COLUMN cutoff_rating INTEGER DEFAULT NULL`, (err) => {
+              if (err) logger.error('Error adding cutoff_rating column:', err.message);
+              else logger.info('Successfully added cutoff_rating column');
+            });
+          }
         });
       });
 

src/models/player.js

@@ -172,11 +172,11 @@ function saveRoundHistoryToDB(pdgaNumber, roundData, isIncremental = false) {
   });
 }
 
-function savePredictedRatingToDB(pdgaNumber, predictedRating, stdDev = null) {
+function savePredictedRatingToDB(pdgaNumber, predictedRating, stdDev = null, excludedRoundsCount = null, cutoffRating = null) {
   return new Promise((resolve, reject) => {
     db.run(
-      'UPDATE players SET predicted_rating = ?, std_dev = ? WHERE pdga_number = ?',
-      [predictedRating, stdDev, pdgaNumber],
+      'UPDATE players SET predicted_rating = ?, std_dev = ?, excluded_rounds_count = ?, cutoff_rating = ? WHERE pdga_number = ?',
+      [predictedRating, stdDev, excludedRoundsCount, cutoffRating, pdgaNumber],
       function(err) {
         if (err) reject(err);
         else resolve();

src/routes/players.js

@@ -400,7 +400,7 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
 
     const result = calculatePredictedRating(roundsForPrediction);
 
-    await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev);
+    await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating);
 
     const officialCount = allRounds.filter(r => r.source === 'official').length;
     const newCount = allRounds.filter(r => r.source === 'new').length;
@@ -409,7 +409,8 @@ router.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
       success: true,
       predictedRating: result.rating,
       stdDev: result.stdDev,
-      debugLog: result.debugLog,
+      excludedRoundsCount: result.excludedRoundsCount,
+      cutoffRating: result.cutoffRating,
       totalRounds: roundsForPrediction.length,
       officialRounds: officialCount,
       newRounds: newCount,

src/scrapers/player-puppeteer.js

@@ -156,81 +156,154 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) {
     
     logger.info(`Looking for tournaments after ${afterDate.toDateString()}...`);
     
-    const newTournamentUrls = await page.evaluate((afterTimestamp) => {
+    const { urls: newTournamentUrls, counts } = await page.evaluate((afterTimestamp) => {
       const afterDate = new Date(afterTimestamp);
       const tables = document.querySelectorAll('table[id*="player-results"]');
       const urls = [];
-      
-      tables.forEach(table => {
-        const rows = table.querySelectorAll('tbody tr');
+      const seenUrls = new Set();
+      let table = 0;
+      let recentEvents = 0;
+      let recentEventsAnchorsSeen = 0;
+      let recentEventsSkippedDuplicates = 0;
+
+      tables.forEach(tbl => {
+        const rows = tbl.querySelectorAll('tbody tr');
         rows.forEach(row => {
           const dateCell = row.querySelector('.dates');
           const tournamentCell = row.querySelector('.tournament a');
-          
+
           if (dateCell && tournamentCell) {
             const dateText = dateCell.innerText.trim();
             const dateMatch = dateText.match(/\d{1,2}-[A-Za-z]{3}-\d{4}/);
-            
+
             if (dateMatch) {
               const dateStr = dateMatch[0];
               const date = new Date(dateStr);
-              
+
               if (date > afterDate) {
                 const href = tournamentCell.getAttribute('href');
                 if (href) {
-                  urls.push({
-                    url: `https://www.pdga.com${href}`,
-                    date: dateStr,
-                    name: tournamentCell.innerText.trim()
-                  });
+                  const absoluteUrl = new URL(href, location.origin).href;
+                  if (!seenUrls.has(absoluteUrl)) {
+                    seenUrls.add(absoluteUrl);
+                    urls.push({
+                      url: absoluteUrl,
+                      date: dateStr,
+                      name: tournamentCell.innerText.trim(),
+                      source: 'table'
+                    });
+                    table++;
+                  }
                 }
               }
             }
           }
         });
       });
-      
-      return urls;
+
+      const recentAnchors = document.querySelectorAll('.recent-events a[href*="/tour/event/"]');
+      recentAnchors.forEach(anchor => {
+        recentEventsAnchorsSeen++;
+        const href = anchor.getAttribute('href');
+        if (href) {
+          const absoluteUrl = new URL(href, location.origin).href;
+          if (seenUrls.has(absoluteUrl)) {
+            recentEventsSkippedDuplicates++;
+          } else {
+            seenUrls.add(absoluteUrl);
+            urls.push({
+              url: absoluteUrl,
+              date: null,
+              name: anchor.innerText.trim() || 'Recent event',
+              source: 'recent-events'
+            });
+            recentEvents++;
+          }
+        }
+      });
+
+      return { urls, counts: { table, recentEvents, recentEventsAnchorsSeen, recentEventsSkippedDuplicates } };
     }, afterDate.getTime());
-    
-    logger.info(`Found ${newTournamentUrls.length} new tournaments after ${afterDate.toDateString()}`);
+
+    logger.info({
+      pdgaNumber,
+      afterDate: afterDate.toISOString(),
+      tableMatches: counts.table,
+      recentEventsMatches: counts.recentEvents,
+      recentEventsAnchorsSeen: counts.recentEventsAnchorsSeen,
+      recentEventsSkippedDuplicates: counts.recentEventsSkippedDuplicates,
+      totalUrlsToScrape: newTournamentUrls.length
+    }, 'new tournament URL discovery completed');
     
     for (const tournamentData of newTournamentUrls) {
       try {
-        logger.info(`Scraping new tournament: ${tournamentData.name} (${tournamentData.date})`);
-        
+        if (tournamentData.source === 'recent-events') {
+          logger.debug({ pdgaNumber, url: tournamentData.url }, 'recent-events: scraping tournament');
+        } else {
+          logger.info(`Scraping new tournament: ${tournamentData.name} (${tournamentData.date})`);
+        }
+
         await page.goto(tournamentData.url, { waitUntil: 'domcontentloaded', timeout: 30000 });
         await new Promise(r => setTimeout(r, 500));
-        
+
+        let parsedDate;
+        if (tournamentData.date !== null) {
+          parsedDate = parseDate(tournamentData.date);
+        } else {
+          const eventDateStr = await page.evaluate(() => {
+            const body = document.body ? document.body.innerText : '';
+            const m = body.match(/\d{1,2}\s+to\s+\d{1,2}-[A-Za-z]{3}-\d{4}/)
+                   || body.match(/\d{1,2}-[A-Za-z]{3}-\d{4}/);
+            return m ? m[0] : null;
+          });
+
+          if (eventDateStr) {
+            parsedDate = parseDate(eventDateStr);
+            if (!(parsedDate > afterDate)) {
+              logger.warn({
+                pdgaNumber,
+                url: tournamentData.url,
+                eventDateStr,
+                parsedDate: parsedDate ? parsedDate.toISOString() : null,
+                afterDate: afterDate.toISOString()
+              }, 'recent-events: extracted event date is not newer than afterDate, likely captured a non-tournament date — skipping');
+              continue;
+            }
+            logger.debug({ pdgaNumber, url: tournamentData.url, eventDateStr }, 'recent-events: extracted date from event page');
+          } else {
+            logger.warn({ pdgaNumber, url: tournamentData.url }, 'recent-events: could not extract date from event page, skipping tournament');
+            continue;
+          }
+        }
+
         const roundRatings = await page.evaluate((pdgaNum) => {
           const rows = document.querySelectorAll('tr');
-          
+
           for (const row of rows) {
             const cells = row.querySelectorAll('td');
-            const hasPlayerNumber = Array.from(cells).some(cell => 
+            const hasPlayerNumber = Array.from(cells).some(cell =>
               cell.innerText && cell.innerText.includes(pdgaNum.toString())
             );
-            
+
             if (hasPlayerNumber) {
               const roundRatingCells = row.querySelectorAll('td.round-rating');
               const ratings = [];
-              
+
               roundRatingCells.forEach(cell => {
                 const rating = parseInt(cell.innerText.trim());
                 if (!isNaN(rating) && rating > 0) {
                   ratings.push(rating);
                 }
               });
-              
+
               return ratings;
             }
           }
-          
+
           return [];
         }, pdgaNumber);
-        
+
         if (roundRatings.length > 0) {
-          const parsedDate = parseDate(tournamentData.date);
           roundRatings.forEach(rating => {
             newRounds.push({
               rating,
@@ -238,10 +311,10 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) {
               competition: tournamentData.name
             });
           });
-          
+
           logger.info(`Found ${roundRatings.length} round ratings for ${tournamentData.name}`);
         }
-        
+
       } catch (error) {
         logger.error(`Error scraping tournament ${tournamentData.name}: ${error.message}`);
       }

src/services/player-service.js

@@ -40,10 +40,14 @@ async function getPlayerDataFromDB(pdgaNumber, { includeMonthlyHistory = true }
 
       let predictedRating = cachedPlayer.predicted_rating;
       let stdDev = cachedPlayer.std_dev;
+      let excludedRoundsCount = cachedPlayer.excluded_rounds_count;
+      let cutoffRating = cachedPlayer.cutoff_rating;
       if (!predictedRating || predictedRating === 0) {
         predictedRating = await getPredictedRatingFromDB(pdgaNumber);
         const updatedPlayer = await getPlayerFromDB(pdgaNumber);
         stdDev = updatedPlayer?.std_dev;
+        excludedRoundsCount = updatedPlayer?.excluded_rounds_count;
+        cutoffRating = updatedPlayer?.cutoff_rating;
       }
 
       const rating = cachedPlayer.current_rating;
@@ -65,6 +69,8 @@ async function getPlayerDataFromDB(pdgaNumber, { includeMonthlyHistory = true }
         ratingChange,
         predictedRating: resolvedPredicted,
         stdDev: resolvedStdDev,
+        excludedRoundsCount: (excludedRoundsCount != null && excludedRoundsCount >= 0) ? excludedRoundsCount : null,
+        cutoffRating: (cutoffRating != null && cutoffRating > 0) ? cutoffRating : null,
         lastMonthRating,
         // gap between next predicted update and current rating (null when either is missing)
         deltaPredicted: (resolvedPredicted != null && rating != null) ? resolvedPredicted - rating : null,
@@ -145,7 +151,7 @@ async function getPredictedRatingFromDB(pdgaNumber) {
 
       const result = calculatePredictedRating(roundRatings);
 
-      await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev);
+      await savePredictedRatingToDB(pdgaNumber, result.rating, result.stdDev, result.excludedRoundsCount, result.cutoffRating);
 
       return result.rating;
     }
@@ -229,6 +235,8 @@ async function getAllRatingsFromDB(progressCallback = null) {
           ratingChange: errorRatingChange,
           predictedRating: null,
           stdDev: null,
+          excludedRoundsCount: null,
+          cutoffRating: null,
           lastMonthRating: (errorRating != null && errorRatingChange != null) ? errorRating - errorRatingChange : null,
           deltaPredicted: null,
           monthlyHistory: [],

src/services/rating-calculator.js

@@ -85,7 +85,7 @@ function calculatePredictedRating(roundRatings) {
 
   if (!roundRatings || roundRatings.length === 0) {
     debugLog.push('❌ No rounds provided for prediction');
-    return { rating: 0, debugLog };
+    return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null };
   }
 
   debugLog.push(`📊 Starting with ${roundRatings.length} total rounds`);
@@ -100,7 +100,7 @@ function calculatePredictedRating(roundRatings) {
 
   if (allSortedRounds.length === 0) {
     debugLog.push('❌ No valid rounds after filtering for update date');
-    return { rating: 0, debugLog };
+    return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null };
   }
 
   debugLog.push(`📊 After update date filter: ${allSortedRounds.length} rounds`);
@@ -127,7 +127,7 @@ function calculatePredictedRating(roundRatings) {
 
   if (eligibleRounds.length === 0) {
     debugLog.push('❌ No eligible rounds found');
-    return { rating: 0, debugLog };
+    return { rating: 0, debugLog, excludedRoundsCount: null, cutoffRating: null };
   }
 
   debugLog.push(`📈 ELIGIBLE ROUNDS: ${eligibleRounds.length}`);
@@ -137,6 +137,8 @@ function calculatePredictedRating(roundRatings) {
 
   let workingRounds = [...eligibleRounds];
   let workingRatings = workingRounds.map(r => r.rating);
+  let excludedRoundsCount = 0;
+  let cutoffRating = null;
 
   if (workingRatings.length >= 7) {
     debugLog.push('🔍 OUTLIER EXCLUSION (≥7 rounds available):');
@@ -160,6 +162,9 @@ function calculatePredictedRating(roundRatings) {
     const stdDevOutliers = workingRatings.filter(rating => rating < stdDevCutoff);
     const hundredPointOutliers = workingRatings.filter(rating => rating < hundredPointCutoff && rating >= stdDevCutoff);
 
+    excludedRoundsCount = stdDevOutliers.length + hundredPointOutliers.length;
+    cutoffRating = Math.round(Math.max(stdDevCutoff, hundredPointCutoff));
+
     if (stdDevOutliers.length > 0) {
       debugLog.push(`   ❌ 2.5σ outliers removed: ${stdDevOutliers.length} rounds`);
       stdDevOutliers.forEach(rating => {
@@ -188,6 +193,8 @@ function calculatePredictedRating(roundRatings) {
       debugLog.push(`   ✅ Using ${filteredRatings.length} rounds after outlier removal`);
     } else {
       debugLog.push(`   ⚠️  Too few rounds after outlier removal (${filteredRatings.length}), keeping all rounds`);
+      excludedRoundsCount = 0;
+      cutoffRating = null;
     }
   } else {
     debugLog.push(`⏭️  OUTLIER EXCLUSION SKIPPED (only ${workingRatings.length} rounds, need ≥7)`);
@@ -228,7 +235,7 @@ function calculatePredictedRating(roundRatings) {
   debugLog.push(`   Final Rating: ${finalRating}`);
   debugLog.push('=== END PDGA CALCULATION ===');
 
-  return { rating: finalRating, stdDev: Math.round(stdDev), debugLog };
+  return { rating: finalRating, stdDev: Math.round(stdDev), debugLog, excludedRoundsCount, cutoffRating };
 }
 
 module.exports = { parseDate, getNextPDGAUpdateDate, calculatePredictedRating, calculateStandardDeviation };

views/pages/index.ejs

@@ -95,15 +95,6 @@
 `; %>
 
 <% var modals = `
-    <!-- Debug Modal -->
-    <div id="debug-modal" class="debug-modal" onclick="closeDebugModal(event)">
-        <div class="debug-content" onclick="event.stopPropagation()">
-            <button class="debug-close" onclick="closeDebugModal()">&times;</button>
-            <div class="debug-header" id="debug-header">Prediction Calculation Details</div>
-            <div class="debug-log" id="debug-log">Loading...</div>
-        </div>
-    </div>
-
     <!-- Add Player Confirmation Modal -->
     <div id="add-player-modal" class="modal" onclick="closeAddPlayerModal(event)">
         <div class="modal-content" onclick="event.stopPropagation()">

views/partials/player-history.ejs

@@ -36,8 +36,19 @@ const chartPdgaNumber = hasPlayer ? player.pdgaNumber : pdgaNumber;
         <dd><%= player.rating - player.stdDev %>–<%= player.rating + player.stdDev %></dd>
       </div>
       <% } %>
+      <% if (player.excludedRoundsCount != null && player.rating) { %>
+      <div>
+        <dt>Excluded rounds</dt>
+        <dd><%= player.excludedRoundsCount %></dd>
+      </div>
+      <% } %>
+      <% if (player.cutoffRating != null && player.rating) { %>
+      <div>
+        <dt>Cutoff rating</dt>
+        <dd><%= player.cutoffRating %></dd>
+      </div>
+      <% } %>
     </dl>
-    <button class="link-btn" onclick="showDebugInfo(<%= player.pdgaNumber %>)" style="margin-top: 4px;">View calculation details →</button>
   </div>
   <% } %>
 

views/partials/ratings-cards.ejs

@@ -124,6 +124,18 @@ function renderSparkline(values, opts) {
                     <dd><%= player.rating - player.stdDev %>–<%= player.rating + player.stdDev %></dd>
                 </div>
                 <% } %>
+                <% if (player.excludedRoundsCount != null && player.rating) { %>
+                <div>
+                    <dt>Excluded rounds</dt>
+                    <dd><%= player.excludedRoundsCount %></dd>
+                </div>
+                <% } %>
+                <% if (player.cutoffRating != null && player.rating) { %>
+                <div>
+                    <dt>Cutoff rating</dt>
+                    <dd><%= player.cutoffRating %></dd>
+                </div>
+                <% } %>
             </dl>
         </div>
     </div>