Add request locking, extended timeouts, and inactive layouts accordion

- Add request locking system to prevent concurrent scrapes of same course
- Extend HTTP timeouts (10-30 min) for long-running scraping operations
- Add comprehensive logging for layout parsing to debug silent failures
- Implement accordion UI to hide layouts not played within 365 days
- Return 409 status when scrape already in progress for a course
- Add visual indicators for active vs inactive layouts

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

courses.html

@@ -159,6 +159,61 @@
             font-style: italic;
             padding: 20px;
         }
+
+        .inactive-layouts-accordion {
+            margin-top: 15px;
+            border: 1px solid #dee2e6;
+            border-radius: 4px;
+            background-color: #f8f9fa;
+        }
+
+        .accordion-header {
+            padding: 12px 15px;
+            cursor: pointer;
+            user-select: none;
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            background-color: #e9ecef;
+            border-radius: 4px;
+            transition: background-color 0.2s;
+        }
+
+        .accordion-header:hover {
+            background-color: #dee2e6;
+        }
+
+        .accordion-header-text {
+            font-weight: 600;
+            color: #6c757d;
+            font-size: 14px;
+        }
+
+        .accordion-icon {
+            transition: transform 0.3s;
+            color: #6c757d;
+        }
+
+        .accordion-icon.expanded {
+            transform: rotate(180deg);
+        }
+
+        .accordion-content {
+            max-height: 0;
+            overflow: hidden;
+            transition: max-height 0.3s ease-out;
+            padding: 0 10px;
+        }
+
+        .accordion-content.expanded {
+            max-height: 2000px;
+            padding: 10px;
+            transition: max-height 0.5s ease-in;
+        }
+
+        .layout-item.inactive {
+            opacity: 0.7;
+        }
         .refresh-icon {
             cursor: pointer;
             opacity: 0.6;
@@ -288,6 +343,19 @@
             tableDiv.innerHTML = tableHTML;
         }
 
+        function toggleAccordion(accordionId) {
+            const content = document.getElementById(accordionId);
+            const icon = document.getElementById(`${accordionId}-icon`);
+
+            if (content.classList.contains('expanded')) {
+                content.classList.remove('expanded');
+                icon.classList.remove('expanded');
+            } else {
+                content.classList.add('expanded');
+                icon.classList.add('expanded');
+            }
+        }
+
         async function toggleCourseLayouts(courseId) {
             const layoutsRow = document.getElementById(`layouts-${courseId}`);
             const layoutsContainer = document.getElementById(`layouts-container-${courseId}`);
@@ -312,24 +380,91 @@
                 const layouts = await response.json();
 
                 if (layouts.length > 0) {
-                    let layoutsHTML = '<h4 style="margin-top: 0;">Layouts:</h4>';
+                    // Calculate date threshold (365 days ago)
+                    const oneYearAgo = new Date();
+                    oneYearAgo.setDate(oneYearAgo.getDate() - 365);
+
+                    // Separate layouts into active and inactive
+                    const activeLayouts = [];
+                    const inactiveLayouts = [];
+
                     layouts.forEach(layout => {
-                        const ratingDisplay = layout.mean_rating ?
+                        if (layout.last_played) {
-                            `<span style="color: #28a745; font-weight: bold; margin-left: 10px;">Rating: ${layout.mean_rating}</span>` :
+                            const lastPlayedDate = new Date(layout.last_played);
-                            '';
+                            if (lastPlayedDate >= oneYearAgo) {
-                        const dateDisplay = layout.last_played ?
+                                activeLayouts.push(layout);
-                            `<span style="color: #6c757d; font-size: 12px; margin-left: 10px;">Last played: ${new Date(layout.last_played).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' })}</span>` :
+                            } else {
-                            '';
+                                inactiveLayouts.push(layout);
-                        layoutsHTML += `
+                            }
-                            <div class="layout-item">
+                        } else {
-                                <div>
+                            // Never played -> inactive
-                                    <span class="layout-name">${layout.name}</span>
+                            inactiveLayouts.push(layout);
-                                    ${dateDisplay}
+                        }
+                    });
+
+                    let layoutsHTML = '<h4 style="margin-top: 0;">Layouts:</h4>';
+
+                    // Render active layouts
+                    if (activeLayouts.length > 0) {
+                        activeLayouts.forEach(layout => {
+                            const ratingDisplay = layout.mean_rating ?
+                                `<span style="color: #28a745; font-weight: bold; margin-left: 10px;">Rating: ${layout.mean_rating}</span>` :
+                                '';
+                            const dateDisplay = layout.last_played ?
+                                `<span style="color: #6c757d; font-size: 12px; margin-left: 10px;">Last played: ${new Date(layout.last_played).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' })}</span>` :
+                                '';
+                            layoutsHTML += `
+                                <div class="layout-item">
+                                    <div>
+                                        <span class="layout-name">${layout.name}</span>
+                                        ${dateDisplay}
+                                    </div>
+                                    <span class="layout-par">Par ${layout.par}${ratingDisplay}</span>
+                                </div>
+                            `;
+                        });
+                    }
+
+                    // Render inactive layouts in accordion
+                    if (inactiveLayouts.length > 0) {
+                        const accordionId = `accordion-${courseId}`;
+                        layoutsHTML += `
+                            <div class="inactive-layouts-accordion">
+                                <div class="accordion-header" onclick="toggleAccordion('${accordionId}')">
+                                    <span class="accordion-header-text">Inactive Layouts (${inactiveLayouts.length}) - Not played in last year</span>
+                                    <span class="accordion-icon" id="${accordionId}-icon">▼</span>
+                                </div>
+                                <div class="accordion-content" id="${accordionId}">
+                        `;
+
+                        inactiveLayouts.forEach(layout => {
+                            const ratingDisplay = layout.mean_rating ?
+                                `<span style="color: #28a745; font-weight: bold; margin-left: 10px;">Rating: ${layout.mean_rating}</span>` :
+                                '';
+                            const dateDisplay = layout.last_played ?
+                                `<span style="color: #6c757d; font-size: 12px; margin-left: 10px;">Last played: ${new Date(layout.last_played).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' })}</span>` :
+                                `<span style="color: #dc3545; font-size: 12px; margin-left: 10px;">Never played</span>`;
+                            layoutsHTML += `
+                                <div class="layout-item inactive">
+                                    <div>
+                                        <span class="layout-name">${layout.name}</span>
+                                        ${dateDisplay}
+                                    </div>
+                                    <span class="layout-par">Par ${layout.par}${ratingDisplay}</span>
+                                </div>
+                            `;
+                        });
+
+                        layoutsHTML += `
                                 </div>
-                                <span class="layout-par">Par ${layout.par}${ratingDisplay}</span>
                             </div>
                         `;
-                    });
+                    }
+
+                    if (activeLayouts.length === 0 && inactiveLayouts.length === 0) {
+                        layoutsHTML = '<div class="no-layouts">No layouts found. Click the refresh icon to scrape layouts.</div>';
+                    }
+
                     layoutsContainer.innerHTML = layoutsHTML;
                     layoutsContainer.dataset.loaded = 'true';
                 } else {
@@ -379,7 +514,10 @@
 
                 const data = await response.json();
 
-                if (data.success) {
+                if (response.status === 409) {
+                    // Scrape already in progress
+                    alert(data.message || 'Scrape already in progress for this course. Please wait.');
+                } else if (data.success) {
                     // Reset the loaded state to force reload
                     const layoutsContainer = document.getElementById(`layouts-container-${courseId}`);
                     layoutsContainer.dataset.loaded = 'false';

server.js

@@ -17,6 +17,9 @@ const db = new sqlite3.Database(dbPath);
 // In-memory cache for layout-division-event mapping
 const layoutEventCache = new Map(); // key: courseId, value: array of {name, par, divisions, eventUrl}
 
+// Request locking to prevent concurrent scrapes of the same resource
+const activeScrapes = new Map(); // key: resourceId, value: Promise
+
 // Initialize database schema
 function initializeDatabase() {
   return new Promise((resolve, reject) => {
@@ -1624,6 +1627,7 @@ async function scrapeCourseDirectory(browser) {
 }
 
 async function scrapeCourseLayouts(browser, courseLink, courseId) {
+  console.log(`\n=== Scraping layouts from: ${courseLink} ===`);
   const page = await browser.newPage();
   const layouts = [];
 
@@ -1656,15 +1660,19 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) {
     });
 
     if (layoutsTabClicked) {
+      console.log('✓ Layouts tab found and clicked');
       await page.waitForTimeout(3000);
+    } else {
+      console.warn('⚠️ Layouts tab not found - may be on a single-layout course page');
     }
 
     // Extract layouts from the page
-    layouts.push(...await page.evaluate(() => {
+    const extractedLayouts = await page.evaluate(() => {
       const layoutData = [];
       const tournamentsDiv = document.querySelector('div.tournaments');
 
       if (!tournamentsDiv) {
+        console.warn('No div.tournaments found on page');
         return layoutData;
       }
 
@@ -1728,17 +1736,28 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) {
               divisions: divisions,
               eventUrl: fullEventUrl
             });
+          } else if (layoutName) {
+            // Log skipped layouts for debugging
+            console.warn(`⚠️ Skipped layout "${layoutName}" - Par: ${par}, Text sample: ${allText.substring(0, 200)}`);
           }
         });
       });
 
       return layoutData;
-    }));
+    });
+
+    if (extractedLayouts.length === 0) {
+      console.warn('⚠️ No layouts extracted from page');
+    }
+
+    layouts.push(...extractedLayouts);
 
     // Store all layout data in memory cache
     const courseIdInt = typeof courseId === 'string' ? parseInt(courseId) : courseId;
     layoutEventCache.set(courseIdInt, layouts);
 
+    console.log(`✓ Successfully parsed ${layouts.length} layouts from course page`);
+
     // Deduplicate for database: same name + same par = same layout
     const uniqueLayouts = [];
     const seen = new Set();
@@ -1751,12 +1770,17 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) {
       }
     }
 
+    if (uniqueLayouts.length < layouts.length) {
+      console.log(`ℹ️ Deduplicated to ${uniqueLayouts.length} unique layouts`);
+    }
+
     // Save layouts to database
     for (const layout of uniqueLayouts) {
       try {
         await saveLayoutToDB(courseId, layout);
+        console.log(`  ✓ Saved layout: ${layout.name} (Par ${layout.par})`);
       } catch (err) {
-        console.error(`Error saving layout ${layout.name}:`, err.message);
+        console.error(`  ✗ Error saving layout ${layout.name}:`, err.message);
       }
     }
 
@@ -2438,10 +2462,14 @@ app.post('/api/refresh-rating-history/:pdgaNumber', async (req, res) => {
 });
 
 app.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
+  // Increase timeout for tournament scraping
+  req.setTimeout(600000); // 10 minutes
+  res.setTimeout(600000);
+
   let browser = null;
   const { pdgaNumber } = req.params;
   try {
-    
+
     // Check when we last updated rounds for this player
     const lastRoundUpdate = await getLastRoundUpdateDate(pdgaNumber);
     const sinceDate = lastRoundUpdate ? new Date(lastRoundUpdate) : null;
@@ -2633,6 +2661,10 @@ app.get('/api/layouts/:courseId', async (req, res) => {
 });
 
 app.post('/api/scrape-courses', async (req, res) => {
+  // Increase timeout for course directory scraping
+  req.setTimeout(600000); // 10 minutes
+  res.setTimeout(600000);
+
   let browser = null;
   try {
     console.log('Starting course directory scraping...');
@@ -2674,23 +2706,40 @@ app.post('/api/scrape-courses', async (req, res) => {
 });
 
 app.post('/api/scrape-layouts/:courseId', async (req, res) => {
-  let browser = null;
+  // Increase timeout for this endpoint since scraping can take several minutes
-  try {
+  req.setTimeout(600000); // 10 minutes
-    const { courseId } = req.params;
+  res.setTimeout(600000);
 
-    // Get course from database
+  const { courseId } = req.params;
-    const course = await new Promise((resolve, reject) => {
+  const lockKey = `layout-${courseId}`;
-      db.get('SELECT * FROM courses WHERE id = ?', [courseId], (err, row) => {
+
-        if (err) reject(err);
+  // Check if there's already a scrape in progress for this course
-        else resolve(row);
+  if (activeScrapes.has(lockKey)) {
-      });
+    console.log(`⚠️ Scrape already in progress for course ${courseId}`);
+    return res.status(409).json({
+      error: 'Scrape already in progress for this course',
+      message: 'Please wait for the current scrape to complete'
     });
+  }
 
-    if (!course) {
+  let browser = null;
-      return res.status(404).json({ error: 'Course not found' });
-    }
 
-    console.log(`Starting layout scraping for course: ${course.name}`);
+  // Create a promise for this scrape operation
+  const scrapePromise = (async () => {
+    try {
+      // Get course from database
+      const course = await new Promise((resolve, reject) => {
+        db.get('SELECT * FROM courses WHERE id = ?', [courseId], (err, row) => {
+          if (err) reject(err);
+          else resolve(row);
+        });
+      });
+
+      if (!course) {
+        throw new Error('Course not found');
+      }
+
+      console.log(`Starting layout scraping for course: ${course.name}`);
 
     browser = await puppeteer.launch({
       headless: "new",
@@ -2810,30 +2859,53 @@ app.post('/api/scrape-layouts/:courseId', async (req, res) => {
       }
     }
 
-    await browser.close();
+      await browser.close();
-    browser = null;
+      browser = null;
 
-    res.json({
+      return {
-      success: true,
+        success: true,
-      layoutsFound: layouts.length,
+        layoutsFound: layouts.length,
-      eventsProcessed: Object.keys(eventGroups).length,
+        eventsProcessed: Object.keys(eventGroups).length,
-      layoutsWithRatings: savedCount,
+        layoutsWithRatings: savedCount,
-      message: `Successfully scraped ${layouts.length} layouts and processed ${Object.keys(eventGroups).length} events for ${course.name}`
+        message: `Successfully scraped ${layouts.length} layouts and processed ${Object.keys(eventGroups).length} events for ${course.name}`
-    });
+      };
-  } catch (error) {
+    } catch (error) {
-    console.error('Error scraping layouts:', error.message);
+      console.error('Error scraping layouts:', error.message);
-    if (browser) {
+      if (browser) {
-      try {
+        try {
-        await browser.close();
+          await browser.close();
-      } catch (closeError) {
+        } catch (closeError) {
-        console.error('Error closing browser:', closeError.message);
+          console.error('Error closing browser:', closeError.message);
+        }
       }
+      throw error;
     }
-    res.status(500).json({ error: 'Failed to scrape layouts' });
+  })();
+
+  // Store the promise in activeScrapes
+  activeScrapes.set(lockKey, scrapePromise);
+
+  try {
+    // Wait for the scrape to complete
+    const result = await scrapePromise;
+    res.json(result);
+  } catch (error) {
+    res.status(500).json({
+      error: 'Failed to scrape layouts',
+      message: error.message
+    });
+  } finally {
+    // Always remove from active scrapes when done
+    activeScrapes.delete(lockKey);
+    console.log(`✓ Released lock for course ${courseId}`);
   }
 });
 
 app.post('/api/scrape-event-results/:courseId', async (req, res) => {
+  // Increase timeout for scraping operations
+  req.setTimeout(600000); // 10 minutes
+  res.setTimeout(600000);
+
   let browser = null;
   try {
     const { courseId } = req.params;
@@ -2970,6 +3042,10 @@ app.post('/api/scrape-event-results/:courseId', async (req, res) => {
 });
 
 app.post('/api/scrape-all-layouts', async (req, res) => {
+  // Increase timeout for bulk scraping operations
+  req.setTimeout(1800000); // 30 minutes for bulk operations
+  res.setTimeout(1800000);
+
   let browser = null;
   try {
     console.log('Starting bulk layout scraping for all courses...');