From 4cd00e35aa89e433f492e7404245c45793e28f9e Mon Sep 17 00:00:00 2001 From: Samuel Enocsson Date: Wed, 1 Oct 2025 22:25:20 +0200 Subject: [PATCH] Add request locking, extended timeouts, and inactive layouts accordion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add request locking system to prevent concurrent scrapes of same course - Extend HTTP timeouts (10-30 min) for long-running scraping operations - Add comprehensive logging for layout parsing to debug silent failures - Implement accordion UI to hide layouts not played within 365 days - Return 409 status when scrape already in progress for a course - Add visual indicators for active vs inactive layouts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- courses.html | 168 ++++++++++++++++++++++++++++++++++++++++++++++----- server.js | 144 ++++++++++++++++++++++++++++++++----------- 2 files changed, 263 insertions(+), 49 deletions(-) diff --git a/courses.html b/courses.html index d28326d..6108615 100644 --- a/courses.html +++ b/courses.html @@ -159,6 +159,61 @@ font-style: italic; padding: 20px; } + + .inactive-layouts-accordion { + margin-top: 15px; + border: 1px solid #dee2e6; + border-radius: 4px; + background-color: #f8f9fa; + } + + .accordion-header { + padding: 12px 15px; + cursor: pointer; + user-select: none; + display: flex; + justify-content: space-between; + align-items: center; + background-color: #e9ecef; + border-radius: 4px; + transition: background-color 0.2s; + } + + .accordion-header:hover { + background-color: #dee2e6; + } + + .accordion-header-text { + font-weight: 600; + color: #6c757d; + font-size: 14px; + } + + .accordion-icon { + transition: transform 0.3s; + color: #6c757d; + } + + .accordion-icon.expanded { + transform: rotate(180deg); + } + + .accordion-content { + max-height: 0; + overflow: hidden; + transition: max-height 0.3s ease-out; + padding: 0 10px; + } + + .accordion-content.expanded { + max-height: 2000px; + padding: 10px; + transition: max-height 0.5s ease-in; + } + + .layout-item.inactive { + opacity: 0.7; + } .refresh-icon { cursor: pointer; opacity: 0.6; @@ -288,6 +343,19 @@ tableDiv.innerHTML = tableHTML; } + function toggleAccordion(accordionId) { + const content = document.getElementById(accordionId); + const icon = document.getElementById(`${accordionId}-icon`); + + if (content.classList.contains('expanded')) { + content.classList.remove('expanded'); + icon.classList.remove('expanded'); + } else { + content.classList.add('expanded'); + icon.classList.add('expanded'); + } + } + async function toggleCourseLayouts(courseId) { const layoutsRow = document.getElementById(`layouts-${courseId}`); const layoutsContainer = document.getElementById(`layouts-container-${courseId}`); @@ -312,24 +380,91 @@ const layouts = await response.json(); if (layouts.length > 0) { - let layoutsHTML = '

Layouts:

'; + // Calculate date threshold (365 days ago) + const oneYearAgo = new Date(); + oneYearAgo.setDate(oneYearAgo.getDate() - 365); + + // Separate layouts into active and inactive + const activeLayouts = []; + const inactiveLayouts = []; + layouts.forEach(layout => { - const ratingDisplay = layout.mean_rating ? - `Rating: ${layout.mean_rating}` : - ''; - const dateDisplay = layout.last_played ? - `Last played: ${new Date(layout.last_played).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' })}` : - ''; - layoutsHTML += ` -
-
- ${layout.name} - ${dateDisplay} + if (layout.last_played) { + const lastPlayedDate = new Date(layout.last_played); + if (lastPlayedDate >= oneYearAgo) { + activeLayouts.push(layout); + } else { + inactiveLayouts.push(layout); + } + } else { + // Never played -> inactive + inactiveLayouts.push(layout); + } + }); + + let layoutsHTML = '

Layouts:

'; + + // Render active layouts + if (activeLayouts.length > 0) { + activeLayouts.forEach(layout => { + const ratingDisplay = layout.mean_rating ? + `Rating: ${layout.mean_rating}` : + ''; + const dateDisplay = layout.last_played ? + `Last played: ${new Date(layout.last_played).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' })}` : + ''; + layoutsHTML += ` +
+
+ ${layout.name} + ${dateDisplay} +
+ Par ${layout.par}${ratingDisplay} +
+ `; + }); + } + + // Render inactive layouts in accordion + if (inactiveLayouts.length > 0) { + const accordionId = `accordion-${courseId}`; + layoutsHTML += ` +
+
+ Inactive Layouts (${inactiveLayouts.length}) - Not played in last year + â–ŧ +
+
+ `; + + inactiveLayouts.forEach(layout => { + const ratingDisplay = layout.mean_rating ? + `Rating: ${layout.mean_rating}` : + ''; + const dateDisplay = layout.last_played ? + `Last played: ${new Date(layout.last_played).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' })}` : + `Never played`; + layoutsHTML += ` +
+
+ ${layout.name} + ${dateDisplay} +
+ Par ${layout.par}${ratingDisplay} +
+ `; + }); + + layoutsHTML += `
- Par ${layout.par}${ratingDisplay}
`; - }); + } + + if (activeLayouts.length === 0 && inactiveLayouts.length === 0) { + layoutsHTML = '
No layouts found. Click the refresh icon to scrape layouts.
'; + } + layoutsContainer.innerHTML = layoutsHTML; layoutsContainer.dataset.loaded = 'true'; } else { @@ -379,7 +514,10 @@ const data = await response.json(); - if (data.success) { + if (response.status === 409) { + // Scrape already in progress + alert(data.message || 'Scrape already in progress for this course. Please wait.'); + } else if (data.success) { // Reset the loaded state to force reload const layoutsContainer = document.getElementById(`layouts-container-${courseId}`); layoutsContainer.dataset.loaded = 'false'; diff --git a/server.js b/server.js index 9a8a8a2..a508984 100644 --- a/server.js +++ b/server.js @@ -17,6 +17,9 @@ const db = new sqlite3.Database(dbPath); // In-memory cache for layout-division-event mapping const layoutEventCache = new Map(); // key: courseId, value: array of {name, par, divisions, eventUrl} +// Request locking to prevent concurrent scrapes of the same resource +const activeScrapes = new Map(); // key: resourceId, value: Promise + // Initialize database schema function initializeDatabase() { return new Promise((resolve, reject) => { @@ -1624,6 +1627,7 @@ async function scrapeCourseDirectory(browser) { } async function scrapeCourseLayouts(browser, courseLink, courseId) { + console.log(`\n=== Scraping layouts from: ${courseLink} ===`); const page = await browser.newPage(); const layouts = []; @@ -1656,15 +1660,19 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) { }); if (layoutsTabClicked) { + console.log('✓ Layouts tab found and clicked'); await page.waitForTimeout(3000); + } else { + console.warn('⚠ī¸ Layouts tab not found - may be on a single-layout course page'); } // Extract layouts from the page - layouts.push(...await page.evaluate(() => { + const extractedLayouts = await page.evaluate(() => { const layoutData = []; const tournamentsDiv = document.querySelector('div.tournaments'); if (!tournamentsDiv) { + console.warn('No div.tournaments found on page'); return layoutData; } @@ -1728,17 +1736,28 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) { divisions: divisions, eventUrl: fullEventUrl }); + } else if (layoutName) { + // Log skipped layouts for debugging + console.warn(`⚠ī¸ Skipped layout "${layoutName}" - Par: ${par}, Text sample: ${allText.substring(0, 200)}`); } }); }); return layoutData; - })); + }); + + if (extractedLayouts.length === 0) { + console.warn('⚠ī¸ No layouts extracted from page'); + } + + layouts.push(...extractedLayouts); // Store all layout data in memory cache const courseIdInt = typeof courseId === 'string' ? parseInt(courseId) : courseId; layoutEventCache.set(courseIdInt, layouts); + console.log(`✓ Successfully parsed ${layouts.length} layouts from course page`); + // Deduplicate for database: same name + same par = same layout const uniqueLayouts = []; const seen = new Set(); @@ -1751,12 +1770,17 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) { } } + if (uniqueLayouts.length < layouts.length) { + console.log(`ℹī¸ Deduplicated to ${uniqueLayouts.length} unique layouts`); + } + // Save layouts to database for (const layout of uniqueLayouts) { try { await saveLayoutToDB(courseId, layout); + console.log(` ✓ Saved layout: ${layout.name} (Par ${layout.par})`); } catch (err) { - console.error(`Error saving layout ${layout.name}:`, err.message); + console.error(` ✗ Error saving layout ${layout.name}:`, err.message); } } @@ -2438,10 +2462,14 @@ app.post('/api/refresh-rating-history/:pdgaNumber', async (req, res) => { }); app.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => { + // Increase timeout for tournament scraping + req.setTimeout(600000); // 10 minutes + res.setTimeout(600000); + let browser = null; const { pdgaNumber } = req.params; try { - + // Check when we last updated rounds for this player const lastRoundUpdate = await getLastRoundUpdateDate(pdgaNumber); const sinceDate = lastRoundUpdate ? new Date(lastRoundUpdate) : null; @@ -2633,6 +2661,10 @@ app.get('/api/layouts/:courseId', async (req, res) => { }); app.post('/api/scrape-courses', async (req, res) => { + // Increase timeout for course directory scraping + req.setTimeout(600000); // 10 minutes + res.setTimeout(600000); + let browser = null; try { console.log('Starting course directory scraping...'); @@ -2674,23 +2706,40 @@ app.post('/api/scrape-courses', async (req, res) => { }); app.post('/api/scrape-layouts/:courseId', async (req, res) => { - let browser = null; - try { - const { courseId } = req.params; + // Increase timeout for this endpoint since scraping can take several minutes + req.setTimeout(600000); // 10 minutes + res.setTimeout(600000); - // Get course from database - const course = await new Promise((resolve, reject) => { - db.get('SELECT * FROM courses WHERE id = ?', [courseId], (err, row) => { - if (err) reject(err); - else resolve(row); - }); + const { courseId } = req.params; + const lockKey = `layout-${courseId}`; + + // Check if there's already a scrape in progress for this course + if (activeScrapes.has(lockKey)) { + console.log(`⚠ī¸ Scrape already in progress for course ${courseId}`); + return res.status(409).json({ + error: 'Scrape already in progress for this course', + message: 'Please wait for the current scrape to complete' }); + } - if (!course) { - return res.status(404).json({ error: 'Course not found' }); - } + let browser = null; - console.log(`Starting layout scraping for course: ${course.name}`); + // Create a promise for this scrape operation + const scrapePromise = (async () => { + try { + // Get course from database + const course = await new Promise((resolve, reject) => { + db.get('SELECT * FROM courses WHERE id = ?', [courseId], (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + + if (!course) { + throw new Error('Course not found'); + } + + console.log(`Starting layout scraping for course: ${course.name}`); browser = await puppeteer.launch({ headless: "new", @@ -2810,30 +2859,53 @@ app.post('/api/scrape-layouts/:courseId', async (req, res) => { } } - await browser.close(); - browser = null; + await browser.close(); + browser = null; - res.json({ - success: true, - layoutsFound: layouts.length, - eventsProcessed: Object.keys(eventGroups).length, - layoutsWithRatings: savedCount, - message: `Successfully scraped ${layouts.length} layouts and processed ${Object.keys(eventGroups).length} events for ${course.name}` - }); - } catch (error) { - console.error('Error scraping layouts:', error.message); - if (browser) { - try { - await browser.close(); - } catch (closeError) { - console.error('Error closing browser:', closeError.message); + return { + success: true, + layoutsFound: layouts.length, + eventsProcessed: Object.keys(eventGroups).length, + layoutsWithRatings: savedCount, + message: `Successfully scraped ${layouts.length} layouts and processed ${Object.keys(eventGroups).length} events for ${course.name}` + }; + } catch (error) { + console.error('Error scraping layouts:', error.message); + if (browser) { + try { + await browser.close(); + } catch (closeError) { + console.error('Error closing browser:', closeError.message); + } } + throw error; } - res.status(500).json({ error: 'Failed to scrape layouts' }); + })(); + + // Store the promise in activeScrapes + activeScrapes.set(lockKey, scrapePromise); + + try { + // Wait for the scrape to complete + const result = await scrapePromise; + res.json(result); + } catch (error) { + res.status(500).json({ + error: 'Failed to scrape layouts', + message: error.message + }); + } finally { + // Always remove from active scrapes when done + activeScrapes.delete(lockKey); + console.log(`✓ Released lock for course ${courseId}`); } }); app.post('/api/scrape-event-results/:courseId', async (req, res) => { + // Increase timeout for scraping operations + req.setTimeout(600000); // 10 minutes + res.setTimeout(600000); + let browser = null; try { const { courseId } = req.params; @@ -2970,6 +3042,10 @@ app.post('/api/scrape-event-results/:courseId', async (req, res) => { }); app.post('/api/scrape-all-layouts', async (req, res) => { + // Increase timeout for bulk scraping operations + req.setTimeout(1800000); // 30 minutes for bulk operations + res.setTimeout(1800000); + let browser = null; try { console.log('Starting bulk layout scraping for all courses...');