const express = require('express'); const router = express.Router(); const { db } = require('../db'); const { getAllCoursesFromDB, getLayoutsForCourse, updateLayoutRating } = require('../models/course'); const { launchBrowser } = require('../scrapers/browser'); const { layoutEventCache, scrapeCourseDirectory, scrapeCourseLayouts, scrapeEventResults } = require('../scrapers/course-puppeteer'); const logger = require('../logger'); // Request locking to prevent concurrent scrapes of the same resource const activeScrapes = new Map(); router.get('/partials/course-table', async (req, res) => { try { const allCourses = await getAllCoursesFromDB(); const query = req.query.q || ''; let courses = allCourses; if (query) { const q = query.toLowerCase(); courses = allCourses.filter(c => c.name.toLowerCase().includes(q) || c.city.toLowerCase().includes(q) ); } res.render('../partials/course-table', { courses, query, total: allCourses.length }); } catch (error) { res.status(500).send('

Error loading courses. Please try again.

'); } }); router.get('/partials/course-layouts/:courseId', async (req, res) => { try { const { courseId } = req.params; const layouts = await getLayoutsForCourse(courseId); res.render('../partials/course-layouts', { layouts, courseId }); } catch (error) { logger.error('Error loading course layouts:', error.message); res.status(500).send('
Error loading layouts
'); } }); router.get('/api/courses', async (req, res) => { try { const courses = await getAllCoursesFromDB(); res.json(courses); } catch (error) { logger.error('Error fetching courses:', error.message); res.status(500).json({ error: 'Failed to fetch courses' }); } }); router.get('/api/layouts/:courseId', async (req, res) => { try { const { courseId } = req.params; const layouts = await getLayoutsForCourse(courseId); res.json(layouts); } catch (error) { logger.error('Error fetching layouts:', error.message); res.status(500).json({ error: 'Failed to fetch layouts' }); } }); router.post('/api/scrape-courses', async (req, res) => { req.setTimeout(600000); res.setTimeout(600000); let browser = null; try { logger.info('Starting course directory scraping...'); browser = await launchBrowser(); const courses = await scrapeCourseDirectory(browser); await browser.close(); browser = null; res.json({ success: true, coursesFound: courses.length, message: `Successfully scraped ${courses.length} courses` }); } catch (error) { logger.error({ err: error }, 'Error scraping courses'); if (browser) { try { await browser.close(); } catch (e) {} } res.status(500).json({ error: 'Failed to scrape courses' }); } }); router.post('/api/scrape-layouts/:courseId', async (req, res) => { req.setTimeout(600000); res.setTimeout(600000); const { courseId } = req.params; const lockKey = `layout-${courseId}`; if (activeScrapes.has(lockKey)) { logger.info(`⚠️ Scrape already in progress for course ${courseId}`); return res.status(409).json({ error: 'Scrape already in progress for this course', message: 'Please wait for the current scrape to complete' }); } let browser = null; const scrapePromise = (async () => { try { const course = await new Promise((resolve, reject) => { db.get('SELECT * FROM courses WHERE id = ?', [courseId], (err, row) => { if (err) reject(err); else resolve(row); }); }); if (!course) { throw new Error('Course not found'); } logger.info(`Starting layout scraping for course: ${course.name}`); browser = await launchBrowser(); const layouts = await scrapeCourseLayouts(browser, course.link, courseId); logger.info(`\n=== Starting event results scraping for ${course.name} ===`); const courseIdInt = parseInt(courseId); const layoutData = layoutEventCache.get(courseIdInt); if (!layoutData || layoutData.length === 0) { logger.info('No event data found in cache, skipping event results scraping'); await browser.close(); browser = null; return { success: true, layoutsFound: layouts.length, message: `Successfully scraped ${layouts.length} layouts for ${course.name} (no events found)` }; } const eventGroups = {}; layoutData.forEach(layout => { if (layout.eventUrl) { if (!eventGroups[layout.eventUrl]) { eventGroups[layout.eventUrl] = []; } eventGroups[layout.eventUrl].push(layout); } }); const allLayoutRatings = {}; let eventCount = 0; for (const eventUrl in eventGroups) { eventCount++; const eventLayouts = eventGroups[eventUrl]; const results = await scrapeEventResults(browser, eventUrl, eventLayouts); for (const layoutKey in results) { const layoutDataResult = results[layoutKey]; if (!allLayoutRatings[layoutKey]) { allLayoutRatings[layoutKey] = { name: layoutDataResult.name, par: layoutDataResult.par, allRatings: [], latestDate: layoutDataResult.eventDate }; } else { if (layoutDataResult.eventDate && (!allLayoutRatings[layoutKey].latestDate || new Date(layoutDataResult.eventDate) > new Date(allLayoutRatings[layoutKey].latestDate))) { allLayoutRatings[layoutKey].latestDate = layoutDataResult.eventDate; } } allLayoutRatings[layoutKey].allRatings.push(...layoutDataResult.ratings); } await new Promise(resolve => setTimeout(resolve, 2000)); } logger.info(`\n=== Calculating final ratings for all layouts ===`); let savedCount = 0; for (const layoutKey in allLayoutRatings) { const layoutDataResult = allLayoutRatings[layoutKey]; if (layoutDataResult.allRatings.length > 0) { const meanRating = Math.round( layoutDataResult.allRatings.reduce((sum, r) => sum + r, 0) / layoutDataResult.allRatings.length ); logger.debug(`Layout: ${layoutDataResult.name} (Par ${layoutDataResult.par})`); logger.debug(` Total ratings collected: ${layoutDataResult.allRatings.length}`); logger.debug(` Mean rating: ${meanRating}`); logger.debug(` Last played: ${layoutDataResult.latestDate || 'Unknown'}`); try { const changes = await updateLayoutRating( courseIdInt, layoutDataResult.name, layoutDataResult.par, meanRating, layoutDataResult.allRatings.length, layoutDataResult.latestDate ); if (changes > 0) { logger.info(` ✓ Updated in database`); savedCount++; } } catch (err) { logger.error(` Error updating layout ${layoutDataResult.name}:`, err.message); } } } await browser.close(); browser = null; return { success: true, layoutsFound: layouts.length, eventsProcessed: Object.keys(eventGroups).length, layoutsWithRatings: savedCount, message: `Successfully scraped ${layouts.length} layouts and processed ${Object.keys(eventGroups).length} events for ${course.name}` }; } catch (error) { logger.error({ err: error }, 'Error scraping layouts'); if (browser) { try { await browser.close(); } catch (e) {} } throw error; } })(); activeScrapes.set(lockKey, scrapePromise); try { const result = await scrapePromise; res.json(result); } catch (error) { res.status(500).json({ error: 'Failed to scrape layouts', message: error.message }); } finally { activeScrapes.delete(lockKey); logger.info(`✓ Released lock for course ${courseId}`); } }); router.post('/api/scrape-event-results/:courseId', async (req, res) => { req.setTimeout(600000); res.setTimeout(600000); let browser = null; try { const { courseId } = req.params; const courseIdInt = parseInt(courseId); const layoutData = layoutEventCache.get(courseIdInt); if (!layoutData || layoutData.length === 0) { return res.status(404).json({ error: 'No layout data found in cache. Please scrape layouts first.' }); } browser = await launchBrowser(); const eventGroups = {}; layoutData.forEach(layout => { if (layout.eventUrl) { if (!eventGroups[layout.eventUrl]) { eventGroups[layout.eventUrl] = []; } eventGroups[layout.eventUrl].push(layout); } }); const allLayoutRatings = {}; let eventCount = 0; for (const eventUrl in eventGroups) { eventCount++; const eventLayouts = eventGroups[eventUrl]; const results = await scrapeEventResults(browser, eventUrl, eventLayouts); for (const layoutKey in results) { const ld = results[layoutKey]; if (!allLayoutRatings[layoutKey]) { allLayoutRatings[layoutKey] = { name: ld.name, par: ld.par, allRatings: [], latestDate: ld.eventDate }; } else { if (ld.eventDate && (!allLayoutRatings[layoutKey].latestDate || new Date(ld.eventDate) > new Date(allLayoutRatings[layoutKey].latestDate))) { allLayoutRatings[layoutKey].latestDate = ld.eventDate; } } allLayoutRatings[layoutKey].allRatings.push(...ld.ratings); } await new Promise(resolve => setTimeout(resolve, 2000)); } await browser.close(); browser = null; logger.info(`\n=== Calculating final ratings for all layouts ===`); let savedCount = 0; for (const layoutKey in allLayoutRatings) { const ld = allLayoutRatings[layoutKey]; if (ld.allRatings.length > 0) { const meanRating = Math.round( ld.allRatings.reduce((sum, r) => sum + r, 0) / ld.allRatings.length ); logger.debug(`Layout: ${ld.name} (Par ${ld.par})`); logger.debug(` Total ratings collected: ${ld.allRatings.length}`); logger.debug(` Mean rating: ${meanRating}`); logger.debug(` Last played: ${ld.latestDate || 'Unknown'}`); try { const changes = await updateLayoutRating( courseIdInt, ld.name, ld.par, meanRating, ld.allRatings.length, ld.latestDate ); if (changes > 0) { logger.info(` ✓ Updated in database`); savedCount++; } } catch (err) { logger.error(` Error updating layout ${ld.name}:`, err.message); } } } res.json({ success: true, eventsProcessed: Object.keys(eventGroups).length, uniqueLayouts: Object.keys(allLayoutRatings).length, layoutsSaved: savedCount, message: `Processed ${Object.keys(eventGroups).length} events, updated ${savedCount} layouts` }); } catch (error) { logger.error({ err: error }, 'Error scraping event results'); if (browser) { try { await browser.close(); } catch (e) {} } res.status(500).json({ error: 'Failed to scrape event results' }); } }); module.exports = router;