Refactor: split server.js monolith into modular architecture
Extract 3410-line server.js into 12 focused modules:
- src/db.js: database init and migrations
- src/models/{player,course}.js: DB helper functions
- src/scrapers/{browser,player-http,player-puppeteer,course-puppeteer}.js
- src/services/{player-service,rating-calculator}.js
- src/routes/{players,courses,pages}.js
Remove dead code: duplicate saveRatingHistoryToDB, legacy
getPlayerCompetitionRatings/getPredictedRating/getAllRatingsWithScraping,
unused getCourseFromDB/getLatestOfficialRoundDate/testPDGARateLimit,
legacy cache Map, and POST /api/predicted-rating route.
Consolidate 5 duplicated Puppeteer launch blocks into launchBrowser().
server.js is now 28 lines: imports, middleware, mount routers, bootstrap.
This commit is contained in:
Samuel Enocsson
@@ -0,0 +1,341 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { db } = require('../db');
|
||||
const { getAllCoursesFromDB, getLayoutsForCourse, updateLayoutRating } = require('../models/course');
|
||||
const { launchBrowser } = require('../scrapers/browser');
|
||||
const { layoutEventCache, scrapeCourseDirectory, scrapeCourseLayouts, scrapeEventResults } = require('../scrapers/course-puppeteer');
|
||||
|
||||
// Request locking to prevent concurrent scrapes of the same resource
|
||||
const activeScrapes = new Map();
|
||||
|
||||
router.get('/api/courses', async (req, res) => {
|
||||
try {
|
||||
const courses = await getAllCoursesFromDB();
|
||||
res.json(courses);
|
||||
} catch (error) {
|
||||
console.error('Error fetching courses:', error.message);
|
||||
res.status(500).json({ error: 'Failed to fetch courses' });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/api/layouts/:courseId', async (req, res) => {
|
||||
try {
|
||||
const { courseId } = req.params;
|
||||
const layouts = await getLayoutsForCourse(courseId);
|
||||
res.json(layouts);
|
||||
} catch (error) {
|
||||
console.error('Error fetching layouts:', error.message);
|
||||
res.status(500).json({ error: 'Failed to fetch layouts' });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/api/scrape-courses', async (req, res) => {
|
||||
req.setTimeout(600000);
|
||||
res.setTimeout(600000);
|
||||
|
||||
let browser = null;
|
||||
try {
|
||||
console.log('Starting course directory scraping...');
|
||||
|
||||
browser = await launchBrowser();
|
||||
|
||||
const courses = await scrapeCourseDirectory(browser);
|
||||
|
||||
await browser.close();
|
||||
browser = null;
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
coursesFound: courses.length,
|
||||
message: `Successfully scraped ${courses.length} courses`
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error scraping courses:', error.message);
|
||||
if (browser) {
|
||||
try { await browser.close(); } catch (e) {}
|
||||
}
|
||||
res.status(500).json({ error: 'Failed to scrape courses' });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/api/scrape-layouts/:courseId', async (req, res) => {
|
||||
req.setTimeout(600000);
|
||||
res.setTimeout(600000);
|
||||
|
||||
const { courseId } = req.params;
|
||||
const lockKey = `layout-${courseId}`;
|
||||
|
||||
if (activeScrapes.has(lockKey)) {
|
||||
console.log(`⚠️ Scrape already in progress for course ${courseId}`);
|
||||
return res.status(409).json({
|
||||
error: 'Scrape already in progress for this course',
|
||||
message: 'Please wait for the current scrape to complete'
|
||||
});
|
||||
}
|
||||
|
||||
let browser = null;
|
||||
|
||||
const scrapePromise = (async () => {
|
||||
try {
|
||||
const course = await new Promise((resolve, reject) => {
|
||||
db.get('SELECT * FROM courses WHERE id = ?', [courseId], (err, row) => {
|
||||
if (err) reject(err);
|
||||
else resolve(row);
|
||||
});
|
||||
});
|
||||
|
||||
if (!course) {
|
||||
throw new Error('Course not found');
|
||||
}
|
||||
|
||||
console.log(`Starting layout scraping for course: ${course.name}`);
|
||||
|
||||
browser = await launchBrowser();
|
||||
|
||||
const layouts = await scrapeCourseLayouts(browser, course.link, courseId);
|
||||
|
||||
console.log(`\n=== Starting event results scraping for ${course.name} ===`);
|
||||
|
||||
const courseIdInt = parseInt(courseId);
|
||||
const layoutData = layoutEventCache.get(courseIdInt);
|
||||
|
||||
if (!layoutData || layoutData.length === 0) {
|
||||
console.log('No event data found in cache, skipping event results scraping');
|
||||
await browser.close();
|
||||
browser = null;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
layoutsFound: layouts.length,
|
||||
message: `Successfully scraped ${layouts.length} layouts for ${course.name} (no events found)`
|
||||
};
|
||||
}
|
||||
|
||||
const eventGroups = {};
|
||||
layoutData.forEach(layout => {
|
||||
if (layout.eventUrl) {
|
||||
if (!eventGroups[layout.eventUrl]) {
|
||||
eventGroups[layout.eventUrl] = [];
|
||||
}
|
||||
eventGroups[layout.eventUrl].push(layout);
|
||||
}
|
||||
});
|
||||
|
||||
const allLayoutRatings = {};
|
||||
|
||||
let eventCount = 0;
|
||||
for (const eventUrl in eventGroups) {
|
||||
eventCount++;
|
||||
const eventLayouts = eventGroups[eventUrl];
|
||||
|
||||
const results = await scrapeEventResults(browser, eventUrl, eventLayouts);
|
||||
|
||||
for (const layoutKey in results) {
|
||||
const layoutDataResult = results[layoutKey];
|
||||
|
||||
if (!allLayoutRatings[layoutKey]) {
|
||||
allLayoutRatings[layoutKey] = {
|
||||
name: layoutDataResult.name,
|
||||
par: layoutDataResult.par,
|
||||
allRatings: [],
|
||||
latestDate: layoutDataResult.eventDate
|
||||
};
|
||||
} else {
|
||||
if (layoutDataResult.eventDate && (!allLayoutRatings[layoutKey].latestDate ||
|
||||
new Date(layoutDataResult.eventDate) > new Date(allLayoutRatings[layoutKey].latestDate))) {
|
||||
allLayoutRatings[layoutKey].latestDate = layoutDataResult.eventDate;
|
||||
}
|
||||
}
|
||||
|
||||
allLayoutRatings[layoutKey].allRatings.push(...layoutDataResult.ratings);
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
}
|
||||
|
||||
console.log(`\n=== Calculating final ratings for all layouts ===`);
|
||||
|
||||
let savedCount = 0;
|
||||
for (const layoutKey in allLayoutRatings) {
|
||||
const layoutDataResult = allLayoutRatings[layoutKey];
|
||||
|
||||
if (layoutDataResult.allRatings.length > 0) {
|
||||
const meanRating = Math.round(
|
||||
layoutDataResult.allRatings.reduce((sum, r) => sum + r, 0) / layoutDataResult.allRatings.length
|
||||
);
|
||||
|
||||
console.log(`Layout: ${layoutDataResult.name} (Par ${layoutDataResult.par})`);
|
||||
console.log(` Total ratings collected: ${layoutDataResult.allRatings.length}`);
|
||||
console.log(` Mean rating: ${meanRating}`);
|
||||
console.log(` Last played: ${layoutDataResult.latestDate || 'Unknown'}`);
|
||||
|
||||
try {
|
||||
const changes = await updateLayoutRating(
|
||||
courseIdInt,
|
||||
layoutDataResult.name,
|
||||
layoutDataResult.par,
|
||||
meanRating,
|
||||
layoutDataResult.allRatings.length,
|
||||
layoutDataResult.latestDate
|
||||
);
|
||||
if (changes > 0) {
|
||||
console.log(` ✓ Updated in database`);
|
||||
savedCount++;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(` Error updating layout ${layoutDataResult.name}:`, err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
browser = null;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
layoutsFound: layouts.length,
|
||||
eventsProcessed: Object.keys(eventGroups).length,
|
||||
layoutsWithRatings: savedCount,
|
||||
message: `Successfully scraped ${layouts.length} layouts and processed ${Object.keys(eventGroups).length} events for ${course.name}`
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error scraping layouts:', error.message);
|
||||
if (browser) {
|
||||
try { await browser.close(); } catch (e) {}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
})();
|
||||
|
||||
activeScrapes.set(lockKey, scrapePromise);
|
||||
|
||||
try {
|
||||
const result = await scrapePromise;
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
error: 'Failed to scrape layouts',
|
||||
message: error.message
|
||||
});
|
||||
} finally {
|
||||
activeScrapes.delete(lockKey);
|
||||
console.log(`✓ Released lock for course ${courseId}`);
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/api/scrape-event-results/:courseId', async (req, res) => {
|
||||
req.setTimeout(600000);
|
||||
res.setTimeout(600000);
|
||||
|
||||
let browser = null;
|
||||
try {
|
||||
const { courseId } = req.params;
|
||||
const courseIdInt = parseInt(courseId);
|
||||
|
||||
const layoutData = layoutEventCache.get(courseIdInt);
|
||||
|
||||
if (!layoutData || layoutData.length === 0) {
|
||||
return res.status(404).json({
|
||||
error: 'No layout data found in cache. Please scrape layouts first.'
|
||||
});
|
||||
}
|
||||
|
||||
browser = await launchBrowser();
|
||||
|
||||
const eventGroups = {};
|
||||
layoutData.forEach(layout => {
|
||||
if (layout.eventUrl) {
|
||||
if (!eventGroups[layout.eventUrl]) {
|
||||
eventGroups[layout.eventUrl] = [];
|
||||
}
|
||||
eventGroups[layout.eventUrl].push(layout);
|
||||
}
|
||||
});
|
||||
|
||||
const allLayoutRatings = {};
|
||||
|
||||
let eventCount = 0;
|
||||
for (const eventUrl in eventGroups) {
|
||||
eventCount++;
|
||||
const eventLayouts = eventGroups[eventUrl];
|
||||
|
||||
const results = await scrapeEventResults(browser, eventUrl, eventLayouts);
|
||||
|
||||
for (const layoutKey in results) {
|
||||
const ld = results[layoutKey];
|
||||
|
||||
if (!allLayoutRatings[layoutKey]) {
|
||||
allLayoutRatings[layoutKey] = {
|
||||
name: ld.name,
|
||||
par: ld.par,
|
||||
allRatings: [],
|
||||
latestDate: ld.eventDate
|
||||
};
|
||||
} else {
|
||||
if (ld.eventDate && (!allLayoutRatings[layoutKey].latestDate ||
|
||||
new Date(ld.eventDate) > new Date(allLayoutRatings[layoutKey].latestDate))) {
|
||||
allLayoutRatings[layoutKey].latestDate = ld.eventDate;
|
||||
}
|
||||
}
|
||||
|
||||
allLayoutRatings[layoutKey].allRatings.push(...ld.ratings);
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
browser = null;
|
||||
|
||||
console.log(`\n=== Calculating final ratings for all layouts ===`);
|
||||
|
||||
let savedCount = 0;
|
||||
for (const layoutKey in allLayoutRatings) {
|
||||
const ld = allLayoutRatings[layoutKey];
|
||||
|
||||
if (ld.allRatings.length > 0) {
|
||||
const meanRating = Math.round(
|
||||
ld.allRatings.reduce((sum, r) => sum + r, 0) / ld.allRatings.length
|
||||
);
|
||||
|
||||
console.log(`Layout: ${ld.name} (Par ${ld.par})`);
|
||||
console.log(` Total ratings collected: ${ld.allRatings.length}`);
|
||||
console.log(` Mean rating: ${meanRating}`);
|
||||
console.log(` Last played: ${ld.latestDate || 'Unknown'}`);
|
||||
|
||||
try {
|
||||
const changes = await updateLayoutRating(
|
||||
courseIdInt,
|
||||
ld.name,
|
||||
ld.par,
|
||||
meanRating,
|
||||
ld.allRatings.length,
|
||||
ld.latestDate
|
||||
);
|
||||
if (changes > 0) {
|
||||
console.log(` ✓ Updated in database`);
|
||||
savedCount++;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(` Error updating layout ${ld.name}:`, err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
eventsProcessed: Object.keys(eventGroups).length,
|
||||
uniqueLayouts: Object.keys(allLayoutRatings).length,
|
||||
layoutsSaved: savedCount,
|
||||
message: `Processed ${Object.keys(eventGroups).length} events, updated ${savedCount} layouts`
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error scraping event results:', error.message);
|
||||
if (browser) {
|
||||
try { await browser.close(); } catch (e) {}
|
||||
}
|
||||
res.status(500).json({ error: 'Failed to scrape event results' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
Reference in New Issue
Block a user