shcizo/pdga-rating
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Add user self-registration and implement rate limiting for predictions

Browse Source 
Allow users to add themselves to the player database through a web form,
eliminating the need for manual pdga-numbers.txt updates. Implement 24-hour
rate limiting on prediction refreshes to prevent abuse while maintaining
reasonable update frequency.

Key changes:
- Add player self-registration with PDGA number lookup and confirmation
- Store predicted ratings in database for persistence across restarts
- Implement 24-hour rate limit on prediction refresh endpoint
- Make database the single source of truth (text file only for initial seed)
- Remove "Scrape All Layouts" bulk operation button
- Update "Load All" to refresh existing players instead of text file

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Samuel Enocsson
2025-10-11 18:20:03 +02:00
parent 4cd00e35aa
commit c88d092b36
3 changed files with 731 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server.js
+324 -166
View File
@@ -9,6 +9,7 @@ const app = express();
const PORT = 3000;
app.use(express.static('public'));
app.use(express.json());
// Initialize SQLite database
const dbPath = process.env.DB_PATH || './ratings.db';
@@ -43,16 +44,17 @@ function initializeDatabase() {
console.error('Error checking table schema:', err);
return;
}
// Check if column exists by querying table info
// Check if columns exist by querying table info
db.all("PRAGMA table_info(players)", (err, columns) => {
if (err) {
console.error('Error getting table info:', err);
return;
}
const hasLastRoundUpdate = columns.some(col => col.name === 'last_round_update');
const hasPredictedRating = columns.some(col => col.name === 'predicted_rating');
if (!hasLastRoundUpdate) {
console.log('Adding last_round_update column to players table...');
db.run(`
@@ -65,6 +67,19 @@ function initializeDatabase() {
}
});
}
if (!hasPredictedRating) {
console.log('Adding predicted_rating column to players table...');
db.run(`
ALTER TABLE players ADD COLUMN predicted_rating INTEGER DEFAULT NULL
`, (err) => {
if (err) {
console.error('Error adding predicted_rating column:', err.message);
} else {
console.log('Successfully added predicted_rating column');
}
});
}
});
});
@@ -140,57 +155,58 @@ function initializeDatabase() {
});
}
// Check and populate database from PDGA numbers file at startup
// Check and populate database from PDGA numbers file at startup (only if DB is empty)
async function checkAndPopulateDatabase() {
try {
console.log('=== Checking database population against PDGA numbers file ===');
// Check if database has any players
const playerCount = await new Promise((resolve, reject) => {
db.get('SELECT COUNT(*) as count FROM players', [], (err, row) => {
if (err) reject(err);
else resolve(row.count);
});
});
if (playerCount > 0) {
console.log(`✓ Database already has ${playerCount} players - skipping text file import`);
console.log('📝 Note: pdga-numbers.txt is only used when database is empty');
return;
}
console.log('=== Database is empty - populating from PDGA numbers file ===');
const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8')
.split('\n')
.map(num => num.trim())
.filter(num => num);
console.log(`Found ${pdgaNumbers.length} PDGA numbers in file`);
const missingPlayers = [];
// Check which players are missing from database
for (const pdgaNumber of pdgaNumbers) {
const player = await getPlayerFromDB(pdgaNumber);
if (!player) {
missingPlayers.push(pdgaNumber);
}
}
if (missingPlayers.length === 0) {
console.log('✓ All players from PDGA numbers file are already in database');
if (pdgaNumbers.length === 0) {
console.log('⚠ No PDGA numbers found in file');
return;
}
console.log(`Found ${missingPlayers.length} missing players: [${missingPlayers.join(', ')}]`);
console.log('=== Starting automatic population of missing players ===');
// Populate missing players
for (let i = 0; i < missingPlayers.length; i++) {
const pdgaNumber = missingPlayers[i];
console.log(`[${i + 1}/${missingPlayers.length}] Scraping missing player PDGA ${pdgaNumber}...`);
console.log('Populating database with players from file...');
for (let i = 0; i < pdgaNumbers.length; i++) {
const pdgaNumber = pdgaNumbers[i];
console.log(`[${i + 1}/${pdgaNumbers.length}] Adding PDGA ${pdgaNumber}...`);
try {
const playerData = await scrapePDGARating(pdgaNumber);
console.log(`✓ Added PDGA ${pdgaNumber}: ${playerData.name}`);
console.log(` ✓ Added ${playerData.name}`);
// Delay between requests to be respectful to PDGA
if (i < missingPlayers.length - 1) {
console.log('Waiting 2s before next request...');
if (i < pdgaNumbers.length - 1) {
await new Promise(resolve => setTimeout(resolve, 2000));
}
} catch (error) {
console.error(`✗ Failed to add PDGA ${pdgaNumber}: ${error.message}`);
console.error(` ✗ Failed to add PDGA ${pdgaNumber}:`, error.message);
}
}
console.log('=== Database population complete ===');
} catch (error) {
console.error('Error during database population check:', error.message);
}
@@ -514,8 +530,13 @@ async function getPlayerDataFromDB(pdgaNumber) {
const cachedPlayer = await getPlayerFromDB(pdgaNumber);
if (cachedPlayer) {
console.log(`Loading PDGA ${pdgaNumber} from DB (source of truth)`);
const predictedRating = await getPredictedRatingFromDB(pdgaNumber);
// Use stored predicted_rating if available, otherwise calculate it from round history
let predictedRating = cachedPlayer.predicted_rating;
if (!predictedRating || predictedRating === 0) {
predictedRating = await getPredictedRatingFromDB(pdgaNumber);
}
return {
pdgaNumber: cachedPlayer.pdga_number,
name: cachedPlayer.name,
@@ -621,15 +642,19 @@ async function getPredictedRatingFromDB(pdgaNumber) {
const roundHistory = await getRoundHistoryFromDB(pdgaNumber);
if (roundHistory.length > 0) {
console.log(`Using ${roundHistory.length} cached rounds for PDGA ${pdgaNumber} prediction`);
// Convert to the format expected by calculatePredictedRating
const roundRatings = roundHistory.map(round => ({
rating: round.rating,
date: new Date(round.date),
competition: round.competition_name || 'Unknown'
}));
const result = calculatePredictedRating(roundRatings);
// Save the calculated prediction to database
await savePredictedRatingToDB(pdgaNumber, result.rating);
return result.rating;
}
return 0;
@@ -639,6 +664,19 @@ async function getPredictedRatingFromDB(pdgaNumber) {
}
}
function savePredictedRatingToDB(pdgaNumber, predictedRating) {
return new Promise((resolve, reject) => {
db.run(
'UPDATE players SET predicted_rating = ? WHERE pdga_number = ?',
[predictedRating, pdgaNumber],
function(err) {
if (err) reject(err);
else resolve();
}
);
});
}
async function getOfficialRatingHistory(browser, pdgaNumber) {
const page = await browser.newPage();
let ratingHistory = [];
@@ -1913,18 +1951,27 @@ async function scrapeEventResults(browser, eventUrl, layoutsWithDivisions) {
async function getAllRatingsFromDB(progressCallback = null) {
try {
const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8')
.split('\n')
.map(num => num.trim())
.filter(num => num);
// Get all players from database instead of text file
const allPlayers = await new Promise((resolve, reject) => {
db.all(
'SELECT pdga_number, name, current_rating, rating_change FROM players ORDER BY pdga_number',
[],
(err, rows) => {
if (err) reject(err);
else resolve(rows || []);
}
);
});
console.log(`Loading ${allPlayers.length} players from database...`);
const ratings = [];
const total = pdgaNumbers.length;
for (let i = 0; i < pdgaNumbers.length; i++) {
const pdgaNumber = pdgaNumbers[i];
console.log(`Loading PDGA ${pdgaNumber} from database... (${i + 1}/${total})`);
const total = allPlayers.length;
for (let i = 0; i < allPlayers.length; i++) {
const player = allPlayers[i];
const pdgaNumber = player.pdga_number;
if (progressCallback) {
progressCallback({
current: i + 1,
@@ -1933,53 +1980,50 @@ async function getAllRatingsFromDB(progressCallback = null) {
status: 'loading'
});
}
try {
// Load from database only (source of truth)
// Load full player data from database
const playerData = await getPlayerDataFromDB(pdgaNumber);
if (playerData) {
ratings.push(playerData);
} else {
console.log(`PDGA ${pdgaNumber} not found in DB - skipping (page load)`);
// Skip players not in DB for page loads
}
if (progressCallback) {
progressCallback({
current: i + 1,
total,
pdgaNumber,
status: playerData ? 'completed' : 'skipped',
name: playerData ? playerData.name : 'Not in DB'
status: 'completed',
name: playerData ? playerData.name : player.name
});
}
} catch (error) {
console.error(`Failed to load PDGA ${pdgaNumber} from database:`, error.message);
const errorData = {
pdgaNumber: parseInt(pdgaNumber),
name: 'Database Error',
rating: null,
ratingChange: null,
name: player.name || 'Database Error',
rating: player.current_rating,
ratingChange: player.rating_change,
predictedRating: null
};
ratings.push(errorData);
if (progressCallback) {
progressCallback({
current: i + 1,
total,
pdgaNumber,
status: 'error',
name: 'Database Error'
name: player.name || 'Database Error'
});
}
}
}
return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0));
} catch (error) {
console.error('Error reading PDGA numbers:', error);
console.error('Error loading players from database:', error);
return [];
}
}
@@ -2040,12 +2084,12 @@ app.post('/api/populate-database', (req, res) => {
res.write(`data: ${JSON.stringify(progress)}\n\n`);
};
console.log('=== Starting database population from PDGA numbers file ===');
// Use the scraping function to populate database
getAllRatingsWithScraping(progressCallback).then(ratings => {
console.log(`=== Database population complete: ${ratings.length} players added ===`);
res.write(`data: ${JSON.stringify({ status: 'complete', ratings, message: `Successfully populated database with ${ratings.length} players` })}\n\n`);
console.log('=== Starting database population from database players ===');
// Populate database by refreshing all players in database
refreshAllPlayersInDB(progressCallback).then(ratings => {
console.log(`=== Database population complete: ${ratings.length} players refreshed ===`);
res.write(`data: ${JSON.stringify({ status: 'complete', ratings, message: `Successfully refreshed ${ratings.length} players` })}\n\n`);
res.end();
}).catch(error => {
console.error('Error populating database:', error);
@@ -2054,25 +2098,19 @@ app.post('/api/populate-database', (req, res) => {
});
});
// Simple endpoint to check if database needs population
// Simple endpoint to check database status
app.get('/api/database-status', async (req, res) => {
try {
const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8')
.split('\n')
.map(num => num.trim())
.filter(num => num);
let playersInDB = 0;
for (const pdgaNumber of pdgaNumbers) {
const player = await getPlayerFromDB(pdgaNumber);
if (player) playersInDB++;
}
const playerCount = await new Promise((resolve, reject) => {
db.get('SELECT COUNT(*) as count FROM players', [], (err, row) => {
if (err) reject(err);
else resolve(row.count);
});
});
res.json({
totalExpected: pdgaNumbers.length,
playersInDB: playersInDB,
needsPopulation: playersInDB === 0,
populationProgress: Math.round((playersInDB / pdgaNumbers.length) * 100)
playersInDB: playerCount,
needsPopulation: playerCount === 0
});
} catch (error) {
res.status(500).json({ error: 'Failed to check database status' });
@@ -2092,8 +2130,8 @@ app.get('/api/load-all-players', (req, res) => {
res.write(`data: ${JSON.stringify(progress)}\n\n`);
};
// Use the original scraping function for bulk loading
getAllRatingsWithScraping(progressCallback).then(ratings => {
// Refresh all players currently in database
refreshAllPlayersInDB(progressCallback).then(ratings => {
res.write(`data: ${JSON.stringify({ status: 'complete', ratings })}\n\n`);
res.end();
}).catch(error => {
@@ -2176,6 +2214,87 @@ async function getAllRatingsWithScraping(progressCallback = null) {
}
}
// Refresh all players currently in database
async function refreshAllPlayersInDB(progressCallback = null) {
try {
// Get all players from database
const allPlayers = await new Promise((resolve, reject) => {
db.all(
'SELECT pdga_number, name FROM players ORDER BY pdga_number',
[],
(err, rows) => {
if (err) reject(err);
else resolve(rows || []);
}
);
});
console.log(`Refreshing ${allPlayers.length} players from database...`);
const ratings = [];
const total = allPlayers.length;
for (let i = 0; i < allPlayers.length; i++) {
const player = allPlayers[i];
const pdgaNumber = player.pdga_number;
console.log(`Refreshing PDGA ${pdgaNumber}... (${i + 1}/${total})`);
if (progressCallback) {
progressCallback({
current: i + 1,
total,
pdgaNumber,
status: 'loading'
});
}
try {
const playerData = await scrapePDGARating(pdgaNumber);
ratings.push(playerData);
if (progressCallback) {
progressCallback({
current: i + 1,
total,
pdgaNumber,
status: 'completed',
name: playerData.name
});
}
// Delay between PDGA scraping requests to be respectful
await new Promise(resolve => setTimeout(resolve, 2000));
} catch (error) {
console.error(`Failed to refresh PDGA ${pdgaNumber}:`, error.message);
const errorData = {
pdgaNumber: parseInt(pdgaNumber),
name: player.name || 'Error',
rating: 0,
ratingChange: null,
predictedRating: null
};
ratings.push(errorData);
if (progressCallback) {
progressCallback({
current: i + 1,
total,
pdgaNumber,
status: 'error',
name: player.name || 'Error'
});
}
}
}
return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0));
} catch (error) {
console.error('Error refreshing all players:', error);
return [];
}
}
async function fetchRatingHistory(pdgaNumber) {
return new Promise((resolve, reject) => {
const options = {
@@ -2380,18 +2499,105 @@ app.post('/api/clear-cache', (req, res) => {
});
// Individual player refresh endpoints
// Search for a player (check if exists in DB and fetch from PDGA)
app.get('/api/search-player/:pdgaNumber', async (req, res) => {
try {
const { pdgaNumber } = req.params;
console.log(`Searching for player with PDGA number ${pdgaNumber}`);
// Check if player already exists in database
const existingPlayer = await getPlayerFromDB(pdgaNumber);
if (existingPlayer) {
return res.json({
alreadyExists: true,
player: {
pdgaNumber: existingPlayer.pdga_number,
name: existingPlayer.name,
rating: existingPlayer.current_rating,
ratingChange: existingPlayer.rating_change
}
});
}
// Fetch player data from PDGA
const html = await fetchPlayerDataHTTP(pdgaNumber);
const playerData = parsePlayerData(html, pdgaNumber);
// Check if player was found (name shouldn't be 'Unknown')
if (playerData.name === 'Unknown' || !playerData.name) {
return res.status(404).json({ error: 'Player not found' });
}
res.json({
alreadyExists: false,
player: playerData
});
} catch (error) {
console.error('Error searching for player:', error.message);
res.status(500).json({ error: 'Failed to search for player' });
}
});
// Add a new player to the database
app.post('/api/add-player', async (req, res) => {
try {
const { pdgaNumber } = req.body;
if (!pdgaNumber) {
return res.status(400).json({ error: 'PDGA number is required' });
}
console.log(`Adding player with PDGA number ${pdgaNumber}`);
// Check if player already exists
const existingPlayer = await getPlayerFromDB(pdgaNumber);
if (existingPlayer) {
return res.status(409).json({
error: 'Player already exists',
player: {
pdgaNumber: existingPlayer.pdga_number,
name: existingPlayer.name,
rating: existingPlayer.current_rating
}
});
}
// Fetch player data from PDGA
const html = await fetchPlayerDataHTTP(pdgaNumber);
const playerData = parsePlayerData(html, pdgaNumber);
// Verify player was found
if (playerData.name === 'Unknown' || !playerData.name) {
return res.status(404).json({ error: 'Player not found' });
}
// Save to database
await savePlayerToDB(playerData);
console.log(`Successfully added player: ${playerData.name} (#${pdgaNumber})`);
res.json({
success: true,
player: playerData
});
} catch (error) {
console.error('Error adding player:', error.message);
res.status(500).json({ error: 'Failed to add player' });
}
});
app.post('/api/refresh-player/:pdgaNumber', async (req, res) => {
try {
const { pdgaNumber } = req.params;
console.log(`Manually refreshing player data for PDGA ${pdgaNumber}`);
// Force refresh by bypassing cache
const html = await fetchPlayerDataHTTP(pdgaNumber);
const playerData = parsePlayerData(html, pdgaNumber);
// Save to database
await savePlayerToDB(playerData);
res.json({
success: true,
player: playerData
@@ -2473,10 +2679,25 @@ app.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
// Check when we last updated rounds for this player
const lastRoundUpdate = await getLastRoundUpdateDate(pdgaNumber);
const sinceDate = lastRoundUpdate ? new Date(lastRoundUpdate) : null;
// Rate limit: Only allow refresh once every 24 hours
if (sinceDate) {
const hoursSinceUpdate = (Date.now() - sinceDate.getTime()) / (1000 * 60 * 60);
if (hoursSinceUpdate < 24) {
const hoursRemaining = Math.ceil(24 - hoursSinceUpdate);
return res.status(429).json({
error: 'Rate limit exceeded',
message: `Prediction can only be refreshed once every 24 hours. Please try again in ${hoursRemaining} hour(s).`,
lastUpdate: sinceDate.toISOString(),
hoursRemaining: hoursRemaining
});
}
}
const isIncremental = !!sinceDate;
console.log(`${isIncremental ? 'Incrementally updating' : 'Fully refreshing'} round history for PDGA ${pdgaNumber}${sinceDate ? ` since ${sinceDate.toDateString()}` : ''}`);
try {
browser = await puppeteer.launch({
headless: "new",
@@ -2548,13 +2769,16 @@ app.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => {
date: new Date(round.date),
competition: round.competition_name
}));
const result = calculatePredictedRating(roundsForPrediction);
// Save the predicted rating to database for persistence
await savePredictedRatingToDB(pdgaNumber, result.rating);
// Count official vs new rounds
const officialCount = allRounds.filter(r => r.source === 'official').length;
const newCount = allRounds.filter(r => r.source === 'new').length;
res.json({
success: true,
predictedRating: result.rating,
@@ -3041,72 +3265,6 @@ app.post('/api/scrape-event-results/:courseId', async (req, res) => {
}
});
app.post('/api/scrape-all-layouts', async (req, res) => {
// Increase timeout for bulk scraping operations
req.setTimeout(1800000); // 30 minutes for bulk operations
res.setTimeout(1800000);
let browser = null;
try {
console.log('Starting bulk layout scraping for all courses...');
const courses = await getAllCoursesFromDB();
console.log(`Found ${courses.length} courses to scrape`);
browser = await puppeteer.launch({
headless: "new",
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-accelerated-2d-canvas',
'--no-first-run',
'--no-zygote',
'--disable-gpu'
]
});
let totalLayouts = 0;
for (let i = 0; i < courses.length; i++) {
const course = courses[i];
console.log(`[${i + 1}/${courses.length}] Scraping layouts for: ${course.name}`);
try {
const layouts = await scrapeCourseLayouts(browser, course.link, course.id);
totalLayouts += layouts.length;
// Delay between requests to be respectful
if (i < courses.length - 1) {
console.log('Waiting 2s before next request...');
await new Promise(resolve => setTimeout(resolve, 2000));
}
} catch (error) {
console.error(`Error scraping layouts for ${course.name}:`, error.message);
}
}
await browser.close();
browser = null;
res.json({
success: true,
coursesProcessed: courses.length,
totalLayouts: totalLayouts,
message: `Successfully scraped layouts for ${courses.length} courses (${totalLayouts} total layouts)`
});
} catch (error) {
console.error('Error scraping all layouts:', error.message);
if (browser) {
try {
await browser.close();
} catch (closeError) {
console.error('Error closing browser:', closeError.message);
}
}
res.status(500).json({ error: 'Failed to scrape all layouts' });
}
});
app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => {
let browser = null;
try {