From d567c4bca9cc5815d6aa259d0bf5ff38c3f1962c Mon Sep 17 00:00:00 2001 From: Samuel Enocsson Date: Thu, 19 Mar 2026 23:03:27 +0100 Subject: [PATCH] fix: upgrade Node 18 to 22 and fix Puppeteer compatibility - Switch from Alpine to Debian slim for correct Chromium architecture (fixes ARM/Apple Silicon support) - Upgrade Puppeteer 21 to 24, use system Chromium via PUPPETEER_EXECUTABLE_PATH - Replace removed page.waitForTimeout() with setTimeout - Set NODE_ENV=production in Dockerfile to prevent pino-pretty import - Improve error logging with Pino's { err: error } pattern - Add build: . to docker-compose for local development builds --- Dockerfile | 22 +- docker-compose.yml | 1 + package-lock.json | 412 ++++++++++++++++++++++--------- package.json | 2 +- src/routes/courses.js | 6 +- src/scrapers/browser.js | 2 +- src/scrapers/course-puppeteer.js | 14 +- src/scrapers/player-puppeteer.js | 6 +- 8 files changed, 322 insertions(+), 143 deletions(-) diff --git a/Dockerfile b/Dockerfile index b0605fc..0a404bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,16 @@ # Use official Node.js runtime as base image -FROM node:18-alpine +FROM node:22-slim -# Install Chromium and dependencies for Puppeteer -RUN apk add --no-cache \ +# Install Chromium from Debian repos (correct architecture for ARM/x86) +RUN apt-get update && apt-get install -y --no-install-recommends \ chromium \ - nss \ - freetype \ - freetype-dev \ - harfbuzz \ ca-certificates \ - ttf-freefont + fonts-freefont-ttf \ + && rm -rf /var/lib/apt/lists/* -# Tell Puppeteer to skip installing Chromium. We'll be using the installed package. +# Use system Chromium instead of Puppeteer's bundled download ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ - PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium # Set working directory WORKDIR /app @@ -31,10 +28,11 @@ COPY . . RUN mkdir -p data # Set database path -ENV DB_PATH=/app/data/ratings.db +ENV DB_PATH=/app/data/ratings.db \ + NODE_ENV=production # Expose port EXPOSE 3000 # Start the application -CMD ["npm", "start"] \ No newline at end of file +CMD ["npm", "start"] diff --git a/docker-compose.yml b/docker-compose.yml index 20991d0..73a8312 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,6 @@ services: pdga-ratings: + build: . image: ghcr.io/shcizo/pdga-rating:latest container_name: pdga-ratings ports: diff --git a/package-lock.json b/package-lock.json index 639d131..1b8fc9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "express": "^4.18.2", "fs": "^0.0.1-security", "pino": "^10.3.1", - "puppeteer": "^21.0.0", + "puppeteer": "^24.40.0", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", "sqlite3": "^5.1.7" @@ -81,29 +81,33 @@ "license": "MIT" }, "node_modules/@puppeteer/browsers": { - "version": "1.9.1", + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.0.tgz", + "integrity": "sha512-46BZJYJjc/WwmKjsvDFykHtXrtomsCIrwYQPOP7VfMJoZY2bsDF9oROBABR3paDjDcmkUye1Pb1BqdcdiipaWA==", "license": "Apache-2.0", "dependencies": { - "debug": "4.3.4", - "extract-zip": "2.0.1", - "progress": "2.0.3", - "proxy-agent": "6.3.1", - "tar-fs": "3.0.4", - "unbzip2-stream": "1.4.3", - "yargs": "17.7.2" + "debug": "^4.4.3", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.4", + "tar-fs": "^3.1.1", + "yargs": "^17.7.2" }, "bin": { "browsers": "lib/cjs/main-cli.js" }, "engines": { - "node": ">=16.3.0" + "node": ">=18" } }, "node_modules/@puppeteer/browsers/node_modules/debug": { - "version": "4.3.4", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -115,7 +119,9 @@ } }, "node_modules/@puppeteer/browsers/node_modules/ms": { - "version": "2.1.2", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/@tootallnate/once": { @@ -130,6 +136,8 @@ }, "node_modules/@tootallnate/quickjs-emscripten": { "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", "license": "MIT" }, "node_modules/@types/debug": { @@ -183,6 +191,8 @@ }, "node_modules/agent-base": { "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", "license": "MIT", "engines": { "node": ">= 14" @@ -288,6 +298,8 @@ }, "node_modules/ast-types": { "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", "license": "MIT", "dependencies": { "tslib": "^2.0.1" @@ -312,17 +324,109 @@ } }, "node_modules/b4a": { - "version": "1.6.7", - "license": "Apache-2.0" + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.0.tgz", + "integrity": "sha512-qRuSmNSkGQaHwNbM7J78Wwy+ghLEYF1zNrSeMxj4Kgw6y33O3mXcQ6Ie9fRvfU/YnxWkOchPXbaLb73TkIsfdg==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } }, "node_modules/balanced-match": { "version": "1.0.2", "license": "MIT" }, "node_modules/bare-events": { - "version": "2.6.1", + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", + "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", "license": "Apache-2.0", - "optional": true + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.5.6", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.6.tgz", + "integrity": "sha512-1QovqDrR80Pmt5HPAsMsXTCFcDYr+NSUKW6nd6WO5v0JBmnItc/irNRzm2KOQ5oZ69P37y+AMujNyNtG+1Rggw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.8.0.tgz", + "integrity": "sha512-Dc9/SlwfxkXIGYhvMQNUtKaXCaGkZYGcd1vuNUUADVqzu4/vQfvnMkYYOUnt2VwQ2AqKr/8qAVFRtwETljgeFg==", + "license": "Apache-2.0", + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.10.0.tgz", + "integrity": "sha512-DOPZF/DDcDruKDA43cOw6e9Quq5daua7ygcAwJE/pKJsRWhgSSemi7qVNGE5kyDIxIeN1533G/zfbvWX7Wcb9w==", + "license": "Apache-2.0", + "dependencies": { + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.0.tgz", + "integrity": "sha512-NSTU5WN+fy/L0DDenfE8SXQna4voXuW0FHM7wH8i3/q9khUSchfPbPezO4zSFMnDGIf9YE+mt/RWhZgNRKRIXA==", + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } }, "node_modules/base64-js": { "version": "1.5.1", @@ -343,7 +447,9 @@ "license": "MIT" }, "node_modules/basic-ftp": { - "version": "5.0.5", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.0.tgz", + "integrity": "sha512-VoMINM2rqJwJgfdHq6RiUudKt2BV+FY5ZFezP/ypmwayk68+NzzAQy4XXLlqsGD4MCzq3DrmNFD/uUmBJuGoXw==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -565,11 +671,13 @@ } }, "node_modules/chromium-bidi": { - "version": "0.5.8", + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-14.0.0.tgz", + "integrity": "sha512-9gYlLtS6tStdRWzrtXaTMnqcM4dudNegMXJxkR0I/CXObHalYeYcAMPrL19eroNZHtJ8DQmu1E+ZNOYu/IXMXw==", "license": "Apache-2.0", "dependencies": { - "mitt": "3.0.1", - "urlpattern-polyfill": "10.0.0" + "mitt": "^3.0.1", + "zod": "^3.24.1" }, "peerDependencies": { "devtools-protocol": "*" @@ -707,15 +815,10 @@ } } }, - "node_modules/cross-fetch": { - "version": "4.0.0", - "license": "MIT", - "dependencies": { - "node-fetch": "^2.6.12" - } - }, "node_modules/data-uri-to-buffer": { "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", "license": "MIT", "engines": { "node": ">= 14" @@ -773,6 +876,8 @@ }, "node_modules/degenerator": { "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", "license": "MIT", "dependencies": { "ast-types": "^0.13.4", @@ -815,7 +920,9 @@ } }, "node_modules/devtools-protocol": { - "version": "0.0.1232444", + "version": "0.0.1581282", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1581282.tgz", + "integrity": "sha512-nv7iKtNZQshSW2hKzYNr46nM/Cfh5SEvE2oV0/SEGgc9XupIY5ggf84Cz8eJIkBce7S3bmTAauFD6aysMpnqsQ==", "license": "BSD-3-Clause" }, "node_modules/dunder-proto": { @@ -948,6 +1055,8 @@ }, "node_modules/escodegen": { "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", "license": "BSD-2-Clause", "dependencies": { "esprima": "^4.0.1", @@ -967,6 +1076,8 @@ }, "node_modules/esprima": { "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", "license": "BSD-2-Clause", "bin": { "esparse": "bin/esparse.js", @@ -978,6 +1089,8 @@ }, "node_modules/estraverse": { "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "license": "BSD-2-Clause", "engines": { "node": ">=4.0" @@ -985,6 +1098,8 @@ }, "node_modules/esutils": { "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "license": "BSD-2-Clause", "engines": { "node": ">=0.10.0" @@ -997,6 +1112,15 @@ "node": ">= 0.6" } }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/expand-template": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", @@ -1096,6 +1220,8 @@ }, "node_modules/fast-fifo": { "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", "license": "MIT" }, "node_modules/fast-safe-stringify": { @@ -1347,6 +1473,8 @@ }, "node_modules/get-uri": { "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", "license": "MIT", "dependencies": { "basic-ftp": "^5.0.2", @@ -1358,7 +1486,9 @@ } }, "node_modules/get-uri/node_modules/debug": { - "version": "4.4.1", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -1374,6 +1504,8 @@ }, "node_modules/get-uri/node_modules/ms": { "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/github-from-package": { @@ -1495,6 +1627,8 @@ }, "node_modules/http-proxy-agent": { "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", "license": "MIT", "dependencies": { "agent-base": "^7.1.0", @@ -1505,7 +1639,9 @@ } }, "node_modules/http-proxy-agent/node_modules/debug": { - "version": "4.4.1", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -1521,10 +1657,14 @@ }, "node_modules/http-proxy-agent/node_modules/ms": { "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/https-proxy-agent": { "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", "license": "MIT", "dependencies": { "agent-base": "^7.1.2", @@ -1535,7 +1675,9 @@ } }, "node_modules/https-proxy-agent/node_modules/debug": { - "version": "4.4.1", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -1551,6 +1693,8 @@ }, "node_modules/https-proxy-agent/node_modules/ms": { "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/humanize-ms": { @@ -1855,6 +1999,8 @@ }, "node_modules/lru-cache": { "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", "license": "ISC", "engines": { "node": ">=12" @@ -2239,6 +2385,8 @@ }, "node_modules/netmask": { "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", "license": "MIT", "engines": { "node": ">= 0.4.0" @@ -2262,24 +2410,6 @@ "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", "license": "MIT" }, - "node_modules/node-fetch": { - "version": "2.7.0", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/node-gyp": { "version": "8.4.1", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz", @@ -2448,6 +2578,8 @@ }, "node_modules/pac-proxy-agent": { "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", "license": "MIT", "dependencies": { "@tootallnate/quickjs-emscripten": "^0.23.0", @@ -2464,7 +2596,9 @@ } }, "node_modules/pac-proxy-agent/node_modules/debug": { - "version": "4.4.1", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -2480,10 +2614,14 @@ }, "node_modules/pac-proxy-agent/node_modules/ms": { "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/pac-resolver": { "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", "license": "MIT", "dependencies": { "degenerator": "^5.0.0", @@ -2749,24 +2887,28 @@ } }, "node_modules/proxy-agent": { - "version": "6.3.1", + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", "license": "MIT", "dependencies": { - "agent-base": "^7.0.2", + "agent-base": "^7.1.2", "debug": "^4.3.4", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.2", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", "lru-cache": "^7.14.1", - "pac-proxy-agent": "^7.0.1", + "pac-proxy-agent": "^7.1.0", "proxy-from-env": "^1.1.0", - "socks-proxy-agent": "^8.0.2" + "socks-proxy-agent": "^8.0.5" }, "engines": { "node": ">= 14" } }, "node_modules/proxy-agent/node_modules/debug": { - "version": "4.4.1", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -2782,10 +2924,14 @@ }, "node_modules/proxy-agent/node_modules/ms": { "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/proxy-from-env": { "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", "license": "MIT" }, "node_modules/pstree.remy": { @@ -2802,41 +2948,51 @@ } }, "node_modules/puppeteer": { - "version": "21.11.0", + "version": "24.40.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.40.0.tgz", + "integrity": "sha512-IxQbDq93XHVVLWHrAkFP7F7iHvb9o0mgfsSIMlhHb+JM+JjM1V4v4MNSQfcRWJopx9dsNOr9adYv0U5fm9BJBQ==", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "1.9.1", - "cosmiconfig": "9.0.0", - "puppeteer-core": "21.11.0" + "@puppeteer/browsers": "2.13.0", + "chromium-bidi": "14.0.0", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1581282", + "puppeteer-core": "24.40.0", + "typed-query-selector": "^2.12.1" }, "bin": { - "puppeteer": "lib/esm/puppeteer/node/cli.js" + "puppeteer": "lib/cjs/puppeteer/node/cli.js" }, "engines": { - "node": ">=16.13.2" + "node": ">=18" } }, "node_modules/puppeteer-core": { - "version": "21.11.0", + "version": "24.40.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.40.0.tgz", + "integrity": "sha512-MWL3XbUCfVgGR0gRsidzT6oKJT2QydPLhMITU6HoVWiiv4gkb6gJi3pcdAa8q4HwjBTbqISOWVP4aJiiyUJvag==", "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "1.9.1", - "chromium-bidi": "0.5.8", - "cross-fetch": "4.0.0", - "debug": "4.3.4", - "devtools-protocol": "0.0.1232444", - "ws": "8.16.0" + "@puppeteer/browsers": "2.13.0", + "chromium-bidi": "14.0.0", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1581282", + "typed-query-selector": "^2.12.1", + "webdriver-bidi-protocol": "0.4.1", + "ws": "^8.19.0" }, "engines": { - "node": ">=16.13.2" + "node": ">=18" } }, "node_modules/puppeteer-core/node_modules/debug": { - "version": "4.3.4", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -2848,7 +3004,9 @@ } }, "node_modules/puppeteer-core/node_modules/ms": { - "version": "2.1.2", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/puppeteer-extra": { @@ -3279,7 +3437,9 @@ "license": "BSD-3-Clause" }, "node_modules/semver": { - "version": "7.7.2", + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -3530,6 +3690,8 @@ }, "node_modules/socks-proxy-agent": { "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", "license": "MIT", "dependencies": { "agent-base": "^7.1.2", @@ -3541,7 +3703,9 @@ } }, "node_modules/socks-proxy-agent/node_modules/debug": { - "version": "4.4.1", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -3557,6 +3721,8 @@ }, "node_modules/socks-proxy-agent/node_modules/ms": { "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, "node_modules/sonic-boom": { @@ -3570,6 +3736,8 @@ }, "node_modules/source-map": { "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "license": "BSD-3-Clause", "optional": true, "engines": { @@ -3630,14 +3798,14 @@ } }, "node_modules/streamx": { - "version": "2.22.1", + "version": "2.25.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.25.0.tgz", + "integrity": "sha512-0nQuG6jf1w+wddNEEXCF4nTg3LtufWINB5eFEN+5TNZW7KWJp6x87+JFL43vaAUPyCfH1wID+mNVyW6OHtFamg==", "license": "MIT", "dependencies": { + "events-universal": "^1.0.0", "fast-fifo": "^1.3.2", "text-decoder": "^1.1.0" - }, - "optionalDependencies": { - "bare-events": "^2.2.0" } }, "node_modules/string_decoder": { @@ -3709,19 +3877,27 @@ } }, "node_modules/tar-fs": { - "version": "3.0.4", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.2.tgz", + "integrity": "sha512-QGxxTxxyleAdyM3kpFs14ymbYmNFrfY+pHj7Z8FgtbZ7w2//VAgLMac7sT6nRpIHjppXO2AwwEOg0bPFVRcmXw==", "license": "MIT", "dependencies": { - "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" } }, "node_modules/tar-stream": { - "version": "3.1.7", + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.8.tgz", + "integrity": "sha512-U6QpVRyCGHva435KoNWy9PRoi2IFYCgtEhq9nmrPPpbRacPs9IH4aJ3gbrFC8dPcXvdSZ4XXfXT5Fshbp2MtlQ==", "license": "MIT", "dependencies": { "b4a": "^1.6.4", + "bare-fs": "^4.5.5", "fast-fifo": "^1.2.0", "streamx": "^2.15.0" } @@ -3735,8 +3911,19 @@ "node": ">=8" } }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, "node_modules/text-decoder": { - "version": "1.2.3", + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", "license": "Apache-2.0", "dependencies": { "b4a": "^1.6.4" @@ -3754,10 +3941,6 @@ "node": ">=20" } }, - "node_modules/through": { - "version": "2.3.8", - "license": "MIT" - }, "node_modules/to-regex-range": { "version": "5.0.1", "dev": true, @@ -3784,12 +3967,10 @@ "nodetouch": "bin/nodetouch.js" } }, - "node_modules/tr46": { - "version": "0.0.3", - "license": "MIT" - }, "node_modules/tslib": { "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, "node_modules/tunnel-agent": { @@ -3815,13 +3996,11 @@ "node": ">= 0.6" } }, - "node_modules/unbzip2-stream": { - "version": "1.4.3", - "license": "MIT", - "dependencies": { - "buffer": "^5.2.1", - "through": "^2.3.8" - } + "node_modules/typed-query-selector": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.1.tgz", + "integrity": "sha512-uzR+FzI8qrUEIu96oaeBJmd9E7CFEiQ3goA5qCVgc4s5llSubcfGHq9yUstZx/k4s9dXHVKsE35YWoFyvEqEHA==", + "license": "MIT" }, "node_modules/undefsafe": { "version": "2.0.5", @@ -3869,10 +4048,6 @@ "node": ">= 0.8" } }, - "node_modules/urlpattern-polyfill": { - "version": "10.0.0", - "license": "MIT" - }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -3893,17 +4068,11 @@ "node": ">= 0.8" } }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz", + "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", + "license": "Apache-2.0" }, "node_modules/which": { "version": "2.0.2", @@ -3951,7 +4120,9 @@ "license": "ISC" }, "node_modules/ws": { - "version": "8.16.0", + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", + "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -4012,6 +4183,15 @@ "buffer-crc32": "~0.2.3", "fd-slicer": "~1.1.0" } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index f0c1a63..b82b386 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "express": "^4.18.2", "fs": "^0.0.1-security", "pino": "^10.3.1", - "puppeteer": "^21.0.0", + "puppeteer": "^24.40.0", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", "sqlite3": "^5.1.7" diff --git a/src/routes/courses.js b/src/routes/courses.js index 62bb548..fa0db78 100644 --- a/src/routes/courses.js +++ b/src/routes/courses.js @@ -81,7 +81,7 @@ router.post('/api/scrape-courses', async (req, res) => { message: `Successfully scraped ${courses.length} courses` }); } catch (error) { - logger.error('Error scraping courses:', error.message); + logger.error({ err: error }, 'Error scraping courses'); if (browser) { try { await browser.close(); } catch (e) {} } @@ -230,7 +230,7 @@ router.post('/api/scrape-layouts/:courseId', async (req, res) => { message: `Successfully scraped ${layouts.length} layouts and processed ${Object.keys(eventGroups).length} events for ${course.name}` }; } catch (error) { - logger.error('Error scraping layouts:', error.message); + logger.error({ err: error }, 'Error scraping layouts'); if (browser) { try { await browser.close(); } catch (e) {} } @@ -361,7 +361,7 @@ router.post('/api/scrape-event-results/:courseId', async (req, res) => { message: `Processed ${Object.keys(eventGroups).length} events, updated ${savedCount} layouts` }); } catch (error) { - logger.error('Error scraping event results:', error.message); + logger.error({ err: error }, 'Error scraping event results'); if (browser) { try { await browser.close(); } catch (e) {} } diff --git a/src/scrapers/browser.js b/src/scrapers/browser.js index 3952263..e6fa79a 100644 --- a/src/scrapers/browser.js +++ b/src/scrapers/browser.js @@ -5,7 +5,7 @@ puppeteer.use(StealthPlugin()); async function launchBrowser() { return await puppeteer.launch({ - headless: 'new', + headless: true, args: [ '--no-sandbox', '--disable-setuid-sandbox', diff --git a/src/scrapers/course-puppeteer.js b/src/scrapers/course-puppeteer.js index dcdbe60..f146ebf 100644 --- a/src/scrapers/course-puppeteer.js +++ b/src/scrapers/course-puppeteer.js @@ -21,7 +21,7 @@ async function scrapeCourseDirectory(browser) { logger.info(`Scraping page ${pageNumber}...`); await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 }); - await page.waitForTimeout(1000); + await new Promise(r => setTimeout(r, 1000)); const courses = await page.evaluate(() => { const courseData = []; @@ -74,7 +74,7 @@ async function scrapeCourseDirectory(browser) { logger.info(`Total courses scraped: ${allCourses.length} across ${pageNumber} pages`); } catch (error) { - logger.error('Error scraping course directory: ' + error.message); + logger.error({ err: error }, 'Error scraping course directory'); } finally { await page.close(); } @@ -89,7 +89,7 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) { try { await page.goto(courseLink, { waitUntil: 'networkidle2', timeout: 45000 }); - await page.waitForTimeout(1000); + await new Promise(r => setTimeout(r, 1000)); const layoutsTabClicked = await page.evaluate(() => { const selectors = [ @@ -116,7 +116,7 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) { if (layoutsTabClicked) { logger.info('Layouts tab found and clicked'); - await page.waitForTimeout(3000); + await new Promise(r => setTimeout(r, 3000)); } else { logger.warn('Layouts tab not found - may be on a single-layout course page'); } @@ -226,7 +226,7 @@ async function scrapeCourseLayouts(browser, courseLink, courseId) { } } catch (error) { - logger.error('Error scraping course layouts: ' + error.message); + logger.error({ err: error }, 'Error scraping course layouts'); } finally { await page.close(); } @@ -240,7 +240,7 @@ async function scrapeEventResults(browser, eventUrl, layoutsWithDivisions) { try { await page.goto(eventUrl, { waitUntil: 'networkidle2', timeout: 45000 }); - await page.waitForTimeout(1000); + await new Promise(r => setTimeout(r, 1000)); const eventDateRaw = await page.evaluate(() => { const allText = document.body.textContent; @@ -333,7 +333,7 @@ async function scrapeEventResults(browser, eventUrl, layoutsWithDivisions) { } } catch (error) { - logger.error('Error scraping event results: ' + error.message); + logger.error({ err: error }, 'Error scraping event results'); } finally { await page.close(); } diff --git a/src/scrapers/player-puppeteer.js b/src/scrapers/player-puppeteer.js index 39ce289..2fcb39b 100644 --- a/src/scrapers/player-puppeteer.js +++ b/src/scrapers/player-puppeteer.js @@ -8,7 +8,7 @@ async function getOfficialRatingHistory(browser, pdgaNumber) { try { const url = `https://www.pdga.com/player/${pdgaNumber}/history`; await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 }); - await page.waitForTimeout(1000); + await new Promise(r => setTimeout(r, 1000)); ratingHistory = await page.evaluate(() => { const history = []; @@ -63,7 +63,7 @@ async function getPlayerTournamentDetails(browser, pdgaNumber) { try { const url = `https://www.pdga.com/player/${pdgaNumber}/details`; await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 }); - await page.waitForTimeout(1000); + await new Promise(r => setTimeout(r, 1000)); tournamentRounds = await page.evaluate(() => { const rounds = []; @@ -200,7 +200,7 @@ async function getNewTournamentRounds(browser, pdgaNumber, afterDate) { logger.info(`Scraping new tournament: ${tournamentData.name} (${tournamentData.date})`); await page.goto(tournamentData.url, { waitUntil: 'domcontentloaded', timeout: 30000 }); - await page.waitForTimeout(500); + await new Promise(r => setTimeout(r, 500)); const roundRatings = await page.evaluate((pdgaNum) => { const rows = document.querySelectorAll('tr');