Connection error. Please refresh the page.
'; + eventSource.close(); + button.textContent = originalText; + button.style.pointerEvents = 'auto'; + }; + } catch (error) { + console.error('Error loading all players:', error); + button.textContent = originalText; + button.style.pointerEvents = 'auto'; + } + } + fetchRatingsWithProgress(); diff --git a/package-lock.json b/package-lock.json index 9311be7..9d9d098 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "dependencies": { "express": "^4.18.2", "fs": "^0.0.1-security", - "puppeteer": "^21.0.0" + "puppeteer": "^21.0.0", + "sqlite3": "^5.1.7" }, "devDependencies": { "nodemon": "^3.0.1" @@ -35,6 +36,39 @@ "node": ">=6.9.0" } }, + "node_modules/@gar/promisify": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", + "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", + "license": "MIT", + "optional": true + }, + "node_modules/@npmcli/fs": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz", + "integrity": "sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "@gar/promisify": "^1.0.1", + "semver": "^7.3.5" + } + }, + "node_modules/@npmcli/move-file": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.1.2.tgz", + "integrity": "sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==", + "deprecated": "This functionality has been moved to @npmcli/fs", + "license": "MIT", + "optional": true, + "dependencies": { + "mkdirp": "^1.0.4", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/@puppeteer/browsers": { "version": "1.9.1", "license": "Apache-2.0", @@ -73,6 +107,16 @@ "version": "2.1.2", "license": "MIT" }, + "node_modules/@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/@tootallnate/quickjs-emscripten": { "version": "0.23.0", "license": "MIT" @@ -93,6 +137,13 @@ "@types/node": "*" } }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "license": "ISC", + "optional": true + }, "node_modules/accepts": { "version": "1.3.8", "license": "MIT", @@ -111,6 +162,33 @@ "node": ">= 14" } }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "license": "MIT", + "optional": true, + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/ansi-regex": { "version": "5.0.1", "license": "MIT", @@ -143,6 +221,28 @@ "node": ">= 8" } }, + "node_modules/aproba": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.1.0.tgz", + "integrity": "sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew==", + "license": "ISC", + "optional": true + }, + "node_modules/are-we-there-yet": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz", + "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "optional": true, + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/argparse": { "version": "2.0.1", "license": "Python-2.0" @@ -167,7 +267,7 @@ }, "node_modules/balanced-match": { "version": "1.0.2", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/bare-events": { @@ -211,6 +311,26 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/body-parser": { "version": "1.20.3", "license": "MIT", @@ -235,7 +355,7 @@ }, "node_modules/brace-expansion": { "version": "1.1.12", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -289,6 +409,49 @@ "node": ">= 0.8" } }, + "node_modules/cacache": { + "version": "15.3.0", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-15.3.0.tgz", + "integrity": "sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "@npmcli/fs": "^1.0.0", + "@npmcli/move-file": "^1.0.1", + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "infer-owner": "^1.0.4", + "lru-cache": "^6.0.0", + "minipass": "^3.1.1", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^1.0.3", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.0.2", + "unique-filename": "^1.1.1" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/cacache/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "license": "ISC", + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/call-bind-apply-helpers": { "version": "1.0.2", "license": "MIT", @@ -344,6 +507,15 @@ "fsevents": "~2.3.2" } }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, "node_modules/chromium-bidi": { "version": "0.5.8", "license": "Apache-2.0", @@ -355,6 +527,16 @@ "devtools-protocol": "*" } }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=6" + } + }, "node_modules/cliui": { "version": "8.0.1", "license": "ISC", @@ -381,11 +563,28 @@ "version": "1.1.4", "license": "MIT" }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "license": "ISC", + "optional": true, + "bin": { + "color-support": "bin.js" + } + }, "node_modules/concat-map": { "version": "0.0.1", - "dev": true, + "devOptional": true, "license": "MIT" }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "license": "ISC", + "optional": true + }, "node_modules/content-disposition": { "version": "0.5.4", "license": "MIT", @@ -459,6 +658,30 @@ "ms": "2.0.0" } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/degenerator": { "version": "5.0.1", "license": "MIT", @@ -471,6 +694,13 @@ "node": ">= 14" } }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "license": "MIT", + "optional": true + }, "node_modules/depd": { "version": "2.0.0", "license": "MIT", @@ -486,6 +716,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", + "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/devtools-protocol": { "version": "0.0.1232444", "license": "BSD-3-Clause" @@ -517,6 +756,29 @@ "node": ">= 0.8" } }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.5", "license": "MIT", @@ -531,6 +793,13 @@ "node": ">=6" } }, + "node_modules/err-code": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", + "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", + "license": "MIT", + "optional": true + }, "node_modules/error-ex": { "version": "1.3.2", "license": "MIT", @@ -624,6 +893,15 @@ "node": ">= 0.6" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/express": { "version": "4.21.2", "license": "MIT", @@ -716,6 +994,12 @@ "pend": "~1.2.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, "node_modules/fill-range": { "version": "7.1.1", "dev": true, @@ -761,6 +1045,31 @@ "version": "0.0.1-security", "license": "ISC" }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC", + "optional": true + }, "node_modules/fsevents": { "version": "2.3.3", "dev": true, @@ -780,6 +1089,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gauge": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz", + "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "optional": true, + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.3", + "console-control-strings": "^1.1.0", + "has-unicode": "^2.0.1", + "signal-exit": "^3.0.7", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.5" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "license": "ISC", @@ -864,6 +1194,34 @@ "version": "2.1.3", "license": "MIT" }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "optional": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "5.1.2", "dev": true, @@ -885,6 +1243,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC", + "optional": true + }, "node_modules/has-flag": { "version": "3.0.0", "dev": true, @@ -903,6 +1268,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "license": "ISC", + "optional": true + }, "node_modules/hasown": { "version": "2.0.2", "license": "MIT", @@ -913,6 +1285,13 @@ "node": ">= 0.4" } }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause", + "optional": true + }, "node_modules/http-errors": { "version": "2.0.0", "license": "MIT", @@ -987,6 +1366,16 @@ "version": "2.1.3", "license": "MIT" }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "license": "MIT", @@ -1034,10 +1423,55 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/infer-owner": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", + "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==", + "license": "ISC", + "optional": true + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "optional": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "node_modules/inherits": { "version": "2.0.4", "license": "ISC" }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, "node_modules/ip-address": { "version": "10.0.1", "license": "MIT", @@ -1093,6 +1527,13 @@ "node": ">=0.10.0" } }, + "node_modules/is-lambda": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz", + "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==", + "license": "MIT", + "optional": true + }, "node_modules/is-number": { "version": "7.0.0", "dev": true, @@ -1101,6 +1542,13 @@ "node": ">=0.12.0" } }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC", + "optional": true + }, "node_modules/js-tokens": { "version": "4.0.0", "license": "MIT" @@ -1130,6 +1578,129 @@ "node": ">=12" } }, + "node_modules/make-fetch-happen": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-9.1.0.tgz", + "integrity": "sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg==", + "license": "ISC", + "optional": true, + "dependencies": { + "agentkeepalive": "^4.1.3", + "cacache": "^15.2.0", + "http-cache-semantics": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-lambda": "^1.0.1", + "lru-cache": "^6.0.0", + "minipass": "^3.1.3", + "minipass-collect": "^1.0.2", + "minipass-fetch": "^1.3.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.2", + "promise-retry": "^2.0.1", + "socks-proxy-agent": "^6.0.0", + "ssri": "^8.0.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/make-fetch-happen/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/make-fetch-happen/node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/make-fetch-happen/node_modules/http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/make-fetch-happen/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "optional": true, + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/make-fetch-happen/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "license": "ISC", + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-fetch-happen/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT", + "optional": true + }, + "node_modules/make-fetch-happen/node_modules/socks-proxy-agent": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.2.1.tgz", + "integrity": "sha512-a6KW9G+6B3nWZ1yB8G7pJwL3ggLy1uTzKAgCb7ttblwqdz9fMGJUuTy3uFzEP48FAs9FLILlmzDlE2JJhVQaXQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "agent-base": "^6.0.2", + "debug": "^4.3.3", + "socks": "^2.6.2" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "license": "MIT", @@ -1185,9 +1756,21 @@ "node": ">= 0.6" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "3.1.2", - "dev": true, + "devOptional": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -1196,10 +1779,126 @@ "node": "*" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-collect": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", + "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-fetch": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.4.1.tgz", + "integrity": "sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw==", + "license": "MIT", + "optional": true, + "dependencies": { + "minipass": "^3.1.0", + "minipass-sized": "^1.0.3", + "minizlib": "^2.0.0" + }, + "engines": { + "node": ">=8" + }, + "optionalDependencies": { + "encoding": "^0.1.12" + } + }, + "node_modules/minipass-flush": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", + "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", + "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "license": "MIT", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/mitt": { "version": "3.0.1", "license": "MIT" }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/mkdirp-classic": { "version": "0.5.3", "license": "MIT" @@ -1208,6 +1907,12 @@ "version": "2.0.0", "license": "MIT" }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, "node_modules/negotiator": { "version": "0.6.3", "license": "MIT", @@ -1222,6 +1927,24 @@ "node": ">= 0.4.0" } }, + "node_modules/node-abi": { + "version": "3.75.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.75.0.tgz", + "integrity": "sha512-OhYaY5sDsIka7H7AtijtI9jwGYLyl29eQn/W623DiN/MIv5sUqc4g7BIDThX+gb7di9f6xK02nkp8sdfFWZLTg==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "license": "MIT" + }, "node_modules/node-fetch": { "version": "2.7.0", "license": "MIT", @@ -1240,6 +1963,31 @@ } } }, + "node_modules/node-gyp": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz", + "integrity": "sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w==", + "license": "MIT", + "optional": true, + "dependencies": { + "env-paths": "^2.2.0", + "glob": "^7.1.4", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^9.1.0", + "nopt": "^5.0.0", + "npmlog": "^6.0.0", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.2", + "which": "^2.0.2" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": ">= 10.12.0" + } + }, "node_modules/nodemon": { "version": "3.1.10", "dev": true, @@ -1288,6 +2036,22 @@ "dev": true, "license": "MIT" }, + "node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "dev": true, @@ -1296,6 +2060,23 @@ "node": ">=0.10.0" } }, + "node_modules/npmlog": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", + "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "optional": true, + "dependencies": { + "are-we-there-yet": "^3.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^4.0.3", + "set-blocking": "^2.0.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, "node_modules/object-inspect": { "version": "1.13.4", "license": "MIT", @@ -1323,6 +2104,22 @@ "wrappy": "1" } }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/pac-proxy-agent": { "version": "7.2.0", "license": "MIT", @@ -1403,6 +2200,16 @@ "node": ">= 0.8" } }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/path-to-regexp": { "version": "0.1.12", "license": "MIT" @@ -1426,6 +2233,66 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/prebuild-install/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/prebuild-install/node_modules/tar-fs": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz", + "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/prebuild-install/node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/progress": { "version": "2.0.3", "license": "MIT", @@ -1433,6 +2300,27 @@ "node": ">=0.4.0" } }, + "node_modules/promise-inflight": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", + "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==", + "license": "ISC", + "optional": true + }, + "node_modules/promise-retry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", + "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "license": "MIT", + "optional": true, + "dependencies": { + "err-code": "^2.0.2", + "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "license": "MIT", @@ -1580,6 +2468,35 @@ "node": ">= 0.8" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/readdirp": { "version": "3.6.0", "dev": true, @@ -1605,6 +2522,33 @@ "node": ">=4" } }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "optional": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "funding": [ @@ -1629,7 +2573,6 @@ }, "node_modules/semver": { "version": "7.7.2", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -1684,6 +2627,13 @@ "node": ">= 0.8.0" } }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "license": "ISC", + "optional": true + }, "node_modules/setprototypeof": { "version": "1.2.0", "license": "ISC" @@ -1752,6 +2702,58 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC", + "optional": true + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/simple-update-notifier": { "version": "2.0.0", "dev": true, @@ -1822,6 +2824,43 @@ "node": ">=0.10.0" } }, + "node_modules/sqlite3": { + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.1.7.tgz", + "integrity": "sha512-GGIyOiFaG+TUra3JIfkI/zGP8yZYLPQ0pl1bH+ODjiX57sPhrLU5sQJn1y9bDKZUFYkX1crlrPfSYt0BKKdkog==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "bindings": "^1.5.0", + "node-addon-api": "^7.0.0", + "prebuild-install": "^7.1.1", + "tar": "^6.1.11" + }, + "optionalDependencies": { + "node-gyp": "8.x" + }, + "peerDependencies": { + "node-gyp": "8.x" + }, + "peerDependenciesMeta": { + "node-gyp": { + "optional": true + } + } + }, + "node_modules/ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "minipass": "^3.1.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/statuses": { "version": "2.0.1", "license": "MIT", @@ -1840,6 +2879,15 @@ "bare-events": "^2.2.0" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-width": { "version": "4.2.3", "license": "MIT", @@ -1862,6 +2910,15 @@ "node": ">=8" } }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/supports-color": { "version": "5.5.0", "dev": true, @@ -1873,6 +2930,23 @@ "node": ">=4" } }, + "node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "license": "ISC", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/tar-fs": { "version": "3.0.4", "license": "MIT", @@ -1891,6 +2965,15 @@ "streamx": "^2.15.0" } }, + "node_modules/tar/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "license": "ISC", + "engines": { + "node": ">=8" + } + }, "node_modules/text-decoder": { "version": "1.2.3", "license": "Apache-2.0", @@ -1936,6 +3019,18 @@ "version": "2.8.1", "license": "0BSD" }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-is": { "version": "1.6.18", "license": "MIT", @@ -1965,6 +3060,26 @@ "license": "MIT", "optional": true }, + "node_modules/unique-filename": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", + "integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==", + "license": "ISC", + "optional": true, + "dependencies": { + "unique-slug": "^2.0.0" + } + }, + "node_modules/unique-slug": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.2.tgz", + "integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==", + "license": "ISC", + "optional": true, + "dependencies": { + "imurmurhash": "^0.1.4" + } + }, "node_modules/unpipe": { "version": "1.0.0", "license": "MIT", @@ -1976,6 +3091,12 @@ "version": "10.0.0", "license": "MIT" }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, "node_modules/utils-merge": { "version": "1.0.1", "license": "MIT", @@ -2002,6 +3123,32 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "optional": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "license": "ISC", + "optional": true, + "dependencies": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "license": "MIT", @@ -2047,6 +3194,12 @@ "node": ">=10" } }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, "node_modules/yargs": { "version": "17.7.2", "license": "MIT", diff --git a/package.json b/package.json index c0168fe..8debd2c 100644 --- a/package.json +++ b/package.json @@ -9,10 +9,11 @@ }, "dependencies": { "express": "^4.18.2", + "fs": "^0.0.1-security", "puppeteer": "^21.0.0", - "fs": "^0.0.1-security" + "sqlite3": "^5.1.7" }, "devDependencies": { "nodemon": "^3.0.1" } -} \ No newline at end of file +} diff --git a/pdga-numbers.txt b/pdga-numbers.txt index 567898b..c2cb65d 100644 --- a/pdga-numbers.txt +++ b/pdga-numbers.txt @@ -6,4 +6,5 @@ 201220 288059 242229 -155816 \ No newline at end of file +155816 +242797 \ No newline at end of file diff --git a/server.js b/server.js index 28db239..a48665e 100644 --- a/server.js +++ b/server.js @@ -3,12 +3,302 @@ const puppeteer = require('puppeteer'); const https = require('https'); const fs = require('fs'); const path = require('path'); +const sqlite3 = require('sqlite3').verbose(); const app = express(); const PORT = 3000; app.use(express.static('public')); +// Initialize SQLite database +const db = new sqlite3.Database('./ratings.db'); + +// Initialize database schema +function initializeDatabase() { + return new Promise((resolve, reject) => { + db.serialize(() => { + // Create players table + db.run(` + CREATE TABLE IF NOT EXISTS players ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + pdga_number INTEGER UNIQUE NOT NULL, + name TEXT NOT NULL, + current_rating INTEGER, + rating_change INTEGER, + last_updated DATETIME DEFAULT CURRENT_TIMESTAMP, + last_round_update DATETIME DEFAULT NULL + ) + `); + + // Migration: Add last_round_update column if it doesn't exist + db.get("PRAGMA table_info(players)", (err, info) => { + if (err) { + console.error('Error checking table schema:', err); + return; + } + + // Check if column exists by querying table info + db.all("PRAGMA table_info(players)", (err, columns) => { + if (err) { + console.error('Error getting table info:', err); + return; + } + + const hasLastRoundUpdate = columns.some(col => col.name === 'last_round_update'); + + if (!hasLastRoundUpdate) { + console.log('Adding last_round_update column to players table...'); + db.run(` + ALTER TABLE players ADD COLUMN last_round_update DATETIME DEFAULT NULL + `, (err) => { + if (err) { + console.error('Error adding last_round_update column:', err.message); + } else { + console.log('Successfully added last_round_update column'); + } + }); + } + }); + }); + + // Create round_history table + db.run(` + CREATE TABLE IF NOT EXISTS round_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + player_id INTEGER NOT NULL, + date DATE NOT NULL, + competition_name TEXT NOT NULL, + rating INTEGER NOT NULL, + FOREIGN KEY (player_id) REFERENCES players (id) + ) + `); + + // Create rating_history table + db.run(` + CREATE TABLE IF NOT EXISTS rating_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + player_id INTEGER NOT NULL, + date DATE NOT NULL, + rating INTEGER NOT NULL, + FOREIGN KEY (player_id) REFERENCES players (id) + ) + `, (err) => { + if (err) { + reject(err); + } else { + console.log('Database initialized successfully'); + resolve(); + } + }); + }); + }); +} + +// Database helper functions +function getPlayerFromDB(pdgaNumber) { + return new Promise((resolve, reject) => { + db.get( + 'SELECT * FROM players WHERE pdga_number = ? AND datetime(last_updated) > datetime("now", "-24 hours")', + [pdgaNumber], + (err, row) => { + if (err) reject(err); + else resolve(row); + } + ); + }); +} + +function savePlayerToDB(playerData) { + return new Promise((resolve, reject) => { + db.run( + `INSERT OR REPLACE INTO players (pdga_number, name, current_rating, rating_change, last_updated) + VALUES (?, ?, ?, ?, datetime('now'))`, + [playerData.pdgaNumber, playerData.name, playerData.rating, playerData.ratingChange], + function(err) { + if (err) reject(err); + else resolve(this.lastID); + } + ); + }); +} + +function getRatingHistoryFromDB(pdgaNumber) { + return new Promise((resolve, reject) => { + db.get('SELECT id FROM players WHERE pdga_number = ?', [pdgaNumber], (err, player) => { + if (err) return reject(err); + if (!player) return resolve(null); + + db.all( + 'SELECT * FROM rating_history WHERE player_id = ? ORDER BY date ASC', + [player.id], + (err, rows) => { + if (err) reject(err); + else resolve(rows); + } + ); + }); + }); +} + +function saveRatingHistoryToDB(pdgaNumber, history) { + return new Promise((resolve, reject) => { + db.get('SELECT id FROM players WHERE pdga_number = ?', [pdgaNumber], (err, player) => { + if (err) return reject(err); + if (!player) return reject(new Error('Player not found')); + + // Clear existing history for this player + db.run('DELETE FROM rating_history WHERE player_id = ?', [player.id], (err) => { + if (err) return reject(err); + + // Insert new history + const stmt = db.prepare('INSERT INTO rating_history (player_id, date, rating) VALUES (?, ?, ?)'); + + for (const entry of history) { + stmt.run([player.id, entry.date, entry.rating]); + } + + stmt.finalize((err) => { + if (err) reject(err); + else resolve(); + }); + }); + }); + }); +} + +function getRoundHistoryFromDB(pdgaNumber) { + return new Promise((resolve, reject) => { + db.get('SELECT id FROM players WHERE pdga_number = ?', [pdgaNumber], (err, player) => { + if (err) return reject(err); + if (!player) return resolve([]); + + db.all( + 'SELECT * FROM round_history WHERE player_id = ? ORDER BY date DESC', + [player.id], + (err, rows) => { + if (err) reject(err); + else resolve(rows); + } + ); + }); + }); +} + +function getLastRoundUpdateDate(pdgaNumber) { + return new Promise((resolve, reject) => { + db.get( + 'SELECT last_round_update FROM players WHERE pdga_number = ?', + [pdgaNumber], + (err, row) => { + if (err) reject(err); + else resolve(row ? row.last_round_update : null); + } + ); + }); +} + +function updateLastRoundUpdateDate(pdgaNumber) { + return new Promise((resolve, reject) => { + db.run( + 'UPDATE players SET last_round_update = CURRENT_TIMESTAMP WHERE pdga_number = ?', + [pdgaNumber], + function(err) { + if (err) reject(err); + else resolve(); + } + ); + }); +} + +function saveRatingHistoryToDB(pdgaNumber, ratingHistory) { + return new Promise((resolve, reject) => { + db.get('SELECT id FROM players WHERE pdga_number = ?', [pdgaNumber], (err, player) => { + if (err) return reject(err); + if (!player) return reject(new Error('Player not found')); + + // Clear existing rating history for this player + db.run('DELETE FROM rating_history WHERE player_id = ?', [player.id], (err) => { + if (err) return reject(err); + + if (ratingHistory.length === 0) { + return resolve(); + } + + let completed = 0; + const total = ratingHistory.length; + + ratingHistory.forEach(entry => { + const parsedDate = parseDate(entry.date); + + db.run( + 'INSERT INTO rating_history (player_id, date, rating) VALUES (?, ?, ?)', + [player.id, parsedDate.toISOString().split('T')[0], entry.rating], + (err) => { + if (err) return reject(err); + + completed++; + if (completed === total) { + resolve(); + } + } + ); + }); + }); + }); + }); +} + +function saveRoundHistoryToDB(pdgaNumber, roundData, isIncremental = false) { + return new Promise((resolve, reject) => { + db.get('SELECT id FROM players WHERE pdga_number = ?', [pdgaNumber], (err, player) => { + if (err) return reject(err); + if (!player) return reject(new Error('Player not found')); + + const processRounds = () => { + if (roundData.length === 0) { + // Update last_round_update timestamp even if no new rounds + db.run('UPDATE players SET last_round_update = datetime("now") WHERE pdga_number = ?', [pdgaNumber], (err) => { + if (err) reject(err); + else resolve(); + }); + return; + } + + // Insert new round history + const stmt = db.prepare('INSERT OR REPLACE INTO round_history (player_id, date, competition_name, rating) VALUES (?, ?, ?, ?)'); + + for (const round of roundData) { + stmt.run([player.id, round.date.toISOString().split('T')[0], round.competition || 'Unknown', round.rating]); + } + + stmt.finalize((err) => { + if (err) { + reject(err); + } else { + // Update last_round_update timestamp + db.run('UPDATE players SET last_round_update = datetime("now") WHERE pdga_number = ?', [pdgaNumber], (updateErr) => { + if (updateErr) reject(updateErr); + else resolve(); + }); + } + }); + }; + + if (!isIncremental) { + // Clear existing round history for full refresh + db.run('DELETE FROM round_history WHERE player_id = ?', [player.id], (err) => { + if (err) return reject(err); + processRounds(); + }); + } else { + // For incremental updates, just add new rounds + processRounds(); + } + }); + }); +} + +// Legacy in-memory cache (will be phased out) const cache = new Map(); const CACHE_DURATION = 24 * 60 * 60 * 1000; @@ -35,12 +325,41 @@ async function fetchPlayerDataHTTP(pdgaNumber) { if (res.statusCode === 200) { resolve(data); } else { - reject(new Error(`HTTP ${res.statusCode}`)); + // Log rate limiting information if available + const rateLimitInfo = { + statusCode: res.statusCode, + headers: res.headers + }; + + console.log(`PDGA Response Status for #${pdgaNumber}: ${res.statusCode}`); + console.log('Response Headers:', JSON.stringify(res.headers, null, 2)); + + // Check for common rate limiting headers + if (res.headers['retry-after']) { + console.log(`Retry-After header: ${res.headers['retry-after']}`); + } + if (res.headers['x-ratelimit-limit']) { + console.log(`Rate Limit: ${res.headers['x-ratelimit-limit']}`); + } + if (res.headers['x-ratelimit-remaining']) { + console.log(`Rate Limit Remaining: ${res.headers['x-ratelimit-remaining']}`); + } + if (res.headers['x-ratelimit-reset']) { + console.log(`Rate Limit Reset: ${res.headers['x-ratelimit-reset']}`); + } + + const error = new Error(`HTTP ${res.statusCode}`); + error.rateLimitInfo = rateLimitInfo; + reject(error); } }); }); req.on('error', (error) => { + console.log(`Request error for PDGA #${pdgaNumber}:`, error.code, error.message); + if (error.code === 'ECONNRESET') { + console.log('Connection reset - likely rate limited by PDGA'); + } reject(error); }); @@ -88,12 +407,21 @@ function parsePlayerData(html, pdgaNumber) { } async function scrapePDGARating(pdgaNumber, retries = 3) { - const cacheKey = `player-${pdgaNumber}`; - const cached = cache.get(cacheKey); - - if (cached && Date.now() - cached.timestamp < CACHE_DURATION) { - console.log(`Using cached data for PDGA ${pdgaNumber}`); - return cached.data; + // Check database first + try { + const cachedPlayer = await getPlayerFromDB(pdgaNumber); + if (cachedPlayer) { + console.log(`Using cached data from DB for PDGA ${pdgaNumber}`); + return { + pdgaNumber: cachedPlayer.pdga_number, + name: cachedPlayer.name, + rating: cachedPlayer.current_rating, + ratingChange: cachedPlayer.rating_change, + predictedRating: null + }; + } + } catch (err) { + console.error(`Database error for PDGA ${pdgaNumber}:`, err.message); } for (let attempt = 1; attempt <= retries; attempt++) { @@ -103,10 +431,13 @@ async function scrapePDGARating(pdgaNumber, retries = 3) { const html = await fetchPlayerDataHTTP(pdgaNumber); const result = parsePlayerData(html, pdgaNumber); - cache.set(cacheKey, { - data: result, - timestamp: Date.now() - }); + // Save to database + try { + await savePlayerToDB(result); + console.log(`Saved PDGA ${pdgaNumber} to database`); + } catch (dbErr) { + console.error(`Failed to save PDGA ${pdgaNumber} to database:`, dbErr.message); + } console.log(`Successfully scraped PDGA ${pdgaNumber} on attempt ${attempt}`); return result; @@ -124,8 +455,25 @@ async function scrapePDGARating(pdgaNumber, retries = 3) { }; } - // Wait before retry - await new Promise(resolve => setTimeout(resolve, 2000 * attempt)); + // Adaptive retry delay based on error type + let retryDelay = 2000 * attempt; // Base delay + + if (error.rateLimitInfo) { + const retryAfter = error.rateLimitInfo.headers['retry-after']; + if (retryAfter) { + // If server tells us when to retry, use that + some buffer + retryDelay = Math.max(retryDelay, (parseInt(retryAfter) + 1) * 1000); + console.log(`Using Retry-After header: waiting ${retryDelay/1000}s`); + } + } + + if (error.code === 'ECONNRESET') { + // Connection reset usually means rate limiting - wait longer + retryDelay = Math.max(retryDelay, 10000); + console.log(`Connection reset detected: waiting ${retryDelay/1000}s`); + } + + await new Promise(resolve => setTimeout(resolve, retryDelay)); } } } @@ -157,9 +505,199 @@ async function getPredictedRating(browser, pdgaNumber, retries = 2) { return 0; } -async function getPlayerCompetitionRatings(browser, pdgaNumber) { +async function getPredictedRatingFromDB(pdgaNumber) { + try { + const roundHistory = await getRoundHistoryFromDB(pdgaNumber); + if (roundHistory.length > 0) { + console.log(`Using ${roundHistory.length} cached rounds for PDGA ${pdgaNumber} prediction`); + + // Convert to the format expected by calculatePredictedRating + const roundRatings = roundHistory.map(round => ({ + rating: round.rating, + date: new Date(round.date) + })); + + return calculatePredictedRating(roundRatings); + } + return 0; + } catch (err) { + console.error(`Error getting predicted rating from DB for ${pdgaNumber}:`, err.message); + return 0; + } +} + +async function getOfficialRatingHistory(browser, pdgaNumber) { + const page = await browser.newPage(); + let ratingHistory = []; + + try { + const url = `https://www.pdga.com/player/${pdgaNumber}/history`; + await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 }); + await page.waitForTimeout(2000); + + // Extract the rating history data + ratingHistory = await page.evaluate(() => { + const history = []; + + // Try each selector until we find rating data + const selectors = [ + 'table tbody tr', + 'table tr', + '.view-content tbody tr' + ]; + + for (const selector of selectors) { + const rows = document.querySelectorAll(selector); + + for (const row of rows) { + const cells = row.querySelectorAll('td'); + if (cells.length >= 3) { + const dateText = cells[0]?.innerText?.trim(); + const ratingText = cells[1]?.innerText?.trim(); + + // Check if this looks like a date and rating + if (dateText && ratingText && /^\d{4}-\d{2}-\d{2}$|^\d{1,2}-\w{3}-\d{4}$|^\w{3} \d{1,2}, \d{4}$/.test(dateText)) { + const rating = parseInt(ratingText); + if (!isNaN(rating) && rating > 800 && rating < 1200) { + history.push({ + date: dateText, + rating: rating, + tournament: cells[2]?.innerText?.trim() || 'Unknown' + }); + } + } + } + } + + if (history.length > 0) break; + } + + return history; + }); + + + } catch (error) { + console.error('Error fetching official rating history:', error.message); + } finally { + await page.close(); + } + + return ratingHistory; +} + +async function getPlayerTournamentDetails(browser, pdgaNumber) { + const page = await browser.newPage(); + let tournamentRounds = []; + + try { + const url = `https://www.pdga.com/player/${pdgaNumber}/details`; + await page.goto(url, { waitUntil: 'networkidle2', timeout: 45000 }); + await page.waitForTimeout(2000); + + // Extract individual tournament rounds with actual dates and ratings + tournamentRounds = await page.evaluate(() => { + const rounds = []; + const rows = document.querySelectorAll('table tbody tr'); + + // Log first few rows to see structure + console.log('First few table rows for debugging:'); + for (let i = 0; i < Math.min(3, rows.length); i++) { + const cells = rows[i].querySelectorAll('td'); + const cellTexts = Array.from(cells).map(cell => cell.innerText.trim()); + console.log(`Row ${i}: [${cellTexts.join(' | ')}]`); + } + + rows.forEach(row => { + const cells = row.querySelectorAll('td'); + + // Try to identify which columns contain date and rating information + if (cells.length >= 4) { + const cellTexts = Array.from(cells).map(cell => cell.innerText.trim()); + + // Look for patterns in the data + let tournamentName = ''; + let dateText = ''; + let rating = 0; + let division = ''; + + // Try to find date and rating in different column positions + cellTexts.forEach((text, index) => { + // Look for date patterns, including multi-day tournaments + // Examples: "2-Sep-2023", "2-Sep to 3-Sep-2023", "2 to 3-Sep-2023" + if (/\d{1,2}(-\w{3})?(\s+to\s+)\d{1,2}-\w{3}-\d{4}/.test(text) || /\d{1,2}-\w{3}-\d{4}/.test(text)) { + dateText = text; + } + // Look for rating patterns (3-4 digit numbers between 800-1200) + if (/^\d{3,4}$/.test(text) && parseInt(text) >= 800 && parseInt(text) <= 1200) { + rating = parseInt(text); + } + // Look for division patterns (like MA3, MPO, etc.) + if (/^M[A-Z]\d*$|^F[A-Z]\d*$/.test(text)) { + division = text; + } + // First cell is usually tournament name + if (index === 0) { + tournamentName = text; + } + }); + + if (tournamentName && dateText && rating > 0) { + rounds.push({ + tournament: tournamentName, + dateText: dateText, + rating: rating, + division: division, + competition: `${tournamentName} (${division})` + }); + } + } + }); + + return rounds; + }); + + // Parse dates properly after extraction + const fixedRounds = tournamentRounds.map(round => { + let validDate = new Date(); + if (round.dateText) { + try { + const pdgaParsed = parseDate(round.dateText); + if (pdgaParsed instanceof Date && !isNaN(pdgaParsed.getTime())) { + validDate = pdgaParsed; + } else { + const nativeParsed = new Date(round.dateText); + if (!isNaN(nativeParsed.getTime())) { + validDate = nativeParsed; + } + } + } catch (e) { + console.log(`Date parsing failed for "${round.dateText}": ${e.message}`); + } + } + return { + tournament: round.tournament, + date: validDate, + rating: round.rating, + division: round.division, + competition: round.competition + }; + }); + tournamentRounds = fixedRounds; + + } catch (error) { + console.error('Error fetching tournament details:', error.message); + } finally { + await page.close(); + } + + return tournamentRounds; +} + +async function getPlayerCompetitionRatings(browser, pdgaNumber, sinceDate = null) { const page = await browser.newPage(); let allRatings = []; + let tournamentCount = 0; + let successfulTournaments = 0; try { const url = `https://www.pdga.com/player/${pdgaNumber}`; @@ -168,8 +706,9 @@ async function getPlayerCompetitionRatings(browser, pdgaNumber) { // Calculate the next PDGA update date to filter tournaments const nextUpdateDate = getNextPDGAUpdateDate(); - const tournamentUrls = await page.evaluate((nextUpdateTimestamp) => { + const tournamentUrls = await page.evaluate((nextUpdateTimestamp, sinceDateString) => { const nextUpdateDate = new Date(nextUpdateTimestamp); + const sinceDate = sinceDateString ? new Date(sinceDateString) : null; const tables = document.querySelectorAll('table[id*="player-results"]'); const urls = []; @@ -189,7 +728,11 @@ async function getPlayerCompetitionRatings(browser, pdgaNumber) { const oneYearAgo = new Date(); oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1); - if (date > oneYearAgo && date < nextUpdateDate) { + // Apply date filters + const dateValid = date > oneYearAgo && date < nextUpdateDate; + const isNewTournament = !sinceDate || date > sinceDate; + + if (dateValid && isNewTournament) { const href = tournamentCell.getAttribute('href'); if (href) { urls.push({ @@ -203,17 +746,40 @@ async function getPlayerCompetitionRatings(browser, pdgaNumber) { }); }); - return urls.slice(0, 8); // Reduce number of tournaments to scrape - }, nextUpdateDate.getTime()); + return urls; // Get all tournaments from the past year + }, nextUpdateDate.getTime(), sinceDate ? sinceDate.toISOString() : null); - console.log(`Found ${tournamentUrls.length} recent tournaments for PDGA ${pdgaNumber}`); + const updateType = sinceDate ? `incremental (since ${sinceDate.toDateString()})` : 'full'; + console.log(`Found ${tournamentUrls.length} tournaments for PDGA ${pdgaNumber} (${updateType})`); for (const tournamentData of tournamentUrls) { + tournamentCount++; try { - await page.goto(tournamentData.url, { waitUntil: 'domcontentloaded', timeout: 45000 }); + console.log(`[${tournamentCount}/${tournamentUrls.length}] Navigating to tournament: ${tournamentData.url}`); + const navigationStart = Date.now(); + + try { + await page.goto(tournamentData.url, { waitUntil: 'domcontentloaded', timeout: 45000 }); + const navigationTime = Date.now() - navigationStart; + console.log(`✓ Navigation completed in ${navigationTime}ms`); + } catch (navError) { + console.error(`✗ Navigation failed for ${tournamentData.url}:`); + console.error('Navigation error details:', { + type: navError.constructor.name, + message: navError.message, + code: navError.code, + stack: navError.stack?.split('\n')[0] + }); + throw navError; // Re-throw to be caught by outer try-catch + } + + console.log(`Waiting 3s before scraping tournament data...`); await page.waitForTimeout(3000); // Longer delay between requests - const roundRatings = await page.evaluate((pdgaNum) => { + console.log(`Starting page evaluation for PDGA ${pdgaNumber}...`); + let roundRatings; + try { + roundRatings = await page.evaluate((pdgaNum) => { const rows = document.querySelectorAll('tr'); for (const row of rows) { @@ -239,25 +805,93 @@ async function getPlayerCompetitionRatings(browser, pdgaNumber) { return []; }, pdgaNumber); + console.log(`✓ Page evaluation completed, found ${roundRatings.length} round ratings`); + } catch (evalError) { + console.error(`✗ Page evaluation failed for ${tournamentData.url}:`); + console.error('Evaluation error details:', { + type: evalError.constructor.name, + message: evalError.message, + code: evalError.code, + stack: evalError.stack?.split('\n')[0] + }); + throw evalError; // Re-throw to be caught by outer try-catch + } if (roundRatings.length > 0) { const parsedDate = parseDate(tournamentData.date); + // Extract tournament name from URL for better database storage + const tournamentName = tournamentData.url.split('/').pop() || 'Unknown Tournament'; + + const newRounds = []; roundRatings.forEach(rating => { - allRatings.push({ + const roundData = { rating, - date: parsedDate - }); + date: parsedDate, + competition: tournamentName + }; + allRatings.push(roundData); + newRounds.push(roundData); }); - console.log(`Found ${roundRatings.length} round ratings for ${tournamentData.url}`); + + successfulTournaments++; + console.log(`✓ [${tournamentCount}/${tournamentUrls.length}] Found ${roundRatings.length} round ratings for ${tournamentName}`); + + // Save rounds immediately to database (partial save) + try { + await saveRoundHistoryToDB(pdgaNumber, newRounds, true); + console.log(`💾 Saved ${newRounds.length} rounds to database`); + } catch (saveError) { + console.error(`⚠️ Could not save rounds to DB: ${saveError.message}`); + } + + } else { + console.log(`✗ [${tournamentCount}/${tournamentUrls.length}] No round ratings found for ${tournamentData.url}`); } } catch (error) { - console.error(`Error scraping tournament ${tournamentData.url}:`, error); + console.error(`✗ [${tournamentCount}/${tournamentUrls.length}] Error scraping tournament ${tournamentData.url}:`); + console.error('Tournament error type:', error.constructor.name); + console.error('Tournament error message:', error.message); + console.error('Tournament error code:', error.code); + console.error('Tournament error name:', error.name); + console.error('Tournament full error object:', JSON.stringify(error, Object.getOwnPropertyNames(error), 2)); + + // Log the current state when error occurs + console.error(`Tournament scraping progress: ${tournamentCount}/${tournamentUrls.length} (${successfulTournaments} successful so far)`); + console.error(`Total rounds collected before this error: ${allRatings.length}`); + + if (error.message.includes('socket hang up')) { + console.error('🔌 Socket hang up detected at tournament level - PDGA may be rate limiting'); + console.error('💡 Will continue trying remaining tournaments after this failure'); + } + if (error.message.includes('Navigation timeout')) { + console.error('⏰ Navigation timeout at tournament level - page took too long to load'); + } + if (error.message.includes('net::ERR_CONNECTION_RESET')) { + console.error('🚫 Connection reset at tournament level - PDGA blocking requests'); + } + + // Don't let individual tournament failures stop the whole process + console.error('⚠️ Continuing with next tournament despite this error...'); } } + // Log summary of scraping results + console.log(`=== Scraping Summary for PDGA ${pdgaNumber} ===`); + console.log(`Tournaments processed: ${tournamentCount}/${tournamentUrls.length}`); + console.log(`Successful tournaments: ${successfulTournaments}`); + console.log(`Total rounds found: ${allRatings.length}`); + console.log(`Completion rate: ${Math.round((successfulTournaments / tournamentUrls.length) * 100)}%`); + } catch (error) { console.error(`Error getting competition ratings for PDGA ${pdgaNumber}:`, error); + console.error(`=== Partial Results Before Error ===`); + console.error(`Tournaments processed: ${tournamentCount}/${tournamentUrls.length || 0}`); + console.error(`Successful tournaments: ${successfulTournaments}`); + console.error(`Total rounds collected: ${allRatings.length}`); + if (allRatings.length > 0) { + console.error(`Rounds saved to database before error occurred`); + } } finally { await page.close(); } @@ -267,6 +901,23 @@ async function getPlayerCompetitionRatings(browser, pdgaNumber) { } function parseDate(dateStr) { + // Handle multi-day tournament formats first + // Examples: "2-Sep to 3-Sep-2023", "2 to 3-Sep-2023" + const multiDayMatch = dateStr.match(/^(\d{1,2})(-([A-Za-z]{3}))?(\s+to\s+)(\d{1,2})-([A-Za-z]{3})-(\d{4})$/); + if (multiDayMatch) { + // Extract first day and use that as the tournament date + const day = parseInt(multiDayMatch[1]); + const month = multiDayMatch[3] || multiDayMatch[6]; // Use first month if available, otherwise second + const year = parseInt(multiDayMatch[7]); + + const monthMap = { + 'Jan': 0, 'Feb': 1, 'Mar': 2, 'Apr': 3, 'May': 4, 'Jun': 5, + 'Jul': 6, 'Aug': 7, 'Sep': 8, 'Oct': 9, 'Nov': 10, 'Dec': 11 + }; + + return new Date(year, monthMap[month], day); + } + const formats = [ /^(\d{1,2})-([A-Za-z]{3})-(\d{4})$/, /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/ @@ -331,59 +982,169 @@ function getNextPDGAUpdateDate() { } function calculatePredictedRating(roundRatings) { - if (!roundRatings || roundRatings.length === 0) return 0; + console.log('\n=== PDGA RATING CALCULATION (Following Official Rules) ==='); - // Get the next PDGA rating update date - only include rounds before this date - const nextUpdateDate = getNextPDGAUpdateDate(); - console.log(`Next PDGA update date: ${nextUpdateDate.toDateString()}`); - - // Filter rounds to only include those before the next update date, then sort by date (most recent first) and extract ratings - const sortedRatings = roundRatings - .filter(r => r.date < nextUpdateDate) // Only include rounds before next update - .sort((a, b) => b.date - a.date) - .map(r => r.rating) - .filter(r => r > 0); - - if (sortedRatings.length === 0) return 0; - - // PDGA Rule: Need at least 7 rounds to apply outlier exclusion - if (sortedRatings.length >= 7) { - // Calculate standard deviation for outlier detection - const mean = sortedRatings.reduce((sum, r) => sum + r, 0) / sortedRatings.length; - const stdDev = calculateStandardDeviation(sortedRatings); - - // PDGA Rule: Exclude rounds more than 2.5 standard deviations below average - const filteredRatings = sortedRatings.filter(rating => - rating >= (mean - 2.5 * stdDev) - ); - - // Use filtered ratings if we still have enough data - if (filteredRatings.length >= 4) { - sortedRatings.splice(0, sortedRatings.length, ...filteredRatings); - } + if (!roundRatings || roundRatings.length === 0) { + console.log('❌ No rounds provided for prediction'); + return 0; } - // PDGA Rule: Most recent 25% of rounds count double if 9+ rounds + console.log(`📊 Starting with ${roundRatings.length} total rounds`); + + // PDGA Simulation: Only include rounds that would be rated by next update + const nextUpdateDate = getNextPDGAUpdateDate(); + console.log(`🎯 PDGA Update Simulation: Next update date is ${nextUpdateDate.toDateString()}`); + console.log(` Only including rounds played before ${nextUpdateDate.toDateString()}`); + + // Sort all rounds by date (most recent first), but only include rounds before next update + const allSortedRounds = roundRatings + .filter(r => r.rating > 0 && r.date < nextUpdateDate) + .sort((a, b) => b.date - a.date); + + if (allSortedRounds.length === 0) { + console.log('❌ No valid rounds after filtering for update date'); + return 0; + } + + console.log(`📊 After update date filter: ${allSortedRounds.length} rounds`); + + // PDGA Rule: Use rounds from 12 months prior to next update date + const twelveMonthsBeforeUpdate = new Date(nextUpdateDate); + twelveMonthsBeforeUpdate.setFullYear(twelveMonthsBeforeUpdate.getFullYear() - 1); + + const mostRecentDate = allSortedRounds[0].date; + console.log(`📅 Most recent round: ${mostRecentDate.toDateString()}`); + console.log(`📅 12-month cutoff: ${twelveMonthsBeforeUpdate.toDateString()} (1 year before update)`); + + // Step 1: Get rounds from last 12 months before update + let eligibleRounds = allSortedRounds.filter(r => r.date >= twelveMonthsBeforeUpdate); + + console.log(`\n🗓️ 12-MONTH FILTERING:`); + console.log(`✅ Rounds in last 12 months: ${eligibleRounds.length}`); + + // PDGA Rule: If fewer than 8 rounds in 12 months, extend to 24 months before update + if (eligibleRounds.length < 8) { + const twentyFourMonthsBeforeUpdate = new Date(nextUpdateDate); + twentyFourMonthsBeforeUpdate.setFullYear(twentyFourMonthsBeforeUpdate.getFullYear() - 2); + + eligibleRounds = allSortedRounds.filter(r => r.date >= twentyFourMonthsBeforeUpdate); + console.log(`⚠️ Extended to 24 months before update (${twentyFourMonthsBeforeUpdate.toDateString()}) - now ${eligibleRounds.length} rounds`); + } + + if (eligibleRounds.length === 0) { + console.log('❌ No eligible rounds found'); + return 0; + } + + console.log(`\n📈 ELIGIBLE ROUNDS: ${eligibleRounds.length}`); + eligibleRounds.forEach((round, index) => { + console.log(` ${index + 1}. ${round.date.toDateString()}: ${round.rating} (${round.competition})`); + }); + + let workingRounds = [...eligibleRounds]; + let workingRatings = workingRounds.map(r => r.rating); + + // PDGA Rule: Apply outlier exclusion if ≥7 rounds + if (workingRatings.length >= 7) { + console.log(`\n🔍 OUTLIER EXCLUSION (≥7 rounds available):`); + + const mean = workingRatings.reduce((sum, r) => sum + r, 0) / workingRatings.length; + const stdDev = calculateStandardDeviation(workingRatings); + + console.log(` Mean: ${mean.toFixed(1)}`); + console.log(` Std Dev: ${stdDev.toFixed(1)}`); + + // Two PDGA exclusion rules: + // 1. More than 2.5 standard deviations below average + const stdDevCutoff = mean - 2.5 * stdDev; + // 2. More than 100 points below average + const hundredPointCutoff = mean - 100; + + console.log(` 2.5σ cutoff: ${stdDevCutoff.toFixed(1)}`); + console.log(` 100-point cutoff: ${hundredPointCutoff.toFixed(1)}`); + + const filteredByStdDev = workingRatings.filter(rating => rating >= stdDevCutoff); + const filteredBy100Points = workingRatings.filter(rating => rating >= hundredPointCutoff); + + // Apply both exclusion rules + const filteredRatings = workingRatings.filter(rating => + rating >= stdDevCutoff && rating >= hundredPointCutoff + ); + + const stdDevOutliers = workingRatings.filter(rating => rating < stdDevCutoff); + const hundredPointOutliers = workingRatings.filter(rating => rating < hundredPointCutoff && rating >= stdDevCutoff); + + if (stdDevOutliers.length > 0) { + console.log(` ❌ 2.5σ outliers removed: ${stdDevOutliers.length} rounds`); + stdDevOutliers.forEach(rating => { + const round = workingRounds.find(r => r.rating === rating); + console.log(` - ${rating} (${round.date.toDateString()}: ${round.competition})`); + }); + } + + if (hundredPointOutliers.length > 0) { + console.log(` ❌ 100-point outliers removed: ${hundredPointOutliers.length} rounds`); + hundredPointOutliers.forEach(rating => { + const round = workingRounds.find(r => r.rating === rating); + console.log(` - ${rating} (${round.date.toDateString()}: ${round.competition})`); + }); + } + + if (stdDevOutliers.length === 0 && hundredPointOutliers.length === 0) { + console.log(` ✅ No outliers detected`); + } + + // Keep filtered rounds only if we still have enough data + if (filteredRatings.length >= 4) { + workingRounds = workingRounds.filter(round => + round.rating >= stdDevCutoff && round.rating >= hundredPointCutoff + ); + workingRatings = filteredRatings; + console.log(` ✅ Using ${filteredRatings.length} rounds after outlier removal`); + } else { + console.log(` ⚠️ Too few rounds after outlier removal (${filteredRatings.length}), keeping all rounds`); + } + } else { + console.log(`\n⏭️ OUTLIER EXCLUSION SKIPPED (only ${workingRatings.length} rounds, need ≥7)`); + } + + // PDGA Rule: Most recent 25% of rounds get double weight if ≥9 rounds + console.log(`\n⚖️ WEIGHTING (Most recent 25% count double if ≥9 rounds):`); const weightedRatings = []; - if (sortedRatings.length >= 9) { - const recentCount = Math.round(sortedRatings.length * 0.25); + + if (workingRatings.length >= 9) { + const recentCount = Math.round(workingRatings.length * 0.25); + console.log(` ✅ Double-weighting most recent ${recentCount} rounds`); // Add all ratings once - weightedRatings.push(...sortedRatings); + weightedRatings.push(...workingRatings); // Add the most recent 25% again (double weight) for (let i = 0; i < recentCount; i++) { - weightedRatings.push(sortedRatings[i]); + weightedRatings.push(workingRatings[i]); + const round = workingRounds[i]; + console.log(` 2x weight: ${workingRatings[i]} (${round.date.toDateString()}: ${round.competition})`); } + + console.log(` 📊 Total values: ${workingRatings.length} + ${recentCount} double-weighted = ${weightedRatings.length}`); } else { - // If fewer than 9 rounds, no double weighting - weightedRatings.push(...sortedRatings); + console.log(` ➡️ No double weighting (${workingRatings.length} rounds, need ≥9)`); + weightedRatings.push(...workingRatings); } - // Calculate final average - const finalRating = weightedRatings.reduce((sum, r) => sum + r, 0) / weightedRatings.length; + // Calculate final rating + const sum = weightedRatings.reduce((sum, r) => sum + r, 0); + const average = sum / weightedRatings.length; + const finalRating = Math.round(average); - return Math.round(finalRating); + console.log(`\n🎯 FINAL CALCULATION:`); + console.log(` Sum: ${sum}`); + console.log(` Count: ${weightedRatings.length}`); + console.log(` Average: ${average.toFixed(1)}`); + console.log(` Final Rating: ${finalRating}`); + console.log('=== END PDGA CALCULATION ===\n'); + + return finalRating; } function calculateStandardDeviation(ratings) { @@ -395,7 +1156,166 @@ function calculateStandardDeviation(ratings) { return Math.sqrt(variance); } -async function getAllRatings(progressCallback = null) { +async function getAllRatingsFromDB(progressCallback = null) { + try { + const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8') + .split('\n') + .map(num => num.trim()) + .filter(num => num); + + const ratings = []; + const total = pdgaNumbers.length; + + for (let i = 0; i < pdgaNumbers.length; i++) { + const pdgaNumber = pdgaNumbers[i]; + console.log(`Loading PDGA ${pdgaNumber} from database... (${i + 1}/${total})`); + + if (progressCallback) { + progressCallback({ + current: i + 1, + total, + pdgaNumber, + status: 'loading' + }); + } + + try { + // Load from database only + const cachedPlayer = await getPlayerFromDB(pdgaNumber); + + let playerData; + if (cachedPlayer) { + // Calculate prediction from database + console.log(`Calculating prediction for PDGA ${pdgaNumber}...`); + const predictedRating = await getPredictedRatingFromDB(pdgaNumber); + + playerData = { + pdgaNumber: cachedPlayer.pdga_number, + name: cachedPlayer.name, + rating: cachedPlayer.current_rating, + ratingChange: cachedPlayer.rating_change, + predictedRating: predictedRating > 0 ? predictedRating : null + }; + } else { + // If not in database, create placeholder entry with PDGA number + playerData = { + pdgaNumber: parseInt(pdgaNumber), + name: `PDGA #${pdgaNumber}`, + rating: null, + ratingChange: null, + predictedRating: null + }; + } + + ratings.push(playerData); + + if (progressCallback) { + progressCallback({ + current: i + 1, + total, + pdgaNumber, + status: 'completed', + name: playerData.name + }); + } + } catch (error) { + console.error(`Failed to load PDGA ${pdgaNumber} from database:`, error.message); + const errorData = { + pdgaNumber: parseInt(pdgaNumber), + name: 'Database Error', + rating: null, + ratingChange: null, + predictedRating: null + }; + ratings.push(errorData); + + if (progressCallback) { + progressCallback({ + current: i + 1, + total, + pdgaNumber, + status: 'error', + name: 'Database Error' + }); + } + } + } + + return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0)); + } catch (error) { + console.error('Error reading PDGA numbers:', error); + return []; + } +} + +app.get('/', (req, res) => { + res.sendFile(path.join(__dirname, 'index.html')); +}); + +app.get('/api/ratings', async (req, res) => { + try { + const ratings = await getAllRatingsFromDB(); + res.json(ratings); + } catch (error) { + res.status(500).json({ error: 'Failed to fetch ratings' }); + } +}); + +app.get('/api/ratings/progress', (req, res) => { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Headers': 'Cache-Control' + }); + + const progressCallback = (progress) => { + res.write(`data: ${JSON.stringify(progress)}\n\n`); + }; + + getAllRatingsFromDB(progressCallback).then(ratings => { + res.write(`data: ${JSON.stringify({ status: 'complete', ratings })}\n\n`); + res.end(); + }).catch(error => { + res.write(`data: ${JSON.stringify({ status: 'error', error: error.message })}\n\n`); + res.end(); + }); + + req.on('close', () => { + res.end(); + }); +}); + +app.get('/api/load-all-players', (req, res) => { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Headers': 'Cache-Control' + }); + + const progressCallback = (progress) => { + res.write(`data: ${JSON.stringify(progress)}\n\n`); + }; + + // Use the original scraping function for bulk loading + getAllRatingsWithScraping(progressCallback).then(ratings => { + res.write(`data: ${JSON.stringify({ status: 'complete', ratings })}\n\n`); + res.end(); + }).catch(error => { + res.write(`data: ${JSON.stringify({ status: 'error', error: error.message })}\n\n`); + res.end(); + }); + + req.on('close', () => { + res.end(); + }); +}); + +// Original scraping function for bulk loading +async function getAllRatingsWithScraping(progressCallback = null) { try { const pdgaNumbers = fs.readFileSync('pdga-numbers.txt', 'utf-8') .split('\n') @@ -419,11 +1339,6 @@ async function getAllRatings(progressCallback = null) { } try { - // Check if data is cached BEFORE scraping - const cacheKey = `player-${pdgaNumber}`; - const cached = cache.get(cacheKey); - const wasFromCache = cached && Date.now() - cached.timestamp < CACHE_DURATION; - const playerData = await scrapePDGARating(pdgaNumber); ratings.push(playerData); @@ -437,14 +1352,12 @@ async function getAllRatings(progressCallback = null) { }); } - if (!wasFromCache) { - // Delay only for fresh scrapes to avoid overwhelming the server - await new Promise(resolve => setTimeout(resolve, 1000)); - } + // Always delay for bulk scraping to be respectful + await new Promise(resolve => setTimeout(resolve, 1000)); } catch (error) { console.error(`Failed to scrape PDGA ${pdgaNumber}:`, error.message); const errorData = { - pdgaNumber, + pdgaNumber: parseInt(pdgaNumber), name: 'Error', rating: 0, ratingChange: null, @@ -464,52 +1377,13 @@ async function getAllRatings(progressCallback = null) { } } - return ratings.sort((a, b) => b.rating - a.rating); + return ratings.sort((a, b) => (b.rating || 0) - (a.rating || 0)); } catch (error) { console.error('Error reading PDGA numbers:', error); return []; } } -app.get('/', (req, res) => { - res.sendFile(path.join(__dirname, 'index.html')); -}); - -app.get('/api/ratings', async (req, res) => { - try { - const ratings = await getAllRatings(); - res.json(ratings); - } catch (error) { - res.status(500).json({ error: 'Failed to fetch ratings' }); - } -}); - -app.get('/api/ratings/progress', (req, res) => { - res.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive', - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Headers': 'Cache-Control' - }); - - const progressCallback = (progress) => { - res.write(`data: ${JSON.stringify(progress)}\n\n`); - }; - - getAllRatings(progressCallback).then(ratings => { - res.write(`data: ${JSON.stringify({ status: 'complete', ratings })}\n\n`); - res.end(); - }).catch(error => { - res.write(`data: ${JSON.stringify({ status: 'error', error: error.message })}\n\n`); - res.end(); - }); - - req.on('close', () => { - res.end(); - }); -}); - async function fetchRatingHistory(pdgaNumber) { return new Promise((resolve, reject) => { const options = { @@ -523,6 +1397,8 @@ async function fetchRatingHistory(pdgaNumber) { timeout: 30000 }; + console.log(`Fetching rating history for PDGA #${pdgaNumber} from: https://www.pdga.com/player/${pdgaNumber}/history`); + const req = https.request(options, (res) => { let data = ''; res.on('data', (chunk) => { @@ -531,18 +1407,61 @@ async function fetchRatingHistory(pdgaNumber) { res.on('end', () => { if (res.statusCode === 200) { + console.log(`Rating history request successful for PDGA #${pdgaNumber}`); resolve(data); } else { - reject(new Error(`HTTP ${res.statusCode}`)); + // Log detailed error information for rating history + console.log(`Rating History Error for PDGA #${pdgaNumber}:`); + console.log(`Status: ${res.statusCode}`); + console.log('Response Headers:', JSON.stringify(res.headers, null, 2)); + + // Check for rate limiting headers + if (res.headers['retry-after']) { + console.log(`Retry-After: ${res.headers['retry-after']} seconds`); + } + if (res.headers['x-ratelimit-limit']) { + console.log(`Rate Limit: ${res.headers['x-ratelimit-limit']}`); + } + if (res.headers['x-ratelimit-remaining']) { + console.log(`Rate Limit Remaining: ${res.headers['x-ratelimit-remaining']}`); + } + + // Log partial response if available + if (data.length > 0) { + console.log(`Partial response received (${data.length} bytes):`, data.substring(0, 200)); + } + + const error = new Error(`HTTP ${res.statusCode} for rating history`); + error.statusCode = res.statusCode; + error.headers = res.headers; + reject(error); } }); }); req.on('error', (error) => { + console.log(`Rating history request error for PDGA #${pdgaNumber}:`, { + code: error.code, + message: error.message, + errno: error.errno, + syscall: error.syscall + }); + + if (error.code === 'ECONNRESET') { + console.log('Connection reset on rating history - likely rate limited by PDGA'); + } + if (error.code === 'ECONNREFUSED') { + console.log('Connection refused - PDGA server may be blocking requests'); + } + if (error.code === 'ETIMEDOUT') { + console.log('Request timed out - server may be overloaded'); + } + reject(error); }); req.on('timeout', () => { + console.log(`Rating history request timeout for PDGA #${pdgaNumber} after 30s`); req.destroy(); reject(new Error('Request timeout')); }); @@ -598,15 +1517,24 @@ function parseRatingHistory(html) { app.get('/api/rating-history/:pdgaNumber', async (req, res) => { try { const { pdgaNumber } = req.params; - const cacheKey = `history-${pdgaNumber}`; - const cached = cache.get(cacheKey); - // Check cache first (24 hour cache for rating history) - if (cached && Date.now() - cached.timestamp < CACHE_DURATION) { - console.log(`Using cached rating history for PDGA ${pdgaNumber}`); + // Check database first + const cachedHistory = await getRatingHistoryFromDB(pdgaNumber); + if (cachedHistory && cachedHistory.length > 0) { + console.log(`Using cached rating history from DB for PDGA ${pdgaNumber}`); + const formattedHistory = cachedHistory.map(row => ({ + date: row.date, + rating: row.rating, + displayDate: new Date(row.date).toLocaleDateString('en-US', { + day: '2-digit', + month: 'short', + year: 'numeric' + }) + })); + res.json({ pdgaNumber: parseInt(pdgaNumber), - history: cached.data + history: formattedHistory }); return; } @@ -615,11 +1543,13 @@ app.get('/api/rating-history/:pdgaNumber', async (req, res) => { const html = await fetchRatingHistory(pdgaNumber); const history = parseRatingHistory(html); - // Cache the result - cache.set(cacheKey, { - data: history, - timestamp: Date.now() - }); + // Save to database + try { + await saveRatingHistoryToDB(pdgaNumber, history); + console.log(`Saved rating history for PDGA ${pdgaNumber} to database`); + } catch (dbErr) { + console.error(`Failed to save rating history to database:`, dbErr.message); + } res.json({ pdgaNumber: parseInt(pdgaNumber), @@ -633,12 +1563,23 @@ app.get('/api/rating-history/:pdgaNumber', async (req, res) => { app.post('/api/clear-cache', (req, res) => { try { - const cacheSize = cache.size; - cache.clear(); - console.log(`Cache cleared - removed ${cacheSize} entries`); - res.json({ - success: true, - message: `Cache cleared - ${cacheSize} entries removed` + // Clear database cache by updating timestamps to force refresh + db.run('UPDATE players SET last_updated = datetime("now", "-25 hours"), last_round_update = NULL', (err) => { + if (err) { + console.error('Error clearing database cache:', err); + res.status(500).json({ error: 'Failed to clear database cache' }); + return; + } + + // Also clear legacy in-memory cache + const cacheSize = cache.size; + cache.clear(); + + console.log('Database cache cleared - all players will be refreshed on next request'); + res.json({ + success: true, + message: `Cache cleared - database and ${cacheSize} memory entries reset` + }); }); } catch (error) { console.error('Error clearing cache:', error); @@ -646,19 +1587,255 @@ app.post('/api/clear-cache', (req, res) => { } }); +// Individual player refresh endpoints +app.post('/api/refresh-player/:pdgaNumber', async (req, res) => { + try { + const { pdgaNumber } = req.params; + console.log(`Manually refreshing player data for PDGA ${pdgaNumber}`); + + // Force refresh by bypassing cache + const html = await fetchPlayerDataHTTP(pdgaNumber); + const playerData = parsePlayerData(html, pdgaNumber); + + // Save to database + await savePlayerToDB(playerData); + + res.json({ + success: true, + player: playerData + }); + } catch (error) { + console.error('Error refreshing player data:', error.message); + res.status(500).json({ error: 'Failed to refresh player data' }); + } +}); + +app.post('/api/refresh-rating-history/:pdgaNumber', async (req, res) => { + try { + const { pdgaNumber } = req.params; + console.log(`=== Manually refreshing rating history for PDGA ${pdgaNumber} ===`); + + const startTime = Date.now(); + const html = await fetchRatingHistory(pdgaNumber); + const fetchTime = Date.now() - startTime; + + console.log(`HTML fetch completed in ${fetchTime}ms, received ${html.length} bytes`); + + const parseStartTime = Date.now(); + const history = parseRatingHistory(html); + const parseTime = Date.now() - parseStartTime; + + console.log(`Parsing completed in ${parseTime}ms, found ${history.length} history entries`); + + if (history.length > 0) { + console.log('Sample history entries:', history.slice(0, 3)); + } else { + console.log('No history entries found. HTML sample:', html.substring(0, 500)); + } + + const dbStartTime = Date.now(); + await saveRatingHistoryToDB(pdgaNumber, history); + const dbTime = Date.now() - dbStartTime; + + console.log(`Database save completed in ${dbTime}ms`); + + const formattedHistory = history.map(entry => ({ + date: entry.date, + rating: entry.rating, + displayDate: entry.displayDate + })); + + console.log(`=== Rating history refresh completed for PDGA ${pdgaNumber} ===`); + + res.json({ + success: true, + history: formattedHistory + }); + } catch (error) { + console.error(`=== Error refreshing rating history for PDGA ${pdgaNumber} ===`); + console.error('Error type:', error.constructor.name); + console.error('Error message:', error.message); + console.error('Error code:', error.code); + console.error('Status code:', error.statusCode); + if (error.stack) { + console.error('Stack trace:', error.stack); + } + + res.status(500).json({ + error: 'Failed to refresh rating history', + details: error.message, + code: error.code + }); + } +}); + +app.post('/api/refresh-round-history/:pdgaNumber', async (req, res) => { + let browser = null; + const { pdgaNumber } = req.params; + try { + + // Check when we last updated rounds for this player + const lastRoundUpdate = await getLastRoundUpdateDate(pdgaNumber); + const sinceDate = lastRoundUpdate ? new Date(lastRoundUpdate) : null; + const isIncremental = !!sinceDate; + + console.log(`${isIncremental ? 'Incrementally updating' : 'Fully refreshing'} round history for PDGA ${pdgaNumber}${sinceDate ? ` since ${sinceDate.toDateString()}` : ''}`); + + try { + browser = await puppeteer.launch({ + headless: "new", + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-accelerated-2d-canvas', + '--no-first-run', + '--no-zygote', + '--disable-gpu' + ] + }); + } catch (launchError) { + // Fallback with minimal options + browser = await puppeteer.launch({ + headless: true, + args: ['--no-sandbox', '--disable-dev-shm-usage'] + }); + } + + // Step 1: Get official rating history + let officialHistory; + try { + officialHistory = await getOfficialRatingHistory(browser, pdgaNumber); + if (officialHistory.length > 0) { + await saveRatingHistoryToDB(pdgaNumber, officialHistory); + } + } catch (historyError) { + console.error('Failed to fetch official history:', historyError.message); + officialHistory = []; + } + + // Step 2: Get tournament round details + let tournamentRounds = []; + try { + tournamentRounds = await getPlayerTournamentDetails(browser, pdgaNumber); + if (tournamentRounds.length > 0) { + await saveRoundHistoryToDB(pdgaNumber, tournamentRounds, false); + } + } catch (detailsError) { + console.error('Failed to fetch tournament details:', detailsError.message); + tournamentRounds = []; + } + + await browser.close(); + browser = null; + + // Update timestamp and calculate prediction + if (tournamentRounds.length > 0) { + await updateLastRoundUpdateDate(pdgaNumber); + } + + const allRounds = await getRoundHistoryFromDB(pdgaNumber); + const allRoundsForPrediction = allRounds.map(round => ({ + rating: round.rating, + date: new Date(round.date), + competition: round.competition_name + })); + + const predictedRating = calculatePredictedRating(allRoundsForPrediction); + + res.json({ + success: true, + predictedRating, + totalRounds: allRoundsForPrediction.length, + officialRounds: officialHistory.length, + newRounds: tournamentRounds.length, + wasIncremental: isIncremental + }); + } catch (error) { + console.error(`=== Error refreshing round history for PDGA ${pdgaNumber} ===`); + console.error('Error type:', error.constructor.name); + console.error('Error message:', error.message); + console.error('Error code:', error.code); + console.error('Error name:', error.name); + + // Log all error properties for debugging + console.error('Full error object:', JSON.stringify(error, Object.getOwnPropertyNames(error), 2)); + + // Check if this is a puppeteer-specific error + if (error.name) { + console.error(`Specific error name: ${error.name}`); + } + + // Log timing information + const currentTime = new Date().toISOString(); + console.error(`Error occurred at: ${currentTime}`); + + // Check if we have browser information + if (browser) { + console.error('Browser was active when error occurred'); + } else { + console.error('No active browser session'); + } + + if (error.message.includes('socket hang up')) { + console.error('🔌 Socket hang up - likely rate limited by PDGA'); + console.error('💡 Try waiting a few minutes before attempting again'); + console.error('🔍 This usually happens when PDGA blocks too many rapid requests'); + } + + if (error.message.includes('Navigation timeout')) { + console.error('⏰ Navigation timeout - PDGA pages loading slowly'); + console.error('💡 Try reducing the number of tournaments scraped'); + } + + if (error.message.includes('net::ERR_CONNECTION_RESET')) { + console.error('🚫 Connection reset by PDGA server'); + console.error('💡 PDGA may be blocking or rate limiting requests'); + } + + if (error.stack) { + console.error('Full stack trace:'); + console.error(error.stack); + } else { + console.error('No stack trace available'); + } + + if (browser) { + try { + await browser.close(); + console.log('Browser closed successfully'); + } catch (closeError) { + console.error('Error closing browser:', closeError.message); + } + } + + res.status(500).json({ + error: 'Failed to refresh round history', + details: error.message, + errorType: error.constructor.name, + errorName: error.name, + timestamp: new Date().toISOString(), + suggestion: error.message.includes('socket hang up') ? + 'Rate limited by PDGA - try again in a few minutes. This happens when too many requests are made too quickly.' : + error.message.includes('timeout') ? + 'PDGA pages are loading slowly - try again later when PDGA servers are less busy.' : + 'Tournament scraping failed - check server logs for detailed error information' + }); + } +}); + app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => { let browser = null; try { const { pdgaNumber } = req.params; - const cacheKey = `predicted-${pdgaNumber}`; - const cached = cache.get(cacheKey); - // Check cache first (24 hour cache for predicted ratings) - if (cached && Date.now() - cached.timestamp < CACHE_DURATION) { - console.log(`Using cached predicted rating for PDGA ${pdgaNumber}`); + // Check database first for cached round history + const cachedPrediction = await getPredictedRatingFromDB(pdgaNumber); + if (cachedPrediction > 0) { + console.log(`Using cached round history for PDGA ${pdgaNumber} prediction`); res.json({ pdgaNumber: parseInt(pdgaNumber), - predictedRating: cached.data + predictedRating: cachedPrediction }); return; } @@ -677,16 +1854,30 @@ app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => { }); console.log(`Calculating predicted rating for PDGA ${pdgaNumber}...`); - const predictedRating = await getPredictedRating(browser, pdgaNumber); + + // Check for incremental update + const lastRoundUpdate = await getLastRoundUpdateDate(pdgaNumber); + const sinceDate = lastRoundUpdate ? new Date(lastRoundUpdate) : null; + const isIncremental = !!sinceDate; + + // Get round ratings and calculate prediction + const newRoundRatings = await getPlayerCompetitionRatings(browser, pdgaNumber, sinceDate); await browser.close(); browser = null; - // Cache the result - cache.set(cacheKey, { - data: predictedRating, - timestamp: Date.now() - }); + // Save new round history to database + await saveRoundHistoryToDB(pdgaNumber, newRoundRatings, isIncremental); + + // Get all rounds for prediction calculation + const allRounds = await getRoundHistoryFromDB(pdgaNumber); + const roundRatings = allRounds.map(round => ({ + rating: round.rating, + date: new Date(round.date), + competition: round.competition_name + })); + + const predictedRating = calculatePredictedRating(roundRatings); res.json({ pdgaNumber: parseInt(pdgaNumber), @@ -705,6 +1896,45 @@ app.post('/api/predicted-rating/:pdgaNumber', async (req, res) => { } }); -app.listen(PORT, () => { - console.log(`PDGA Ratings app running on http://localhost:${PORT}`); +// Test function to probe PDGA rate limiting +async function testPDGARateLimit() { + console.log('Testing PDGA rate limiting behavior...'); + + const testPdgaNumbers = ['60954', '178737', '251092']; // First few from our list + const requestTimes = []; + + for (let i = 0; i < testPdgaNumbers.length; i++) { + const startTime = Date.now(); + try { + console.log(`Test request ${i + 1}: PDGA #${testPdgaNumbers[i]}`); + await fetchPlayerDataHTTP(testPdgaNumbers[i]); + const endTime = Date.now(); + requestTimes.push(endTime - startTime); + console.log(`Request ${i + 1} completed in ${endTime - startTime}ms`); + } catch (error) { + const endTime = Date.now(); + requestTimes.push(endTime - startTime); + console.log(`Request ${i + 1} failed after ${endTime - startTime}ms:`, error.message); + } + + // Small delay between test requests + if (i < testPdgaNumbers.length - 1) { + await new Promise(resolve => setTimeout(resolve, 500)); + } + } + + console.log('Rate limit test completed. Request times:', requestTimes); +} + +// Uncomment the line below to run rate limit test on startup +// testPDGARateLimit(); + +// Initialize database and start server +initializeDatabase().then(() => { + app.listen(PORT, () => { + console.log(`PDGA Ratings app running on http://localhost:${PORT}`); + }); +}).catch(err => { + console.error('Failed to initialize database:', err); + process.exit(1); }); \ No newline at end of file