feat: Add similarity checking and support for April, May, June in Multivision scraper
This commit is contained in:
parent
46102324db
commit
f444425e35
403
package-lock.json
generated
403
package-lock.json
generated
@ -22,15 +22,18 @@
|
||||
"@types/dompurify": "^3.0.5",
|
||||
"@types/video.js": "^7.3.58",
|
||||
"appwrite": "^22.4.1",
|
||||
"axios": "^1.13.6",
|
||||
"axios": "^1.18.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"clsx": "^2.1.1",
|
||||
"dompurify": "^3.3.3",
|
||||
"dotenv": "^17.4.2",
|
||||
"isomorphic-dompurify": "^2.20.0",
|
||||
"lucide-react": "^0.577.0",
|
||||
"next": "16.1.6",
|
||||
"node-appwrite": "^22.0.0",
|
||||
"node-appwrite": "^22.1.3",
|
||||
"react": "19.2.3",
|
||||
"react-dom": "19.2.3",
|
||||
"string-similarity": "^4.0.4",
|
||||
"tailwind-merge": "^3.5.0",
|
||||
"video.js": "^8.23.7",
|
||||
"zod": "^4.3.6",
|
||||
@ -3408,14 +3411,40 @@
|
||||
}
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "1.13.6",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-1.13.6.tgz",
|
||||
"integrity": "sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==",
|
||||
"version": "1.18.0",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-1.18.0.tgz",
|
||||
"integrity": "sha512-E32NzpYKp++W7XRe52rHiXV2ehxmh3wbdgO7MHeFM+vqxLBYHzt0ElkiImtOBxtOmyp0yoC8C6uESVV84Y2/hw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"follow-redirects": "^1.15.11",
|
||||
"follow-redirects": "^1.16.0",
|
||||
"form-data": "^4.0.5",
|
||||
"proxy-from-env": "^1.1.0"
|
||||
"https-proxy-agent": "^5.0.1",
|
||||
"proxy-from-env": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/axios/node_modules/agent-base": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
|
||||
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/axios/node_modules/https-proxy-agent": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
|
||||
"integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"agent-base": "6",
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/axobject-query": {
|
||||
@ -3465,6 +3494,12 @@
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/boolbase": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
|
||||
"integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "1.1.12",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
|
||||
@ -3619,6 +3654,81 @@
|
||||
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/cheerio": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz",
|
||||
"integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cheerio-select": "^2.1.0",
|
||||
"dom-serializer": "^2.0.0",
|
||||
"domhandler": "^5.0.3",
|
||||
"domutils": "^3.2.2",
|
||||
"encoding-sniffer": "^0.2.1",
|
||||
"htmlparser2": "^10.1.0",
|
||||
"parse5": "^7.3.0",
|
||||
"parse5-htmlparser2-tree-adapter": "^7.1.0",
|
||||
"parse5-parser-stream": "^7.1.2",
|
||||
"undici": "^7.19.0",
|
||||
"whatwg-mimetype": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.18.1"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/cheeriojs/cheerio?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/cheerio-select": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
|
||||
"integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"boolbase": "^1.0.0",
|
||||
"css-select": "^5.1.0",
|
||||
"css-what": "^6.1.0",
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.3",
|
||||
"domutils": "^3.0.1"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/cheerio/node_modules/entities": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
|
||||
"integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/cheerio/node_modules/parse5": {
|
||||
"version": "7.3.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
|
||||
"integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"entities": "^6.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/cheerio/node_modules/whatwg-mimetype": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
|
||||
"integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/client-only": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
|
||||
@ -3701,6 +3811,22 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/css-select": {
|
||||
"version": "5.2.2",
|
||||
"resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz",
|
||||
"integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"boolbase": "^1.0.0",
|
||||
"css-what": "^6.1.0",
|
||||
"domhandler": "^5.0.2",
|
||||
"domutils": "^3.0.1",
|
||||
"nth-check": "^2.0.1"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/css-tree": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz",
|
||||
@ -3714,6 +3840,18 @@
|
||||
"node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/css-what": {
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz",
|
||||
"integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/cssstyle": {
|
||||
"version": "6.2.0",
|
||||
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-6.2.0.tgz",
|
||||
@ -3916,11 +4054,52 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/dom-serializer": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
|
||||
"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.2",
|
||||
"entities": "^4.2.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/dom-walk": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.2.tgz",
|
||||
"integrity": "sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w=="
|
||||
},
|
||||
"node_modules/domelementtype": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
|
||||
"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
],
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/domhandler": {
|
||||
"version": "5.0.3",
|
||||
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
|
||||
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 4"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/domhandler?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/dompurify": {
|
||||
"version": "3.3.3",
|
||||
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.3.tgz",
|
||||
@ -3930,6 +4109,32 @@
|
||||
"@types/trusted-types": "^2.0.7"
|
||||
}
|
||||
},
|
||||
"node_modules/domutils": {
|
||||
"version": "3.2.2",
|
||||
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
|
||||
"integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"dom-serializer": "^2.0.0",
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.3"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/domutils?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/dotenv": {
|
||||
"version": "17.4.2",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz",
|
||||
"integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://dotenvx.com"
|
||||
}
|
||||
},
|
||||
"node_modules/dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
@ -3958,6 +4163,19 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/encoding-sniffer": {
|
||||
"version": "0.2.1",
|
||||
"resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz",
|
||||
"integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"iconv-lite": "^0.6.3",
|
||||
"whatwg-encoding": "^3.1.1"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/encoding-sniffer?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/enhanced-resolve": {
|
||||
"version": "5.20.1",
|
||||
"resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.20.1.tgz",
|
||||
@ -4745,9 +4963,9 @@
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/follow-redirects": {
|
||||
"version": "1.15.11",
|
||||
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz",
|
||||
"integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==",
|
||||
"version": "1.16.0",
|
||||
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
|
||||
"integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "individual",
|
||||
@ -5130,6 +5348,37 @@
|
||||
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/htmlparser2": {
|
||||
"version": "10.1.0",
|
||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
|
||||
"integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
|
||||
"funding": [
|
||||
"https://github.com/fb55/htmlparser2?sponsor=1",
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.3",
|
||||
"domutils": "^3.2.2",
|
||||
"entities": "^7.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/htmlparser2/node_modules/entities": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",
|
||||
"integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/http-proxy-agent": {
|
||||
"version": "7.0.2",
|
||||
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
|
||||
@ -5156,6 +5405,18 @@
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/iconv-lite": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
|
||||
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"safer-buffer": ">= 2.1.2 < 3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ignore": {
|
||||
"version": "5.3.2",
|
||||
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
|
||||
@ -6535,6 +6796,18 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/nth-check": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
|
||||
"integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"boolbase": "^1.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/nth-check?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/object-assign": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
|
||||
@ -6757,6 +7030,79 @@
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5-htmlparser2-tree-adapter": {
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
|
||||
"integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domhandler": "^5.0.3",
|
||||
"parse5": "^7.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5-htmlparser2-tree-adapter/node_modules/entities": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
|
||||
"integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5-htmlparser2-tree-adapter/node_modules/parse5": {
|
||||
"version": "7.3.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
|
||||
"integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"entities": "^6.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5-parser-stream": {
|
||||
"version": "7.1.2",
|
||||
"resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
|
||||
"integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"parse5": "^7.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5-parser-stream/node_modules/entities": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
|
||||
"integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5-parser-stream/node_modules/parse5": {
|
||||
"version": "7.3.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
|
||||
"integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"entities": "^6.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5/node_modules/entities": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
|
||||
@ -7100,10 +7446,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/proxy-from-env": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
|
||||
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
|
||||
"license": "MIT"
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz",
|
||||
"integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/punycode": {
|
||||
"version": "2.3.1",
|
||||
@ -7362,6 +7711,12 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/saxes": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",
|
||||
@ -7626,6 +7981,13 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/string-similarity": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/string-similarity/-/string-similarity-4.0.4.tgz",
|
||||
"integrity": "sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==",
|
||||
"deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/string.prototype.includes": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/string.prototype.includes/-/string.prototype.includes-2.0.1.tgz",
|
||||
@ -8329,6 +8691,19 @@
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-encoding": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
|
||||
"integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
|
||||
"deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"iconv-lite": "0.6.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-mimetype": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz",
|
||||
|
||||
@ -24,15 +24,18 @@
|
||||
"@types/dompurify": "^3.0.5",
|
||||
"@types/video.js": "^7.3.58",
|
||||
"appwrite": "^22.4.1",
|
||||
"axios": "^1.13.6",
|
||||
"axios": "^1.18.0",
|
||||
"cheerio": "^1.2.0",
|
||||
"clsx": "^2.1.1",
|
||||
"dompurify": "^3.3.3",
|
||||
"dotenv": "^17.4.2",
|
||||
"isomorphic-dompurify": "^2.20.0",
|
||||
"lucide-react": "^0.577.0",
|
||||
"next": "16.1.6",
|
||||
"node-appwrite": "^22.0.0",
|
||||
"node-appwrite": "^22.1.3",
|
||||
"react": "19.2.3",
|
||||
"react-dom": "19.2.3",
|
||||
"string-similarity": "^4.0.4",
|
||||
"tailwind-merge": "^3.5.0",
|
||||
"video.js": "^8.23.7",
|
||||
"zod": "^4.3.6",
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
const axios = require('axios');
|
||||
const fs = require('fs');
|
||||
const cheerio = require('cheerio');
|
||||
const stringSimilarity = require('string-similarity');
|
||||
const { Client, Databases, Storage, ID, Query } = require('node-appwrite');
|
||||
require('dotenv').config({ path: '.env.local' });
|
||||
|
||||
@ -19,16 +20,6 @@ const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID;
|
||||
|
||||
const excludedNodes = ['962', '963', '895', '894', '893', '964'];
|
||||
|
||||
async function deleteOldScrapes(postsCollectionId) {
|
||||
console.log('Deleting poorly scraped posts...');
|
||||
const res = await databases.listDocuments(DB_ID, postsCollectionId, [Query.limit(100)]);
|
||||
const toDelete = res.documents.filter(p => p.slug && p.slug.startsWith('multivision-'));
|
||||
for (const d of toDelete) {
|
||||
await databases.deleteDocument(DB_ID, postsCollectionId, d.$id);
|
||||
console.log('Deleted:', d.slug);
|
||||
}
|
||||
}
|
||||
|
||||
async function scrapeNode(url) {
|
||||
try {
|
||||
const res = await axios.get(url);
|
||||
@ -39,20 +30,19 @@ async function scrapeNode(url) {
|
||||
// Extract date
|
||||
let dateObj = null;
|
||||
const timeText = $('span').filter((i, el) => $(el).text().match(/\d{2}\/\d{2}\/\d{4} - \d{2}:\d{2}/)).first().text();
|
||||
// e.g. "Mar, 31/03/2026 - 10:16"
|
||||
if (timeText) {
|
||||
const match = timeText.match(/(\d{2})\/(\d{2})\/(\d{4}) - (\d{2}:\d{2})/);
|
||||
if (match) {
|
||||
// DD/MM/YYYY
|
||||
dateObj = new Date(`${match[3]}-${match[2]}-${match[1]}T${match[4]}:00.000Z`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!dateObj) return null; // Skip if no date found
|
||||
if (!dateObj) return null;
|
||||
|
||||
// We only want March 2026
|
||||
if (dateObj.getMonth() !== 2 || dateObj.getFullYear() !== 2026) {
|
||||
return { skipped: true, reason: 'Not in March 2026' };
|
||||
// We only want April (3), May (4), June (5) 2026
|
||||
const validMonths = [3, 4, 5];
|
||||
if (!validMonths.includes(dateObj.getMonth()) || dateObj.getFullYear() !== 2026) {
|
||||
return { skipped: true, reason: 'Not in April, May, or June 2026' };
|
||||
}
|
||||
|
||||
// Extract image
|
||||
@ -61,7 +51,7 @@ async function scrapeNode(url) {
|
||||
const src = $(el).attr('src');
|
||||
if (src && (src.includes('public/imagenes') || src.includes('styles/')) && !src.includes('logo')) {
|
||||
image = src.startsWith('http') ? src : 'https://www.multivisiondecolombia.com' + src;
|
||||
return false; // break
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
@ -71,11 +61,9 @@ async function scrapeNode(url) {
|
||||
if (bodyField.length > 0) {
|
||||
contentHtml = bodyField.html();
|
||||
} else {
|
||||
// fallback
|
||||
contentHtml = $('article').html() || '';
|
||||
}
|
||||
|
||||
// Clean content: Keep only p, ul, li, strong, em, b, i
|
||||
const $content = cheerio.load(contentHtml || '');
|
||||
$content('*').each((i, el) => {
|
||||
if (!['p', 'ul', 'li', 'strong', 'em', 'b', 'i'].includes(el.tagName.toLowerCase())) {
|
||||
@ -107,13 +95,25 @@ async function run() {
|
||||
const typeRes = await databases.listDocuments(DB_ID, pubTypesCollectionId, [Query.equal('codigo', 'NOTICIA')]);
|
||||
const typeId = typeRes.documents.length > 0 ? typeRes.documents[0].$id : null;
|
||||
|
||||
await deleteOldScrapes(postsCollectionId);
|
||||
console.log('Fetching existing titles from Appwrite to prevent duplicates...');
|
||||
let allExistingTitles = [];
|
||||
let cursor = null;
|
||||
while (true) {
|
||||
const queries = [Query.limit(100), Query.select(['titulo'])];
|
||||
if (cursor) queries.push(Query.cursorAfter(cursor));
|
||||
const existingRes = await databases.listDocuments(DB_ID, postsCollectionId, queries);
|
||||
|
||||
existingRes.documents.forEach(d => allExistingTitles.push(d.titulo));
|
||||
|
||||
if (existingRes.documents.length < 100) break;
|
||||
cursor = existingRes.documents[existingRes.documents.length - 1].$id;
|
||||
}
|
||||
console.log(`Found ${allExistingTitles.length} existing posts.`);
|
||||
|
||||
console.log('Fetching news list from Multivision...');
|
||||
console.log('Fetching news list from Multivision (Pages 0-15)...');
|
||||
const allNodes = new Set();
|
||||
|
||||
// Loop through some pages (0 to 5)
|
||||
for (let page = 0; page <= 5; page++) {
|
||||
for (let page = 0; page <= 15; page++) {
|
||||
try {
|
||||
const listUrl = `https://www.multivisiondecolombia.com/noticias?page=${page}`;
|
||||
const res = await axios.get(listUrl);
|
||||
@ -149,9 +149,27 @@ async function run() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check similarity
|
||||
let isDuplicate = false;
|
||||
let highestSimilarity = 0;
|
||||
let mostSimilarTitle = '';
|
||||
|
||||
for (const exTitle of allExistingTitles) {
|
||||
if (!exTitle) continue;
|
||||
const sim = stringSimilarity.compareTwoStrings(data.title.toLowerCase(), exTitle.toLowerCase());
|
||||
if (sim > highestSimilarity) {
|
||||
highestSimilarity = sim;
|
||||
mostSimilarTitle = exTitle;
|
||||
}
|
||||
}
|
||||
|
||||
if (highestSimilarity > 0.60) {
|
||||
console.log(`Skipped due to similarity (${(highestSimilarity*100).toFixed(1)}%) with: "${mostSimilarTitle}"`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log('Extracted Title:', data.title);
|
||||
console.log('Extracted Date:', data.date);
|
||||
console.log('Extracted Image:', data.image);
|
||||
|
||||
let mediaId = null;
|
||||
if (data.image) {
|
||||
@ -178,7 +196,7 @@ async function run() {
|
||||
|
||||
const doc = {
|
||||
titulo: data.title.substring(0, 100),
|
||||
slug: `multivision-marzo-${Date.now()}-${count}`,
|
||||
slug: `multivision-amj-${Date.now()}-${count}`,
|
||||
tipo_publicacion_id: typeId,
|
||||
estado: 'Publicado',
|
||||
fecha_publicacion: data.date,
|
||||
@ -188,9 +206,11 @@ async function run() {
|
||||
|
||||
const res = await databases.createDocument(DB_ID, postsCollectionId, ID.unique(), doc);
|
||||
console.log('Published! ID:', res.$id);
|
||||
// Add to existing to prevent duplicates within this run
|
||||
allExistingTitles.push(data.title);
|
||||
count++;
|
||||
}
|
||||
console.log(`\nAll done! Successfully published ${count} news articles from March.`);
|
||||
console.log(`\nAll done! Successfully published ${count} unique news articles from April, May, and June.`);
|
||||
} catch(e) {
|
||||
console.error(e);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user