diff --git a/scrape-multivision.js b/scrape-multivision.js
index 1afc1bb..73ade7f 100644
--- a/scrape-multivision.js
+++ b/scrape-multivision.js
@@ -20,6 +20,25 @@ const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID;
const excludedNodes = ['962', '963', '895', '894', '893', '964'];
+async function deleteOldScrapes(postsCollectionId) {
+ console.log('Deleting all poorly scraped multivision posts...');
+ let cursor = null;
+ let deletedCount = 0;
+ while (true) {
+ const queries = [Query.limit(100)];
+ if (cursor) queries.push(Query.cursorAfter(cursor));
+ const res = await databases.listDocuments(DB_ID, postsCollectionId, queries);
+ const toDelete = res.documents.filter(p => p.slug && (p.slug.startsWith('multivision-amj-') || p.slug.startsWith('multivision-final-') || p.slug.startsWith('multivision-marzo-')));
+ for (const d of toDelete) {
+ await databases.deleteDocument(DB_ID, postsCollectionId, d.$id);
+ deletedCount++;
+ }
+ if (res.documents.length < 100) break;
+ cursor = res.documents[res.documents.length - 1].$id;
+ }
+ console.log(`Deleted ${deletedCount} old posts.`);
+}
+
async function scrapeNode(url) {
try {
const res = await axios.get(url);
@@ -39,17 +58,18 @@ async function scrapeNode(url) {
if (!dateObj) return null;
- // We only want April (3), May (4), June (5) 2026
- const validMonths = [3, 4, 5];
+ // We only want March (2), April (3), May (4), June (5) 2026
+ const validMonths = [2, 3, 4, 5];
if (!validMonths.includes(dateObj.getMonth()) || dateObj.getFullYear() !== 2026) {
- return { skipped: true, reason: 'Not in April, May, or June 2026' };
+ return { skipped: true, reason: 'Not in Mar, Apr, May, or Jun 2026' };
}
// Extract image
let image = null;
$('img').each((i, el) => {
const src = $(el).attr('src');
- if (src && (src.includes('public/imagenes') || src.includes('styles/')) && !src.includes('logo')) {
+ // Changed filter to match Multivision's actual image paths
+ if (src && src.includes('sites/default/files') && !src.includes('logo')) {
image = src.startsWith('http') ? src : 'https://www.multivisiondecolombia.com' + src;
return false;
}
@@ -57,26 +77,49 @@ async function scrapeNode(url) {
// Extract content
let contentHtml = '';
- const bodyField = $('.field--name-body');
- if (bodyField.length > 0) {
- contentHtml = bodyField.html();
- } else {
- contentHtml = $('article').html() || '';
- }
-
- const $content = cheerio.load(contentHtml || '');
- $content('*').each((i, el) => {
- if (!['p', 'ul', 'li', 'strong', 'em', 'b', 'i'].includes(el.tagName.toLowerCase())) {
- if (el.tagName.toLowerCase() === 'br') {
- $content(el).replaceWith('
');
- } else {
- $content(el).replaceWith($content(el).text());
- }
+ let maxLen = 0;
+
+ // Find the block with the most text
+ $('.field--name-body, .field--name-field-resumen-de-la-noticia').each((i, el) => {
+ const text = $(el).text().trim();
+ if (text.length > maxLen) {
+ maxLen = text.length;
+ contentHtml = $(el).html();
}
});
- const finalContent = $content.html() || '