fix: Correctly extract content body, map images, and auto-generate SEO metadata

scrape-multivision.js

@@ -20,6 +20,25 @@ const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID;
 
 const excludedNodes = ['962', '963', '895', '894', '893', '964'];
 
+async function deleteOldScrapes(postsCollectionId) {
+    console.log('Deleting all poorly scraped multivision posts...');
+    let cursor = null;
+    let deletedCount = 0;
+    while (true) {
+        const queries = [Query.limit(100)];
+        if (cursor) queries.push(Query.cursorAfter(cursor));
+        const res = await databases.listDocuments(DB_ID, postsCollectionId, queries);
+        const toDelete = res.documents.filter(p => p.slug && (p.slug.startsWith('multivision-amj-') || p.slug.startsWith('multivision-final-') || p.slug.startsWith('multivision-marzo-')));
+        for (const d of toDelete) {
+            await databases.deleteDocument(DB_ID, postsCollectionId, d.$id);
+            deletedCount++;
+        }
+        if (res.documents.length < 100) break;
+        cursor = res.documents[res.documents.length - 1].$id;
+    }
+    console.log(`Deleted ${deletedCount} old posts.`);
+}
+
 async function scrapeNode(url) {
     try {
         const res = await axios.get(url);
@@ -39,17 +58,18 @@ async function scrapeNode(url) {
 
         if (!dateObj) return null;
 
-        // We only want April (3), May (4), June (5) 2026
+        // We only want March (2), April (3), May (4), June (5) 2026
-        const validMonths = [3, 4, 5];
+        const validMonths = [2, 3, 4, 5];
         if (!validMonths.includes(dateObj.getMonth()) || dateObj.getFullYear() !== 2026) {
-            return { skipped: true, reason: 'Not in April, May, or June 2026' };
+            return { skipped: true, reason: 'Not in Mar, Apr, May, or Jun 2026' };
         }
 
         // Extract image
         let image = null;
         $('img').each((i, el) => {
             const src = $(el).attr('src');
-            if (src && (src.includes('public/imagenes') || src.includes('styles/')) && !src.includes('logo')) {
+            // Changed filter to match Multivision's actual image paths
+            if (src && src.includes('sites/default/files') && !src.includes('logo')) {
                 image = src.startsWith('http') ? src : 'https://www.multivisiondecolombia.com' + src;
                 return false; 
             }
@@ -57,26 +77,49 @@ async function scrapeNode(url) {
 
         // Extract content
         let contentHtml = '';
-        const bodyField = $('.field--name-body');
+        let maxLen = 0;
-        if (bodyField.length > 0) {
+        
-            contentHtml = bodyField.html();
+        // Find the block with the most text
-        } else {
+        $('.field--name-body, .field--name-field-resumen-de-la-noticia').each((i, el) => {
-            contentHtml = $('article').html() || '';
+            const text = $(el).text().trim();
-        }
+            if (text.length > maxLen) {
-
+                maxLen = text.length;
-        const $content = cheerio.load(contentHtml || '');
+                contentHtml = $(el).html();
-        $content('*').each((i, el) => {
-            if (!['p', 'ul', 'li', 'strong', 'em', 'b', 'i'].includes(el.tagName.toLowerCase())) {
-                if (el.tagName.toLowerCase() === 'br') {
-                    $content(el).replaceWith('<br/>');
-                } else {
-                    $content(el).replaceWith($content(el).text());
-                }
             }
         });
-        const finalContent = $content.html() || '<p></p>';
+        
+        if (!contentHtml || maxLen < 100) {
+            // Fallback to finding the longest div if both fail
+            maxLen = 0;
+            $('div').each((i, el) => {
+                const text = $(el).text().trim();
+                if (text.length > maxLen && $(el).children().length < 5) {
+                    maxLen = text.length;
+                    contentHtml = $(el).html();
+                }
+            });
+        }
 
-        return { title, image, date: dateObj.toISOString(), content: finalContent };
+        // Clean content just a bit
+        let cleanContent = (contentHtml || '<p></p>').trim();
+        
+        // Auto-generate fields
+        const plainText = cheerio.load(cleanContent).text().replace(/\s+/g, ' ').trim();
+        let resumen = plainText.substring(0, 150);
+        if (plainText.length > 150) resumen += '...';
+        
+        // Keywords
+        const words = plainText.split(' ').filter(w => w.length > 5).slice(0, 8);
+        const keywords = words.join(', ').replace(/[^a-zA-ZáéíóúÁÉÍÓÚñÑ, ]/g, '');
+
+        return { 
+            title, 
+            image, 
+            date: dateObj.toISOString(), 
+            content: cleanContent,
+            resumen,
+            keywords
+        };
     } catch (e) {
         console.error('Error scraping node', url, e.message);
         return null;
@@ -95,25 +138,32 @@ async function run() {
         const typeRes = await databases.listDocuments(DB_ID, pubTypesCollectionId, [Query.equal('codigo', 'NOTICIA')]);
         const typeId = typeRes.documents.length > 0 ? typeRes.documents[0].$id : null;
 
+        await deleteOldScrapes(postsCollectionId);
+
         console.log('Fetching existing titles from Appwrite to prevent duplicates...');
         let allExistingTitles = [];
         let cursor = null;
         while (true) {
-            const queries = [Query.limit(100), Query.select(['titulo'])];
+            const queries = [Query.limit(100), Query.select(['titulo', 'slug'])];
             if (cursor) queries.push(Query.cursorAfter(cursor));
             const existingRes = await databases.listDocuments(DB_ID, postsCollectionId, queries);
             
-            existingRes.documents.forEach(d => allExistingTitles.push(d.titulo));
+            existingRes.documents.forEach(d => {
+                // Don't add multivision ones since we just deleted them or are replacing them
+                if (!d.slug || !d.slug.startsWith('multivision-')) {
+                    allExistingTitles.push(d.titulo);
+                }
+            });
             
             if (existingRes.documents.length < 100) break;
             cursor = existingRes.documents[existingRes.documents.length - 1].$id;
         }
-        console.log(`Found ${allExistingTitles.length} existing posts.`);
+        console.log(`Found ${allExistingTitles.length} existing native posts.`);
 
-        console.log('Fetching news list from Multivision (Pages 0-15)...');
+        console.log('Fetching news list from Multivision (Pages 0-25)...');
         const allNodes = new Set();
         
-        for (let page = 0; page <= 15; page++) {
+        for (let page = 0; page <= 25; page++) {
             try {
                 const listUrl = `https://www.multivisiondecolombia.com/noticias?page=${page}`;
                 const res = await axios.get(listUrl);
@@ -150,7 +200,6 @@ async function run() {
             }
 
             // Check similarity
-            let isDuplicate = false;
             let highestSimilarity = 0;
             let mostSimilarTitle = '';
 
@@ -196,11 +245,14 @@ async function run() {
 
             const doc = {
                 titulo: data.title.substring(0, 100),
-                slug: `multivision-amj-${Date.now()}-${count}`,
+                slug: `multivision-final-${Date.now()}-${count}`,
                 tipo_publicacion_id: typeId,
                 estado: 'Publicado',
                 fecha_publicacion: data.date,
                 contenido: data.content,
+                resumen_redes: data.resumen,
+                seo_descripcion: data.resumen,
+                seo_keywords: data.keywords,
                 imagen_destacada_id: mediaId
             };
 
@@ -210,7 +262,7 @@ async function run() {
             allExistingTitles.push(data.title);
             count++;
         }
-        console.log(`\nAll done! Successfully published ${count} unique news articles from April, May, and June.`);
+        console.log(`\nAll done! Successfully published ${count} perfect news articles.`);
     } catch(e) {
         console.error(e);
     }