diff --git a/scrape-multivision.js b/scrape-multivision.js index 1afc1bb..73ade7f 100644 --- a/scrape-multivision.js +++ b/scrape-multivision.js @@ -20,6 +20,25 @@ const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID; const excludedNodes = ['962', '963', '895', '894', '893', '964']; +async function deleteOldScrapes(postsCollectionId) { + console.log('Deleting all poorly scraped multivision posts...'); + let cursor = null; + let deletedCount = 0; + while (true) { + const queries = [Query.limit(100)]; + if (cursor) queries.push(Query.cursorAfter(cursor)); + const res = await databases.listDocuments(DB_ID, postsCollectionId, queries); + const toDelete = res.documents.filter(p => p.slug && (p.slug.startsWith('multivision-amj-') || p.slug.startsWith('multivision-final-') || p.slug.startsWith('multivision-marzo-'))); + for (const d of toDelete) { + await databases.deleteDocument(DB_ID, postsCollectionId, d.$id); + deletedCount++; + } + if (res.documents.length < 100) break; + cursor = res.documents[res.documents.length - 1].$id; + } + console.log(`Deleted ${deletedCount} old posts.`); +} + async function scrapeNode(url) { try { const res = await axios.get(url); @@ -39,17 +58,18 @@ async function scrapeNode(url) { if (!dateObj) return null; - // We only want April (3), May (4), June (5) 2026 - const validMonths = [3, 4, 5]; + // We only want March (2), April (3), May (4), June (5) 2026 + const validMonths = [2, 3, 4, 5]; if (!validMonths.includes(dateObj.getMonth()) || dateObj.getFullYear() !== 2026) { - return { skipped: true, reason: 'Not in April, May, or June 2026' }; + return { skipped: true, reason: 'Not in Mar, Apr, May, or Jun 2026' }; } // Extract image let image = null; $('img').each((i, el) => { const src = $(el).attr('src'); - if (src && (src.includes('public/imagenes') || src.includes('styles/')) && !src.includes('logo')) { + // Changed filter to match Multivision's actual image paths + if (src && src.includes('sites/default/files') && !src.includes('logo')) { image = src.startsWith('http') ? src : 'https://www.multivisiondecolombia.com' + src; return false; } @@ -57,26 +77,49 @@ async function scrapeNode(url) { // Extract content let contentHtml = ''; - const bodyField = $('.field--name-body'); - if (bodyField.length > 0) { - contentHtml = bodyField.html(); - } else { - contentHtml = $('article').html() || ''; - } - - const $content = cheerio.load(contentHtml || ''); - $content('*').each((i, el) => { - if (!['p', 'ul', 'li', 'strong', 'em', 'b', 'i'].includes(el.tagName.toLowerCase())) { - if (el.tagName.toLowerCase() === 'br') { - $content(el).replaceWith('
'); - } else { - $content(el).replaceWith($content(el).text()); - } + let maxLen = 0; + + // Find the block with the most text + $('.field--name-body, .field--name-field-resumen-de-la-noticia').each((i, el) => { + const text = $(el).text().trim(); + if (text.length > maxLen) { + maxLen = text.length; + contentHtml = $(el).html(); } }); - const finalContent = $content.html() || '

'; + + if (!contentHtml || maxLen < 100) { + // Fallback to finding the longest div if both fail + maxLen = 0; + $('div').each((i, el) => { + const text = $(el).text().trim(); + if (text.length > maxLen && $(el).children().length < 5) { + maxLen = text.length; + contentHtml = $(el).html(); + } + }); + } - return { title, image, date: dateObj.toISOString(), content: finalContent }; + // Clean content just a bit + let cleanContent = (contentHtml || '

').trim(); + + // Auto-generate fields + const plainText = cheerio.load(cleanContent).text().replace(/\s+/g, ' ').trim(); + let resumen = plainText.substring(0, 150); + if (plainText.length > 150) resumen += '...'; + + // Keywords + const words = plainText.split(' ').filter(w => w.length > 5).slice(0, 8); + const keywords = words.join(', ').replace(/[^a-zA-ZáéíóúÁÉÍÓÚñÑ, ]/g, ''); + + return { + title, + image, + date: dateObj.toISOString(), + content: cleanContent, + resumen, + keywords + }; } catch (e) { console.error('Error scraping node', url, e.message); return null; @@ -95,25 +138,32 @@ async function run() { const typeRes = await databases.listDocuments(DB_ID, pubTypesCollectionId, [Query.equal('codigo', 'NOTICIA')]); const typeId = typeRes.documents.length > 0 ? typeRes.documents[0].$id : null; + await deleteOldScrapes(postsCollectionId); + console.log('Fetching existing titles from Appwrite to prevent duplicates...'); let allExistingTitles = []; let cursor = null; while (true) { - const queries = [Query.limit(100), Query.select(['titulo'])]; + const queries = [Query.limit(100), Query.select(['titulo', 'slug'])]; if (cursor) queries.push(Query.cursorAfter(cursor)); const existingRes = await databases.listDocuments(DB_ID, postsCollectionId, queries); - existingRes.documents.forEach(d => allExistingTitles.push(d.titulo)); + existingRes.documents.forEach(d => { + // Don't add multivision ones since we just deleted them or are replacing them + if (!d.slug || !d.slug.startsWith('multivision-')) { + allExistingTitles.push(d.titulo); + } + }); if (existingRes.documents.length < 100) break; cursor = existingRes.documents[existingRes.documents.length - 1].$id; } - console.log(`Found ${allExistingTitles.length} existing posts.`); + console.log(`Found ${allExistingTitles.length} existing native posts.`); - console.log('Fetching news list from Multivision (Pages 0-15)...'); + console.log('Fetching news list from Multivision (Pages 0-25)...'); const allNodes = new Set(); - for (let page = 0; page <= 15; page++) { + for (let page = 0; page <= 25; page++) { try { const listUrl = `https://www.multivisiondecolombia.com/noticias?page=${page}`; const res = await axios.get(listUrl); @@ -150,7 +200,6 @@ async function run() { } // Check similarity - let isDuplicate = false; let highestSimilarity = 0; let mostSimilarTitle = ''; @@ -196,11 +245,14 @@ async function run() { const doc = { titulo: data.title.substring(0, 100), - slug: `multivision-amj-${Date.now()}-${count}`, + slug: `multivision-final-${Date.now()}-${count}`, tipo_publicacion_id: typeId, estado: 'Publicado', fecha_publicacion: data.date, contenido: data.content, + resumen_redes: data.resumen, + seo_descripcion: data.resumen, + seo_keywords: data.keywords, imagen_destacada_id: mediaId }; @@ -210,7 +262,7 @@ async function run() { allExistingTitles.push(data.title); count++; } - console.log(`\nAll done! Successfully published ${count} unique news articles from April, May, and June.`); + console.log(`\nAll done! Successfully published ${count} perfect news articles.`); } catch(e) { console.error(e); }