diff --git a/scrape-multivision.js b/scrape-multivision.js new file mode 100644 index 0000000..b2c6b3b --- /dev/null +++ b/scrape-multivision.js @@ -0,0 +1,198 @@ +const axios = require('axios'); +const fs = require('fs'); +const cheerio = require('cheerio'); +const { Client, Databases, Storage, ID, Query } = require('node-appwrite'); +require('dotenv').config({ path: '.env.local' }); + +let InputFile; +try { InputFile = require('node-appwrite/file').InputFile; } catch (e) { InputFile = null; } + +const client = new Client() + .setEndpoint(process.env.NEXT_PUBLIC_APPWRITE_ENDPOINT) + .setProject(process.env.NEXT_PUBLIC_APPWRITE_PROJECT_ID) + .setKey(process.env.APPWRITE_API_KEY); + +const databases = new Databases(client); +const storage = new Storage(client); +const DB_ID = process.env.APPWRITE_DATABASE_ID; +const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID; + +const excludedNodes = ['962', '963', '895', '894', '893', '964']; + +async function deleteOldScrapes(postsCollectionId) { + console.log('Deleting poorly scraped posts...'); + const res = await databases.listDocuments(DB_ID, postsCollectionId, [Query.limit(100)]); + const toDelete = res.documents.filter(p => p.slug && p.slug.startsWith('multivision-')); + for (const d of toDelete) { + await databases.deleteDocument(DB_ID, postsCollectionId, d.$id); + console.log('Deleted:', d.slug); + } +} + +async function scrapeNode(url) { + try { + const res = await axios.get(url); + const $ = cheerio.load(res.data); + + const title = $('h1').first().text().replace(/\|.*/, '').trim() || $('title').text().replace(/\|.*/, '').trim(); + + // Extract date + let dateObj = null; + const timeText = $('span').filter((i, el) => $(el).text().match(/\d{2}\/\d{2}\/\d{4} - \d{2}:\d{2}/)).first().text(); + // e.g. "Mar, 31/03/2026 - 10:16" + if (timeText) { + const match = timeText.match(/(\d{2})\/(\d{2})\/(\d{4}) - (\d{2}:\d{2})/); + if (match) { + // DD/MM/YYYY + dateObj = new Date(`${match[3]}-${match[2]}-${match[1]}T${match[4]}:00.000Z`); + } + } + + if (!dateObj) return null; // Skip if no date found + + // We only want March 2026 + if (dateObj.getMonth() !== 2 || dateObj.getFullYear() !== 2026) { + return { skipped: true, reason: 'Not in March 2026' }; + } + + // Extract image + let image = null; + $('img').each((i, el) => { + const src = $(el).attr('src'); + if (src && (src.includes('public/imagenes') || src.includes('styles/')) && !src.includes('logo')) { + image = src.startsWith('http') ? src : 'https://www.multivisiondecolombia.com' + src; + return false; // break + } + }); + + // Extract content + let contentHtml = ''; + const bodyField = $('.field--name-body'); + if (bodyField.length > 0) { + contentHtml = bodyField.html(); + } else { + // fallback + contentHtml = $('article').html() || ''; + } + + // Clean content: Keep only p, ul, li, strong, em, b, i + const $content = cheerio.load(contentHtml || ''); + $content('*').each((i, el) => { + if (!['p', 'ul', 'li', 'strong', 'em', 'b', 'i'].includes(el.tagName.toLowerCase())) { + if (el.tagName.toLowerCase() === 'br') { + $content(el).replaceWith('
'); + } else { + $content(el).replaceWith($content(el).text()); + } + } + }); + const finalContent = $content.html() || '

'; + + return { title, image, date: dateObj.toISOString(), content: finalContent }; + } catch (e) { + console.error('Error scraping node', url, e.message); + return null; + } +} + +async function run() { + try { + const collections = await databases.listCollections(DB_ID); + const map = {}; + collections.collections.forEach(c => { map[c.name.toLowerCase()] = c.$id; }); + const postsCollectionId = map['posts'] || 'posts'; + const mediaCollectionId = map['media'] || 'media'; + const pubTypesCollectionId = map['publication types'] || map['publication_types'] || 'publication_types'; + + const typeRes = await databases.listDocuments(DB_ID, pubTypesCollectionId, [Query.equal('codigo', 'NOTICIA')]); + const typeId = typeRes.documents.length > 0 ? typeRes.documents[0].$id : null; + + await deleteOldScrapes(postsCollectionId); + + console.log('Fetching news list from Multivision...'); + const allNodes = new Set(); + + // Loop through some pages (0 to 5) + for (let page = 0; page <= 5; page++) { + try { + const listUrl = `https://www.multivisiondecolombia.com/noticias?page=${page}`; + const res = await axios.get(listUrl); + const $ = cheerio.load(res.data); + + $('a').each((i, el) => { + const href = $(el).attr('href'); + if (href && href.startsWith('/node/')) { + const nodeId = href.split('/')[2]; + if (nodeId && !excludedNodes.includes(nodeId)) { + allNodes.add(`https://www.multivisiondecolombia.com/node/${nodeId}`); + } + } + }); + } catch(e) { + console.log(`Failed to fetch page ${page}`); + } + } + + console.log(`Found ${allNodes.size} valid nodes to process.`); + + let count = 0; + for (const url of allNodes) { + console.log(`\nProcessing: ${url}`); + const data = await scrapeNode(url); + + if (!data) { + console.log('No data extracted.'); + continue; + } + if (data.skipped) { + console.log('Skipped:', data.reason); + continue; + } + + console.log('Extracted Title:', data.title); + console.log('Extracted Date:', data.date); + console.log('Extracted Image:', data.image); + + let mediaId = null; + if (data.image) { + try { + const imgRes = await axios.get(data.image, { responseType: 'arraybuffer' }); + const buffer = Buffer.from(imgRes.data, 'binary'); + const filePath = `/tmp/multivision_${count}.jpg`; + fs.writeFileSync(filePath, buffer); + + let fileParam; + if (InputFile && InputFile.fromPath) fileParam = InputFile.fromPath(filePath, `image${count}.jpg`); + else fileParam = fs.createReadStream(filePath); + + const file = await storage.createFile(BUCKET_ID, ID.unique(), fileParam); + const mediaDoc = await databases.createDocument(DB_ID, mediaCollectionId, ID.unique(), { + file_id: file.$id, bucket_id: BUCKET_ID, + titulo: data.title, tipo: 'imagen' + }); + mediaId = mediaDoc.$id; + } catch(err) { + console.error('Error uploading image:', err.message); + } + } + + const doc = { + titulo: data.title.substring(0, 100), + slug: `multivision-marzo-${Date.now()}-${count}`, + tipo_publicacion_id: typeId, + estado: 'Publicado', + fecha_publicacion: data.date, + contenido: data.content, + imagen_destacada_id: mediaId + }; + + const res = await databases.createDocument(DB_ID, postsCollectionId, ID.unique(), doc); + console.log('Published! ID:', res.$id); + count++; + } + console.log(`\nAll done! Successfully published ${count} news articles from March.`); + } catch(e) { + console.error(e); + } +} +run();