almeira-web/scrape-multivision.js

const axios = require('axios');
const fs = require('fs');
const cheerio = require('cheerio');
const { Client, Databases, Storage, ID, Query } = require('node-appwrite');
require('dotenv').config({ path: '.env.local' });

let InputFile;
try { InputFile = require('node-appwrite/file').InputFile; } catch (e) { InputFile = null; }

const client = new Client()
    .setEndpoint(process.env.NEXT_PUBLIC_APPWRITE_ENDPOINT)
    .setProject(process.env.NEXT_PUBLIC_APPWRITE_PROJECT_ID)
    .setKey(process.env.APPWRITE_API_KEY);

const databases = new Databases(client);
const storage = new Storage(client);
const DB_ID = process.env.APPWRITE_DATABASE_ID;
const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID;

const excludedNodes = ['962', '963', '895', '894', '893', '964'];

async function deleteOldScrapes(postsCollectionId) {
    console.log('Deleting poorly scraped posts...');
    const res = await databases.listDocuments(DB_ID, postsCollectionId, [Query.limit(100)]);
    const toDelete = res.documents.filter(p => p.slug && p.slug.startsWith('multivision-'));
    for (const d of toDelete) {
        await databases.deleteDocument(DB_ID, postsCollectionId, d.$id);
        console.log('Deleted:', d.slug);
    }
}

async function scrapeNode(url) {
    try {
        const res = await axios.get(url);
        const $ = cheerio.load(res.data);

        const title = $('h1').first().text().replace(/\|.*/, '').trim() || $('title').text().replace(/\|.*/, '').trim();

        // Extract date
        let dateObj = null;
        const timeText = $('span').filter((i, el) => $(el).text().match(/\d{2}\/\d{2}\/\d{4} - \d{2}:\d{2}/)).first().text();
        // e.g. "Mar, 31/03/2026 - 10:16"
        if (timeText) {
            const match = timeText.match(/(\d{2})\/(\d{2})\/(\d{4}) - (\d{2}:\d{2})/);
            if (match) {
                // DD/MM/YYYY
                dateObj = new Date(`${match[3]}-${match[2]}-${match[1]}T${match[4]}:00.000Z`);
            }
        }

        if (!dateObj) return null; // Skip if no date found

        // We only want March 2026
        if (dateObj.getMonth() !== 2 || dateObj.getFullYear() !== 2026) {
            return { skipped: true, reason: 'Not in March 2026' };
        }

        // Extract image
        let image = null;
        $('img').each((i, el) => {
            const src = $(el).attr('src');
            if (src && (src.includes('public/imagenes') || src.includes('styles/')) && !src.includes('logo')) {
                image = src.startsWith('http') ? src : 'https://www.multivisiondecolombia.com' + src;
                return false; // break
            }
        });

        // Extract content
        let contentHtml = '';
        const bodyField = $('.field--name-body');
        if (bodyField.length > 0) {
            contentHtml = bodyField.html();
        } else {
            // fallback
            contentHtml = $('article').html() || '';
        }

        // Clean content: Keep only p, ul, li, strong, em, b, i
        const $content = cheerio.load(contentHtml || '');
        $content('*').each((i, el) => {
            if (!['p', 'ul', 'li', 'strong', 'em', 'b', 'i'].includes(el.tagName.toLowerCase())) {
                if (el.tagName.toLowerCase() === 'br') {
                    $content(el).replaceWith('<br/>');
                } else {
                    $content(el).replaceWith($content(el).text());
                }
            }
        });
        const finalContent = $content.html() || '<p></p>';

        return { title, image, date: dateObj.toISOString(), content: finalContent };
    } catch (e) {
        console.error('Error scraping node', url, e.message);
        return null;
    }
}

async function run() {
    try {
        const collections = await databases.listCollections(DB_ID);
        const map = {};
        collections.collections.forEach(c => { map[c.name.toLowerCase()] = c.$id; });
        const postsCollectionId = map['posts'] || 'posts';
        const mediaCollectionId = map['media'] || 'media';
        const pubTypesCollectionId = map['publication types'] || map['publication_types'] || 'publication_types';

        const typeRes = await databases.listDocuments(DB_ID, pubTypesCollectionId, [Query.equal('codigo', 'NOTICIA')]);
        const typeId = typeRes.documents.length > 0 ? typeRes.documents[0].$id : null;

        await deleteOldScrapes(postsCollectionId);

        console.log('Fetching news list from Multivision...');
        const allNodes = new Set();
        
        // Loop through some pages (0 to 5)
        for (let page = 0; page <= 5; page++) {
            try {
                const listUrl = `https://www.multivisiondecolombia.com/noticias?page=${page}`;
                const res = await axios.get(listUrl);
                const $ = cheerio.load(res.data);
                
                $('a').each((i, el) => {
                    const href = $(el).attr('href');
                    if (href && href.startsWith('/node/')) {
                        const nodeId = href.split('/')[2];
                        if (nodeId && !excludedNodes.includes(nodeId)) {
                            allNodes.add(`https://www.multivisiondecolombia.com/node/${nodeId}`);
                        }
                    }
                });
            } catch(e) {
                console.log(`Failed to fetch page ${page}`);
            }
        }

        console.log(`Found ${allNodes.size} valid nodes to process.`);
        
        let count = 0;
        for (const url of allNodes) {
            console.log(`\nProcessing: ${url}`);
            const data = await scrapeNode(url);
            
            if (!data) {
                console.log('No data extracted.');
                continue;
            }
            if (data.skipped) {
                console.log('Skipped:', data.reason);
                continue;
            }

            console.log('Extracted Title:', data.title);
            console.log('Extracted Date:', data.date);
            console.log('Extracted Image:', data.image);

            let mediaId = null;
            if (data.image) {
                try {
                    const imgRes = await axios.get(data.image, { responseType: 'arraybuffer' });
                    const buffer = Buffer.from(imgRes.data, 'binary');
                    const filePath = `/tmp/multivision_${count}.jpg`;
                    fs.writeFileSync(filePath, buffer);

                    let fileParam;
                    if (InputFile && InputFile.fromPath) fileParam = InputFile.fromPath(filePath, `image${count}.jpg`);
                    else fileParam = fs.createReadStream(filePath);

                    const file = await storage.createFile(BUCKET_ID, ID.unique(), fileParam);
                    const mediaDoc = await databases.createDocument(DB_ID, mediaCollectionId, ID.unique(), {
                        file_id: file.$id, bucket_id: BUCKET_ID,
                        titulo: data.title, tipo: 'imagen'
                    });
                    mediaId = mediaDoc.$id;
                } catch(err) {
                    console.error('Error uploading image:', err.message);
                }
            }

            const doc = {
                titulo: data.title.substring(0, 100),
                slug: `multivision-marzo-${Date.now()}-${count}`,
                tipo_publicacion_id: typeId,
                estado: 'Publicado',
                fecha_publicacion: data.date,
                contenido: data.content,
                imagen_destacada_id: mediaId
            };

            const res = await databases.createDocument(DB_ID, postsCollectionId, ID.unique(), doc);
            console.log('Published! ID:', res.$id);
            count++;
        }
        console.log(`\nAll done! Successfully published ${count} news articles from March.`);
    } catch(e) {
        console.error(e);
    }
}
run();