almeira-web/publish-scraper.js

const axios = require('axios');
const fs = require('fs');
const { Client, Databases, Storage, ID, Query } = require('node-appwrite');
require('dotenv').config({ path: '.env.local' });

let InputFile;
try { InputFile = require('node-appwrite/file').InputFile; } catch (e) { InputFile = null; }

const client = new Client()
    .setEndpoint(process.env.NEXT_PUBLIC_APPWRITE_ENDPOINT)
    .setProject(process.env.NEXT_PUBLIC_APPWRITE_PROJECT_ID)
    .setKey(process.env.APPWRITE_API_KEY);

const databases = new Databases(client);
const storage = new Storage(client);
const DB_ID = process.env.APPWRITE_DATABASE_ID;
const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID;

const urls = [
    'https://www.multivisiondecolombia.com/node/962',
    'https://www.multivisiondecolombia.com/node/963',
    'https://www.multivisiondecolombia.com/node/895',
    'https://www.multivisiondecolombia.com/node/894',
    'https://www.multivisiondecolombia.com/node/893',
    'https://www.multivisiondecolombia.com/node/964'
];

async function extractData(url) {
    const res = await axios.get(url);
    const html = res.data;

    let title = '';
    const titleMatch = html.match(/<h1[^>]*>(.*?)<\/h1>/i) || html.match(/<title>(.*?)<\/title>/i);
    if (titleMatch) title = titleMatch[1].replace(/\|.*/, '').trim();

    let image = null;
    const imgMatch = html.match(/<img[^>]+src="([^"]+)"[^>]*>/ig);
    if (imgMatch) {
        // Find the main image, usually in a field like field--name-field-image
        for (const imgTag of imgMatch) {
            const srcMatch = imgTag.match(/src="([^"]+)"/);
            if (srcMatch && (srcMatch[1].includes('public/imagenes') || srcMatch[1].includes('styles/'))) {
                image = srcMatch[1].startsWith('http') ? srcMatch[1] : 'https://www.multivisiondecolombia.com' + srcMatch[1];
                break;
            }
        }
    }

    let date = new Date().toISOString();
    // Look for <time datetime="..."> or a date string
    const timeMatch = html.match(/<time[^>]+datetime="([^"]+)"/);
    if (timeMatch) date = new Date(timeMatch[1]).toISOString();

    let content = '';
    // Look for the main content body, often in field--name-body or a specific div
    const bodyMatch = html.match(/<div[^>]+property="schema:text"[^>]*>([\s\S]*?)<\/div>/i) || html.match(/<div[^>]+class="[^"]*field--name-body[^"]*"[^>]*>([\s\S]*?)<\/div>\s*<\/div>/i);
    if (bodyMatch) {
        content = bodyMatch[1].trim();
    } else {
        // Fallback: extract p tags between article
        const articleMatch = html.match(/<article[^>]*>([\s\S]*?)<\/article>/i);
        if (articleMatch) {
            const pTags = articleMatch[1].match(/<p>([\s\S]*?)<\/p>/ig);
            if (pTags) content = pTags.join('\n');
        }
    }

    // Strip out tags except p, b, i, strong, em, ul, li, ol
    content = content.replace(/<(?!p|b|i|strong|em|ul|li|ol|\/p|\/b|\/i|\/strong|\/em|\/ul|\/li|\/ol)[^>]+>/ig, '');

    return { title, image, date, content };
}

async function run() {
    try {
        const list = await databases.listCollections(DB_ID);
        const map = {};
        list.collections.forEach(c => { map[c.name.toLowerCase()] = c.$id; });
        const postsCollectionId = map['posts'] || 'posts';
        const mediaCollectionId = map['media'] || 'media';
        const pubTypesCollectionId = map['publication types'] || map['publication_types'] || 'publication_types';

        const typeRes = await databases.listDocuments(DB_ID, pubTypesCollectionId, [Query.equal('codigo', 'NOTICIA')]);
        const typeId = typeRes.documents.length > 0 ? typeRes.documents[0].$id : null;

        for (let i = 0; i < urls.length; i++) {
            const url = urls[i];
            console.log('Processing:', url);
            const data = await extractData(url);
            console.log('Extracted Title:', data.title);
            console.log('Extracted Date:', data.date);
            console.log('Extracted Image:', data.image);

            let mediaId = null;
            if (data.image) {
                try {
                    const imgRes = await axios.get(data.image, { responseType: 'arraybuffer' });
                    const buffer = Buffer.from(imgRes.data, 'binary');
                    const filePath = `/tmp/scraped_img_${i}.jpg`;
                    fs.writeFileSync(filePath, buffer);

                    let fileParam;
                    if (InputFile && InputFile.fromPath) fileParam = InputFile.fromPath(filePath, `image${i}.jpg`);
                    else fileParam = fs.createReadStream(filePath);

                    const file = await storage.createFile(BUCKET_ID, ID.unique(), fileParam);
                    const mediaDoc = await databases.createDocument(DB_ID, mediaCollectionId, ID.unique(), {
                        file_id: file.$id, bucket_id: BUCKET_ID,
                        titulo: data.title, tipo: 'imagen'
                    });
                    mediaId = mediaDoc.$id;
                } catch(err) {
                    console.error('Error uploading image:', err.message);
                }
            }

            const doc = {
                titulo: data.title.substring(0, 100),
                slug: `multivision-${Date.now()}-${i}`,
                tipo_publicacion_id: typeId,
                estado: 'Publicado',
                fecha_publicacion: data.date,
                contenido: data.content,
                imagen_destacada_id: mediaId
            };

            const res = await databases.createDocument(DB_ID, postsCollectionId, ID.unique(), doc);
            console.log('Published! ID:', res.$id);
            console.log('---------------------------');
        }
    } catch(e) {
        console.error(e);
    }
}
run();