271 lines
11 KiB
JavaScript
271 lines
11 KiB
JavaScript
const axios = require('axios');
|
|
const fs = require('fs');
|
|
const cheerio = require('cheerio');
|
|
const stringSimilarity = require('string-similarity');
|
|
const { Client, Databases, Storage, ID, Query } = require('node-appwrite');
|
|
require('dotenv').config({ path: '.env.local' });
|
|
|
|
let InputFile;
|
|
try { InputFile = require('node-appwrite/file').InputFile; } catch (e) { InputFile = null; }
|
|
|
|
const client = new Client()
|
|
.setEndpoint(process.env.NEXT_PUBLIC_APPWRITE_ENDPOINT)
|
|
.setProject(process.env.NEXT_PUBLIC_APPWRITE_PROJECT_ID)
|
|
.setKey(process.env.APPWRITE_API_KEY);
|
|
|
|
const databases = new Databases(client);
|
|
const storage = new Storage(client);
|
|
const DB_ID = process.env.APPWRITE_DATABASE_ID;
|
|
const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID;
|
|
|
|
const excludedNodes = ['962', '963', '895', '894', '893', '964'];
|
|
|
|
async function deleteOldScrapes(postsCollectionId) {
|
|
console.log('Deleting all poorly scraped multivision posts...');
|
|
let cursor = null;
|
|
let deletedCount = 0;
|
|
while (true) {
|
|
const queries = [Query.limit(100)];
|
|
if (cursor) queries.push(Query.cursorAfter(cursor));
|
|
const res = await databases.listDocuments(DB_ID, postsCollectionId, queries);
|
|
const toDelete = res.documents.filter(p => p.slug && (p.slug.startsWith('multivision-amj-') || p.slug.startsWith('multivision-final-') || p.slug.startsWith('multivision-marzo-')));
|
|
for (const d of toDelete) {
|
|
await databases.deleteDocument(DB_ID, postsCollectionId, d.$id);
|
|
deletedCount++;
|
|
}
|
|
if (res.documents.length < 100) break;
|
|
cursor = res.documents[res.documents.length - 1].$id;
|
|
}
|
|
console.log(`Deleted ${deletedCount} old posts.`);
|
|
}
|
|
|
|
async function scrapeNode(url) {
|
|
try {
|
|
const res = await axios.get(url);
|
|
const $ = cheerio.load(res.data);
|
|
|
|
const title = $('h1').first().text().replace(/\|.*/, '').trim() || $('title').text().replace(/\|.*/, '').trim();
|
|
|
|
// Extract date
|
|
let dateObj = null;
|
|
const timeText = $('span').filter((i, el) => $(el).text().match(/\d{2}\/\d{2}\/\d{4} - \d{2}:\d{2}/)).first().text();
|
|
if (timeText) {
|
|
const match = timeText.match(/(\d{2})\/(\d{2})\/(\d{4}) - (\d{2}:\d{2})/);
|
|
if (match) {
|
|
dateObj = new Date(`${match[3]}-${match[2]}-${match[1]}T${match[4]}:00.000Z`);
|
|
}
|
|
}
|
|
|
|
if (!dateObj) return null;
|
|
|
|
// We only want March (2), April (3), May (4), June (5) 2026
|
|
const validMonths = [2, 3, 4, 5];
|
|
if (!validMonths.includes(dateObj.getMonth()) || dateObj.getFullYear() !== 2026) {
|
|
return { skipped: true, reason: 'Not in Mar, Apr, May, or Jun 2026' };
|
|
}
|
|
|
|
// Extract image
|
|
let image = null;
|
|
$('img').each((i, el) => {
|
|
const src = $(el).attr('src');
|
|
// Changed filter to match Multivision's actual image paths
|
|
if (src && src.includes('sites/default/files') && !src.includes('logo')) {
|
|
image = src.startsWith('http') ? src : 'https://www.multivisiondecolombia.com' + src;
|
|
return false;
|
|
}
|
|
});
|
|
|
|
// Extract content
|
|
let contentHtml = '';
|
|
let maxLen = 0;
|
|
|
|
// Find the block with the most text
|
|
$('.field--name-body, .field--name-field-resumen-de-la-noticia').each((i, el) => {
|
|
const text = $(el).text().trim();
|
|
if (text.length > maxLen) {
|
|
maxLen = text.length;
|
|
contentHtml = $(el).html();
|
|
}
|
|
});
|
|
|
|
if (!contentHtml || maxLen < 100) {
|
|
// Fallback to finding the longest div if both fail
|
|
maxLen = 0;
|
|
$('div').each((i, el) => {
|
|
const text = $(el).text().trim();
|
|
if (text.length > maxLen && $(el).children().length < 5) {
|
|
maxLen = text.length;
|
|
contentHtml = $(el).html();
|
|
}
|
|
});
|
|
}
|
|
|
|
// Clean content just a bit
|
|
let cleanContent = (contentHtml || '<p></p>').trim();
|
|
|
|
// Auto-generate fields
|
|
const plainText = cheerio.load(cleanContent).text().replace(/\s+/g, ' ').trim();
|
|
let resumen = plainText.substring(0, 150);
|
|
if (plainText.length > 150) resumen += '...';
|
|
|
|
// Keywords
|
|
const words = plainText.split(' ').filter(w => w.length > 5).slice(0, 8);
|
|
const keywords = words.join(', ').replace(/[^a-zA-ZáéíóúÁÉÍÓÚñÑ, ]/g, '');
|
|
|
|
return {
|
|
title,
|
|
image,
|
|
date: dateObj.toISOString(),
|
|
content: cleanContent,
|
|
resumen,
|
|
keywords
|
|
};
|
|
} catch (e) {
|
|
console.error('Error scraping node', url, e.message);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function run() {
|
|
try {
|
|
const collections = await databases.listCollections(DB_ID);
|
|
const map = {};
|
|
collections.collections.forEach(c => { map[c.name.toLowerCase()] = c.$id; });
|
|
const postsCollectionId = map['posts'] || 'posts';
|
|
const mediaCollectionId = map['media'] || 'media';
|
|
const pubTypesCollectionId = map['publication types'] || map['publication_types'] || 'publication_types';
|
|
|
|
const typeRes = await databases.listDocuments(DB_ID, pubTypesCollectionId, [Query.equal('codigo', 'NOTICIA')]);
|
|
const typeId = typeRes.documents.length > 0 ? typeRes.documents[0].$id : null;
|
|
|
|
await deleteOldScrapes(postsCollectionId);
|
|
|
|
console.log('Fetching existing titles from Appwrite to prevent duplicates...');
|
|
let allExistingTitles = [];
|
|
let cursor = null;
|
|
while (true) {
|
|
const queries = [Query.limit(100), Query.select(['titulo', 'slug'])];
|
|
if (cursor) queries.push(Query.cursorAfter(cursor));
|
|
const existingRes = await databases.listDocuments(DB_ID, postsCollectionId, queries);
|
|
|
|
existingRes.documents.forEach(d => {
|
|
// Don't add multivision ones since we just deleted them or are replacing them
|
|
if (!d.slug || !d.slug.startsWith('multivision-')) {
|
|
allExistingTitles.push(d.titulo);
|
|
}
|
|
});
|
|
|
|
if (existingRes.documents.length < 100) break;
|
|
cursor = existingRes.documents[existingRes.documents.length - 1].$id;
|
|
}
|
|
console.log(`Found ${allExistingTitles.length} existing native posts.`);
|
|
|
|
console.log('Fetching news list from Multivision (Pages 0-25)...');
|
|
const allNodes = new Set();
|
|
|
|
for (let page = 0; page <= 25; page++) {
|
|
try {
|
|
const listUrl = `https://www.multivisiondecolombia.com/noticias?page=${page}`;
|
|
const res = await axios.get(listUrl);
|
|
const $ = cheerio.load(res.data);
|
|
|
|
$('a').each((i, el) => {
|
|
const href = $(el).attr('href');
|
|
if (href && href.startsWith('/node/')) {
|
|
const nodeId = href.split('/')[2];
|
|
if (nodeId && !excludedNodes.includes(nodeId)) {
|
|
allNodes.add(`https://www.multivisiondecolombia.com/node/${nodeId}`);
|
|
}
|
|
}
|
|
});
|
|
} catch(e) {
|
|
console.log(`Failed to fetch page ${page}`);
|
|
}
|
|
}
|
|
|
|
console.log(`Found ${allNodes.size} valid nodes to process.`);
|
|
|
|
let count = 0;
|
|
for (const url of allNodes) {
|
|
console.log(`\nProcessing: ${url}`);
|
|
const data = await scrapeNode(url);
|
|
|
|
if (!data) {
|
|
console.log('No data extracted.');
|
|
continue;
|
|
}
|
|
if (data.skipped) {
|
|
console.log('Skipped:', data.reason);
|
|
continue;
|
|
}
|
|
|
|
// Check similarity
|
|
let highestSimilarity = 0;
|
|
let mostSimilarTitle = '';
|
|
|
|
for (const exTitle of allExistingTitles) {
|
|
if (!exTitle) continue;
|
|
const sim = stringSimilarity.compareTwoStrings(data.title.toLowerCase(), exTitle.toLowerCase());
|
|
if (sim > highestSimilarity) {
|
|
highestSimilarity = sim;
|
|
mostSimilarTitle = exTitle;
|
|
}
|
|
}
|
|
|
|
if (highestSimilarity > 0.60) {
|
|
console.log(`Skipped due to similarity (${(highestSimilarity*100).toFixed(1)}%) with: "${mostSimilarTitle}"`);
|
|
continue;
|
|
}
|
|
|
|
console.log('Extracted Title:', data.title);
|
|
console.log('Extracted Date:', data.date);
|
|
|
|
let mediaId = null;
|
|
if (data.image) {
|
|
try {
|
|
const imgRes = await axios.get(data.image, { responseType: 'arraybuffer' });
|
|
const buffer = Buffer.from(imgRes.data, 'binary');
|
|
const filePath = `/tmp/multivision_${count}.jpg`;
|
|
fs.writeFileSync(filePath, buffer);
|
|
|
|
let fileParam;
|
|
if (InputFile && InputFile.fromPath) fileParam = InputFile.fromPath(filePath, `image${count}.jpg`);
|
|
else fileParam = fs.createReadStream(filePath);
|
|
|
|
const file = await storage.createFile(BUCKET_ID, ID.unique(), fileParam);
|
|
const mediaDoc = await databases.createDocument(DB_ID, mediaCollectionId, ID.unique(), {
|
|
file_id: file.$id, bucket_id: BUCKET_ID,
|
|
titulo: data.title, tipo: 'imagen'
|
|
});
|
|
mediaId = mediaDoc.$id;
|
|
} catch(err) {
|
|
console.error('Error uploading image:', err.message);
|
|
}
|
|
}
|
|
|
|
const doc = {
|
|
titulo: data.title.substring(0, 100),
|
|
slug: `multivision-final-${Date.now()}-${count}`,
|
|
tipo_publicacion_id: typeId,
|
|
estado: 'Publicado',
|
|
fecha_publicacion: data.date,
|
|
contenido: data.content,
|
|
resumen_redes: data.resumen,
|
|
seo_descripcion: data.resumen,
|
|
seo_keywords: data.keywords,
|
|
imagen_destacada_id: mediaId
|
|
};
|
|
|
|
const res = await databases.createDocument(DB_ID, postsCollectionId, ID.unique(), doc);
|
|
console.log('Published! ID:', res.$id);
|
|
// Add to existing to prevent duplicates within this run
|
|
allExistingTitles.push(data.title);
|
|
count++;
|
|
}
|
|
console.log(`\nAll done! Successfully published ${count} perfect news articles.`);
|
|
} catch(e) {
|
|
console.error(e);
|
|
}
|
|
}
|
|
run();
|