fix: Correctly extract content body, map images, and auto-generate SEO metadata
This commit is contained in:
parent
f444425e35
commit
0d110dae11
@ -20,6 +20,25 @@ const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID;
|
||||
|
||||
const excludedNodes = ['962', '963', '895', '894', '893', '964'];
|
||||
|
||||
async function deleteOldScrapes(postsCollectionId) {
|
||||
console.log('Deleting all poorly scraped multivision posts...');
|
||||
let cursor = null;
|
||||
let deletedCount = 0;
|
||||
while (true) {
|
||||
const queries = [Query.limit(100)];
|
||||
if (cursor) queries.push(Query.cursorAfter(cursor));
|
||||
const res = await databases.listDocuments(DB_ID, postsCollectionId, queries);
|
||||
const toDelete = res.documents.filter(p => p.slug && (p.slug.startsWith('multivision-amj-') || p.slug.startsWith('multivision-final-') || p.slug.startsWith('multivision-marzo-')));
|
||||
for (const d of toDelete) {
|
||||
await databases.deleteDocument(DB_ID, postsCollectionId, d.$id);
|
||||
deletedCount++;
|
||||
}
|
||||
if (res.documents.length < 100) break;
|
||||
cursor = res.documents[res.documents.length - 1].$id;
|
||||
}
|
||||
console.log(`Deleted ${deletedCount} old posts.`);
|
||||
}
|
||||
|
||||
async function scrapeNode(url) {
|
||||
try {
|
||||
const res = await axios.get(url);
|
||||
@ -39,17 +58,18 @@ async function scrapeNode(url) {
|
||||
|
||||
if (!dateObj) return null;
|
||||
|
||||
// We only want April (3), May (4), June (5) 2026
|
||||
const validMonths = [3, 4, 5];
|
||||
// We only want March (2), April (3), May (4), June (5) 2026
|
||||
const validMonths = [2, 3, 4, 5];
|
||||
if (!validMonths.includes(dateObj.getMonth()) || dateObj.getFullYear() !== 2026) {
|
||||
return { skipped: true, reason: 'Not in April, May, or June 2026' };
|
||||
return { skipped: true, reason: 'Not in Mar, Apr, May, or Jun 2026' };
|
||||
}
|
||||
|
||||
// Extract image
|
||||
let image = null;
|
||||
$('img').each((i, el) => {
|
||||
const src = $(el).attr('src');
|
||||
if (src && (src.includes('public/imagenes') || src.includes('styles/')) && !src.includes('logo')) {
|
||||
// Changed filter to match Multivision's actual image paths
|
||||
if (src && src.includes('sites/default/files') && !src.includes('logo')) {
|
||||
image = src.startsWith('http') ? src : 'https://www.multivisiondecolombia.com' + src;
|
||||
return false;
|
||||
}
|
||||
@ -57,26 +77,49 @@ async function scrapeNode(url) {
|
||||
|
||||
// Extract content
|
||||
let contentHtml = '';
|
||||
const bodyField = $('.field--name-body');
|
||||
if (bodyField.length > 0) {
|
||||
contentHtml = bodyField.html();
|
||||
} else {
|
||||
contentHtml = $('article').html() || '';
|
||||
}
|
||||
|
||||
const $content = cheerio.load(contentHtml || '');
|
||||
$content('*').each((i, el) => {
|
||||
if (!['p', 'ul', 'li', 'strong', 'em', 'b', 'i'].includes(el.tagName.toLowerCase())) {
|
||||
if (el.tagName.toLowerCase() === 'br') {
|
||||
$content(el).replaceWith('<br/>');
|
||||
} else {
|
||||
$content(el).replaceWith($content(el).text());
|
||||
}
|
||||
let maxLen = 0;
|
||||
|
||||
// Find the block with the most text
|
||||
$('.field--name-body, .field--name-field-resumen-de-la-noticia').each((i, el) => {
|
||||
const text = $(el).text().trim();
|
||||
if (text.length > maxLen) {
|
||||
maxLen = text.length;
|
||||
contentHtml = $(el).html();
|
||||
}
|
||||
});
|
||||
const finalContent = $content.html() || '<p></p>';
|
||||
|
||||
if (!contentHtml || maxLen < 100) {
|
||||
// Fallback to finding the longest div if both fail
|
||||
maxLen = 0;
|
||||
$('div').each((i, el) => {
|
||||
const text = $(el).text().trim();
|
||||
if (text.length > maxLen && $(el).children().length < 5) {
|
||||
maxLen = text.length;
|
||||
contentHtml = $(el).html();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return { title, image, date: dateObj.toISOString(), content: finalContent };
|
||||
// Clean content just a bit
|
||||
let cleanContent = (contentHtml || '<p></p>').trim();
|
||||
|
||||
// Auto-generate fields
|
||||
const plainText = cheerio.load(cleanContent).text().replace(/\s+/g, ' ').trim();
|
||||
let resumen = plainText.substring(0, 150);
|
||||
if (plainText.length > 150) resumen += '...';
|
||||
|
||||
// Keywords
|
||||
const words = plainText.split(' ').filter(w => w.length > 5).slice(0, 8);
|
||||
const keywords = words.join(', ').replace(/[^a-zA-ZáéíóúÁÉÍÓÚñÑ, ]/g, '');
|
||||
|
||||
return {
|
||||
title,
|
||||
image,
|
||||
date: dateObj.toISOString(),
|
||||
content: cleanContent,
|
||||
resumen,
|
||||
keywords
|
||||
};
|
||||
} catch (e) {
|
||||
console.error('Error scraping node', url, e.message);
|
||||
return null;
|
||||
@ -95,25 +138,32 @@ async function run() {
|
||||
const typeRes = await databases.listDocuments(DB_ID, pubTypesCollectionId, [Query.equal('codigo', 'NOTICIA')]);
|
||||
const typeId = typeRes.documents.length > 0 ? typeRes.documents[0].$id : null;
|
||||
|
||||
await deleteOldScrapes(postsCollectionId);
|
||||
|
||||
console.log('Fetching existing titles from Appwrite to prevent duplicates...');
|
||||
let allExistingTitles = [];
|
||||
let cursor = null;
|
||||
while (true) {
|
||||
const queries = [Query.limit(100), Query.select(['titulo'])];
|
||||
const queries = [Query.limit(100), Query.select(['titulo', 'slug'])];
|
||||
if (cursor) queries.push(Query.cursorAfter(cursor));
|
||||
const existingRes = await databases.listDocuments(DB_ID, postsCollectionId, queries);
|
||||
|
||||
existingRes.documents.forEach(d => allExistingTitles.push(d.titulo));
|
||||
existingRes.documents.forEach(d => {
|
||||
// Don't add multivision ones since we just deleted them or are replacing them
|
||||
if (!d.slug || !d.slug.startsWith('multivision-')) {
|
||||
allExistingTitles.push(d.titulo);
|
||||
}
|
||||
});
|
||||
|
||||
if (existingRes.documents.length < 100) break;
|
||||
cursor = existingRes.documents[existingRes.documents.length - 1].$id;
|
||||
}
|
||||
console.log(`Found ${allExistingTitles.length} existing posts.`);
|
||||
console.log(`Found ${allExistingTitles.length} existing native posts.`);
|
||||
|
||||
console.log('Fetching news list from Multivision (Pages 0-15)...');
|
||||
console.log('Fetching news list from Multivision (Pages 0-25)...');
|
||||
const allNodes = new Set();
|
||||
|
||||
for (let page = 0; page <= 15; page++) {
|
||||
for (let page = 0; page <= 25; page++) {
|
||||
try {
|
||||
const listUrl = `https://www.multivisiondecolombia.com/noticias?page=${page}`;
|
||||
const res = await axios.get(listUrl);
|
||||
@ -150,7 +200,6 @@ async function run() {
|
||||
}
|
||||
|
||||
// Check similarity
|
||||
let isDuplicate = false;
|
||||
let highestSimilarity = 0;
|
||||
let mostSimilarTitle = '';
|
||||
|
||||
@ -196,11 +245,14 @@ async function run() {
|
||||
|
||||
const doc = {
|
||||
titulo: data.title.substring(0, 100),
|
||||
slug: `multivision-amj-${Date.now()}-${count}`,
|
||||
slug: `multivision-final-${Date.now()}-${count}`,
|
||||
tipo_publicacion_id: typeId,
|
||||
estado: 'Publicado',
|
||||
fecha_publicacion: data.date,
|
||||
contenido: data.content,
|
||||
resumen_redes: data.resumen,
|
||||
seo_descripcion: data.resumen,
|
||||
seo_keywords: data.keywords,
|
||||
imagen_destacada_id: mediaId
|
||||
};
|
||||
|
||||
@ -210,7 +262,7 @@ async function run() {
|
||||
allExistingTitles.push(data.title);
|
||||
count++;
|
||||
}
|
||||
console.log(`\nAll done! Successfully published ${count} unique news articles from April, May, and June.`);
|
||||
console.log(`\nAll done! Successfully published ${count} perfect news articles.`);
|
||||
} catch(e) {
|
||||
console.error(e);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user