339 lines
15 KiB
JavaScript
339 lines
15 KiB
JavaScript
const axios = require('axios');
|
|
const cheerio = require('cheerio');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const os = require('os');
|
|
const { Client, Databases, Storage, ID, Query } = require('node-appwrite');
|
|
require('dotenv').config({ path: '.env.local' });
|
|
|
|
let InputFile;
|
|
try { InputFile = require('node-appwrite/file').InputFile; } catch (e) { InputFile = null; }
|
|
|
|
const { applyTablesDbShim } = require('../src/lib/appwrite-tables-shim');
|
|
const client = new Client()
|
|
.setEndpoint(process.env.NEXT_PUBLIC_APPWRITE_ENDPOINT)
|
|
.setProject(process.env.NEXT_PUBLIC_APPWRITE_PROJECT_ID)
|
|
.setKey(process.env.APPWRITE_API_KEY);
|
|
applyTablesDbShim(client);
|
|
|
|
const databases = new Databases(client);
|
|
const storage = new Storage(client);
|
|
const DB_ID = process.env.APPWRITE_DATABASE_ID;
|
|
const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID;
|
|
|
|
function slugify(text) {
|
|
return text
|
|
.toString()
|
|
.toLowerCase()
|
|
.normalize('NFD')
|
|
.replace(/[\u0300-\u036f]/g, '')
|
|
.replace(/[^a-z0-9 -]/g, '')
|
|
.replace(/\s+/g, '-')
|
|
.replace(/-+/g, '-')
|
|
.trim();
|
|
}
|
|
|
|
function cleanContentHtml(html) {
|
|
if (!html) return '';
|
|
const $ = cheerio.load(html, null, false);
|
|
|
|
$('style, script, link, iframe, object, embed, video, audio, source, noscript').remove();
|
|
|
|
const allowedTags = ['p', 'b', 'i', 'strong', 'em', 'ul', 'ol', 'li', 'br', 'a'];
|
|
|
|
function processNode(node) {
|
|
$(node).children().each((i, child) => {
|
|
processNode(child);
|
|
});
|
|
|
|
const tagName = node.tagName ? node.tagName.toLowerCase() : null;
|
|
if (tagName) {
|
|
const attribs = Object.keys(node.attribs || {});
|
|
attribs.forEach(attr => {
|
|
if (tagName !== 'a' || attr !== 'href') {
|
|
$(node).removeAttr(attr);
|
|
}
|
|
});
|
|
|
|
if (!allowedTags.includes(tagName)) {
|
|
$(node).replaceWith($(node).html() || '');
|
|
}
|
|
}
|
|
}
|
|
|
|
$.root().children().each((i, child) => {
|
|
processNode(child);
|
|
});
|
|
|
|
return $.html().trim();
|
|
}
|
|
|
|
async function getCollectionIds() {
|
|
const list = await databases.listCollections(DB_ID);
|
|
const map = {};
|
|
list.collections.forEach(c => { map[c.name.toLowerCase()] = c.$id; });
|
|
return map;
|
|
}
|
|
|
|
const sleep = ms => new Promise(res => setTimeout(res, ms));
|
|
|
|
async function run() {
|
|
console.log("===============================================================================");
|
|
console.log("Iniciando publicación de noticias de la Gobernación del Cauca (17 Sep a la fecha)");
|
|
console.log("===============================================================================");
|
|
|
|
try {
|
|
const collections = await getCollectionIds();
|
|
const postsCollectionId = collections['posts'] || 'posts';
|
|
const mediaCollectionId = collections['media'] || 'media';
|
|
const pubTypesCollectionId = collections['publication types'] || collections['publication_types'] || 'publication_types';
|
|
|
|
// 1. Obtener ID del tipo de publicación NOTICIA
|
|
const pubTypesRes = await databases.listDocuments(DB_ID, pubTypesCollectionId);
|
|
const typeMap = {};
|
|
pubTypesRes.documents.forEach(d => {
|
|
typeMap[d.codigo] = d.$id;
|
|
});
|
|
const typeId = typeMap['NOTICIA'];
|
|
if (!typeId) {
|
|
throw new Error("No se encontró el tipo de publicación NOTICIA en Appwrite.");
|
|
}
|
|
console.log(`Tipo de publicación NOTICIA ID: ${typeId}`);
|
|
|
|
// 2. Obtener lista de noticias desde la API SharePoint de la Gobernación
|
|
console.log("Consultando API SharePoint de cauca.gov.co...");
|
|
const listUrl = "https://cauca.gov.co/Prensa/SaladePrensa/_api/web/lists(guid'e1840969-e886-4c1e-8cde-7498ab33f10a')/items?$select=Id,Title,ArticleStartDate,Created,Modified,File/ServerRelativeUrl&$expand=File&$orderby=Created desc&$top=100";
|
|
const spRes = await axios.get(listUrl, {
|
|
headers: { 'Accept': 'application/json;odata=verbose', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' },
|
|
timeout: 15000
|
|
});
|
|
|
|
const rawItems = spRes.data.d?.results || [];
|
|
console.log(`Total artículos obtenidos de SharePoint: ${rawItems.length}`);
|
|
|
|
// Filtrar desde el 21 de agosto de 2026 (00:00:00) hasta la fecha
|
|
const targetStartDate = new Date('2026-08-21T00:00:00');
|
|
const filteredItems = rawItems.filter(it => {
|
|
const dateStr = it.ArticleStartDate || it.Created;
|
|
const d = new Date(dateStr);
|
|
return d >= targetStartDate;
|
|
});
|
|
|
|
console.log(`Artículos identificados desde el 17 de septiembre: ${filteredItems.length}`);
|
|
|
|
// Ordenar cronológicamente (los más antiguos primero para que los más nuevos queden en la parte superior del portal)
|
|
filteredItems.sort((a, b) => {
|
|
const da = new Date(a.ArticleStartDate || a.Created).getTime();
|
|
const db = new Date(b.ArticleStartDate || b.Created).getTime();
|
|
return da - db;
|
|
});
|
|
|
|
// 3. Consultar slugs existentes en Appwrite para evitar duplicados
|
|
console.log("Verificando publicaciones existentes en Appwrite...");
|
|
const existingSlugs = new Set();
|
|
let cursor = null;
|
|
while (true) {
|
|
const queries = [Query.limit(100), Query.select(['slug'])];
|
|
if (cursor) queries.push(Query.cursorAfter(cursor));
|
|
const existingRes = await databases.listDocuments(DB_ID, postsCollectionId, queries);
|
|
existingRes.documents.forEach(d => {
|
|
if (d.slug) existingSlugs.add(d.slug);
|
|
});
|
|
if (existingRes.documents.length < 100) break;
|
|
cursor = existingRes.documents[existingRes.documents.length - 1].$id;
|
|
}
|
|
console.log(`Slugs existentes en posts: ${existingSlugs.size}`);
|
|
|
|
let publishedCount = 0;
|
|
let skippedCount = 0;
|
|
let errorsCount = 0;
|
|
|
|
for (let i = 0; i < filteredItems.length; i++) {
|
|
const item = filteredItems[i];
|
|
const pubDate = new Date(item.ArticleStartDate || item.Created);
|
|
const rawTitle = (item.Title || '').trim();
|
|
const slug = `gobernacion-web-${slugify(rawTitle)}`.substring(0, 95);
|
|
|
|
console.log(`\n-------------------------------------------------------------------------------`);
|
|
console.log(`[${i + 1}/${filteredItems.length}] Noticia ID: ${item.Id} | Fecha: ${pubDate.toISOString().split('T')[0]}`);
|
|
console.log(`Título: "${rawTitle}"`);
|
|
console.log(`Slug: ${slug}`);
|
|
|
|
if (existingSlugs.has(slug)) {
|
|
console.log(`-> Ya existe en Appwrite con slug: ${slug}. Se omite duplicado.`);
|
|
skippedCount++;
|
|
continue;
|
|
}
|
|
|
|
const relativeUrl = item.File?.ServerRelativeUrl;
|
|
if (!relativeUrl) {
|
|
console.error("-> Error: No se encontró URL relativa para este artículo.");
|
|
errorsCount++;
|
|
continue;
|
|
}
|
|
|
|
const pageUrl = "https://cauca.gov.co" + encodeURI(decodeURI(relativeUrl));
|
|
console.log(`-> Obteniendo contenido de página: ${pageUrl}`);
|
|
|
|
try {
|
|
const pageRes = await axios.get(pageUrl, {
|
|
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36' },
|
|
timeout: 15000
|
|
});
|
|
|
|
const $ = cheerio.load(pageRes.data);
|
|
|
|
// A. Extraer imagen destacada
|
|
let imageUrl = null;
|
|
$('.ms-rtestate-field img').each((idx, el) => {
|
|
const src = $(el).attr('src');
|
|
if (src && !src.includes('logo') && !src.includes('icon')) {
|
|
imageUrl = src.startsWith('http') ? src : 'https://cauca.gov.co' + src;
|
|
return false;
|
|
}
|
|
});
|
|
|
|
if (!imageUrl) {
|
|
$('img').each((idx, el) => {
|
|
const src = $(el).attr('src');
|
|
if (src && src.includes('/Prensa/PublishingImages/')) {
|
|
imageUrl = src.startsWith('http') ? src : 'https://cauca.gov.co' + src;
|
|
return false;
|
|
}
|
|
});
|
|
}
|
|
|
|
if (!imageUrl) {
|
|
$('img').each((idx, el) => {
|
|
const src = $(el).attr('src');
|
|
if (src && !src.includes('logo') && !src.includes('icon') && !src.includes('menu') && src.length > 10) {
|
|
imageUrl = src.startsWith('http') ? src : 'https://cauca.gov.co' + src;
|
|
return false;
|
|
}
|
|
});
|
|
}
|
|
|
|
// B. Extraer contenido HTML
|
|
let contentHtml = null;
|
|
let maxLen = 0;
|
|
$('.ms-rtestate-field').each((idx, el) => {
|
|
const textLen = $(el).text().trim().length;
|
|
if (textLen > maxLen) {
|
|
maxLen = textLen;
|
|
contentHtml = $(el).html();
|
|
}
|
|
});
|
|
|
|
if (!contentHtml) {
|
|
contentHtml = $('.lg\\:container.mx-auto.flex.items-center.justify-center.my-\\[80px\\]').html();
|
|
}
|
|
|
|
if (!contentHtml || contentHtml.trim().length === 0) {
|
|
console.error("-> Error: Contenido de la noticia vacío.");
|
|
errorsCount++;
|
|
continue;
|
|
}
|
|
|
|
const cleanedContent = cleanContentHtml(contentHtml);
|
|
const plainText = cheerio.load(cleanedContent).text().replace(/\s+/g, ' ').trim();
|
|
let summary = plainText.substring(0, 150);
|
|
if (plainText.length > 150) summary += '...';
|
|
|
|
// C. Descargar y subir imagen destacada a Appwrite Storage y Media
|
|
let mediaId = null;
|
|
if (imageUrl) {
|
|
try {
|
|
console.log(`-> Descargando imagen: ${imageUrl}`);
|
|
const imgRes = await axios.get(encodeURI(decodeURI(imageUrl)), {
|
|
responseType: 'arraybuffer',
|
|
timeout: 15000,
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'
|
|
}
|
|
});
|
|
|
|
const buffer = Buffer.from(imgRes.data, 'binary');
|
|
const isHtml = buffer.toString('utf-8', 0, 50).trim().toLowerCase().startsWith('<!doctype') || buffer.toString('utf-8', 0, 20).trim().toLowerCase().startsWith('<html');
|
|
|
|
if (!isHtml && buffer.length > 500) {
|
|
const urlParts = imageUrl.split('/');
|
|
let filename = urlParts[urlParts.length - 1] || 'imagen.jpg';
|
|
filename = decodeURIComponent(filename).replace(/[^a-zA-Z0-9._-]/g, '_');
|
|
if (!filename.includes('.')) filename += '.jpg';
|
|
|
|
const tempFilePath = path.join(os.tmpdir(), `gob_${ID.unique()}_${filename}`);
|
|
fs.writeFileSync(tempFilePath, buffer);
|
|
|
|
let fileParam;
|
|
if (InputFile && InputFile.fromPath) {
|
|
fileParam = InputFile.fromPath(tempFilePath, filename);
|
|
} else {
|
|
fileParam = fs.createReadStream(tempFilePath);
|
|
}
|
|
|
|
console.log(`-> Subiendo imagen al bucket (${BUCKET_ID})...`);
|
|
const fileUploaded = await storage.createFile(BUCKET_ID, ID.unique(), fileParam);
|
|
|
|
console.log(`-> Registrando en colección media...`);
|
|
const mediaDoc = await databases.createDocument(DB_ID, mediaCollectionId, ID.unique(), {
|
|
file_id: fileUploaded.$id,
|
|
bucket_id: BUCKET_ID,
|
|
titulo: rawTitle.substring(0, 245),
|
|
tipo: 'imagen'
|
|
});
|
|
mediaId = mediaDoc.$id;
|
|
|
|
try { fs.unlinkSync(tempFilePath); } catch (e) {}
|
|
} else {
|
|
console.log("-> Aviso: La imagen obtenida no es un binario válido.");
|
|
}
|
|
} catch (imgErr) {
|
|
console.error(`-> Error al procesar imagen: ${imgErr.message}`);
|
|
}
|
|
}
|
|
|
|
// D. Crear publicación en Appwrite
|
|
const postDoc = {
|
|
titulo: rawTitle.substring(0, 245),
|
|
slug: slug,
|
|
tipo: 'noticia',
|
|
tipo_publicacion_id: typeId,
|
|
estado: 'publicado',
|
|
fecha_publicacion: pubDate.toISOString(),
|
|
contenido: cleanedContent,
|
|
resumen_redes: summary,
|
|
seo_descripcion: summary.substring(0, 155),
|
|
seo_titulo: rawTitle.substring(0, 58),
|
|
imagen_destacada_id: mediaId
|
|
};
|
|
|
|
console.log(`-> Guardando documento en Appwrite posts...`);
|
|
const createdPost = await databases.createDocument(DB_ID, postsCollectionId, ID.unique(), postDoc);
|
|
console.log(`✓ [ÉXITO] Publicada correctamente! Post ID: ${createdPost.$id}`);
|
|
existingSlugs.add(slug);
|
|
publishedCount++;
|
|
|
|
// Pausa prudente entre peticiones
|
|
await sleep(1200);
|
|
|
|
} catch (err) {
|
|
console.error(`-> Error publicando noticia: ${err.message}`);
|
|
errorsCount++;
|
|
}
|
|
}
|
|
|
|
console.log("\n===============================================================================");
|
|
console.log(`RESUMEN FINAL:`);
|
|
console.log(`Total identificadas: ${filteredItems.length}`);
|
|
console.log(`Publicadas exitosamente: ${publishedCount}`);
|
|
console.log(`Omitidas por ya existir: ${skippedCount}`);
|
|
console.log(`Errores: ${errorsCount}`);
|
|
console.log("===============================================================================");
|
|
|
|
} catch (e) {
|
|
console.error("Error fatal en proceso de publicación:", e.message);
|
|
}
|
|
}
|
|
|
|
run();
|