fix: resolve publication detail line breaks, standardize featured image to 640x360 aspect-video, improve scraper robustness, and add placeholders
This commit is contained in:
parent
43b78c32aa
commit
80eff1a72a
45
package-lock.json
generated
45
package-lock.json
generated
@ -10,6 +10,7 @@
|
||||
"dependencies": {
|
||||
"@livekit/components-react": "^2.9.21",
|
||||
"@livekit/components-styles": "^1.2.0",
|
||||
"@tailwindcss/typography": "^0.5.20",
|
||||
"@tiptap/extension-color": "^3.21.0",
|
||||
"@tiptap/extension-font-family": "^3.21.0",
|
||||
"@tiptap/extension-highlight": "^3.21.0",
|
||||
@ -1895,6 +1896,18 @@
|
||||
"tailwindcss": "4.2.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/typography": {
|
||||
"version": "0.5.20",
|
||||
"resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.20.tgz",
|
||||
"integrity": "sha512-hwbzQuNUfcPvbegQFatVPl/MY/tcM9KLl963hQ5laJKPh81TEZ1+dNG9PirGvcaDBkp+BCshExAyKVPW91dozw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"postcss-selector-parser": "6.0.10"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"tailwindcss": ">=3.0.0 || >=4.0.0 || insiders"
|
||||
}
|
||||
},
|
||||
"node_modules/@tiptap/core": {
|
||||
"version": "3.21.0",
|
||||
"resolved": "https://registry.npmjs.org/@tiptap/core/-/core-3.21.0.tgz",
|
||||
@ -3980,6 +3993,18 @@
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/cssesc": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz",
|
||||
"integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==",
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"cssesc": "bin/cssesc"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/cssstyle": {
|
||||
"version": "6.2.0",
|
||||
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-6.2.0.tgz",
|
||||
@ -7459,6 +7484,19 @@
|
||||
"node": "^10 || ^12 || >=14"
|
||||
}
|
||||
},
|
||||
"node_modules/postcss-selector-parser": {
|
||||
"version": "6.0.10",
|
||||
"resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz",
|
||||
"integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cssesc": "^3.0.0",
|
||||
"util-deprecate": "^1.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/postcss-value-parser": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
|
||||
@ -8476,7 +8514,6 @@
|
||||
"version": "4.2.2",
|
||||
"resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.2.tgz",
|
||||
"integrity": "sha512-KWBIxs1Xb6NoLdMVqhbhgwZf2PGBpPEiwOqgI4pFIYbNTfBXiKYyWoTsXgBQ9WFg/OlhnvHaY+AEpW7wSmFo2Q==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tapable": {
|
||||
@ -8932,6 +8969,12 @@
|
||||
"react": "^16.8.0 || ^17 || ^18 || ^19 || ^19.0.0-rc"
|
||||
}
|
||||
},
|
||||
"node_modules/util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/video.js": {
|
||||
"version": "8.23.7",
|
||||
"resolved": "https://registry.npmjs.org/video.js/-/video.js-8.23.7.tgz",
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
"dependencies": {
|
||||
"@livekit/components-react": "^2.9.21",
|
||||
"@livekit/components-styles": "^1.2.0",
|
||||
"@tailwindcss/typography": "^0.5.20",
|
||||
"@tiptap/extension-color": "^3.21.0",
|
||||
"@tiptap/extension-font-family": "^3.21.0",
|
||||
"@tiptap/extension-highlight": "^3.21.0",
|
||||
|
||||
461
publish-cauca-web.js
Normal file
461
publish-cauca-web.js
Normal file
@ -0,0 +1,461 @@
|
||||
const axios = require('axios');
|
||||
const cheerio = require('cheerio');
|
||||
const fs = require('fs');
|
||||
const { Client, Databases, Storage, ID, Query } = require('node-appwrite');
|
||||
require('dotenv').config({ path: '.env.local' });
|
||||
|
||||
let InputFile;
|
||||
try { InputFile = require('node-appwrite/file').InputFile; } catch (e) { InputFile = null; }
|
||||
|
||||
const client = new Client()
|
||||
.setEndpoint(process.env.NEXT_PUBLIC_APPWRITE_ENDPOINT)
|
||||
.setProject(process.env.NEXT_PUBLIC_APPWRITE_PROJECT_ID)
|
||||
.setKey(process.env.APPWRITE_API_KEY);
|
||||
|
||||
const databases = new Databases(client);
|
||||
const storage = new Storage(client);
|
||||
const DB_ID = process.env.APPWRITE_DATABASE_ID;
|
||||
const BUCKET_ID = process.env.NEXT_PUBLIC_APPWRITE_MEDIA_BUCKET_ID;
|
||||
|
||||
// The list of 32 articles scraped from the website
|
||||
const scrapedNews = [
|
||||
{
|
||||
title: "Gobernación del Cauca lideró mesa técnica para fortalecer la respuesta ante la actividad del volcán Puracé",
|
||||
dateStr: "jueves, 20 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-lideró-mesa-técnica-para-fortalecer-la-respuesta-ante-la-actividad-del-volcán-Puracé.aspx"
|
||||
},
|
||||
{
|
||||
title: "¡Gracias Cauca! Ayudas humanitarias recolectadas superan las 80 toneladas",
|
||||
dateStr: "miércoles, 19 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/-¡Gracias-Cauca!-Ayudas-humanitarias-recolectadas-superan-las-80-toneladas.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca fortalece la conectividad vial en la vereda El Túnel, Cajibío",
|
||||
dateStr: "martes, 18 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-fortalece-la-conectividad-vial-en-la-vereda-El-Túnel,-Cajibío.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobierno Departamental mantiene seguimiento a afectaciones ocasionadas por el sismo del 10 de agosto",
|
||||
dateStr: "martes, 18 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobierno-Departamental-mantiene-seguimiento-a-afectaciones-ocasionadas-por-el-sismo-del-10-de-agosto-.aspx"
|
||||
},
|
||||
{
|
||||
title: "Más de 40 toneladas de ayudas serán entregadas en las regiones afectadas del país",
|
||||
dateStr: "viernes, 14 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Más-de-40-toneladas-de-ayudas-serán-entregadas-en-las-regiones-afectadas-del-país.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobierno departamental hace llamado vehemente al Gobierno Nacional frente a la situación de orden público que atenta contra...",
|
||||
dateStr: "jueves, 13 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobierno-departamental-hace-llamado-vehemente-al-Gobierno-Nacional-frente-a-la-situación-de-orden-público-que-atenta-contra.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca sigue consolidando afectaciones tras el sismo del 10 de agosto",
|
||||
dateStr: "jueves, 13 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-sigue-consolidando-afectaciones-tras-el-sismo-del-10-de-agosto.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca sustentó proyecto para reducir hasta el 80 % de intereses y sanciones en impuestos departamentales",
|
||||
dateStr: "jueves, 13 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-sustentó-proyecto-para-reducir-hasta-el-80--de-intereses-y-sanciones-en-impuestos-departamentales.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca activa Plan de Evaluación Integral tras sismo de 7.4 registrado en el país",
|
||||
dateStr: "lunes, 10 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-activa-Plan-de-Evaluación-Integral-tras-sismo-de-7-4-registrado-en-el-país.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca está en permanente monitoreo y atención a municipios afectados tras el sismo de 7.4",
|
||||
dateStr: "lunes, 10 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-está-en-permanente-monitoreo-y-atención-a-municipios-afectados-tras-el-sismo-de-7-4.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca desplegó acciones simultáneas frente a la alerta volcánica del Puracé, la protección de la vía Panamericana...",
|
||||
dateStr: "domingo, 9 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-desplegó-acciones-simultáneas-frente-a-la-alerta-volcánica-del-Puracé,-la-protección-de-la-vía-Paname.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca entregó ayudas humanitarias ante la emergencia generada por la actividad del volcán Puracé",
|
||||
dateStr: "miércoles, 5 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-entregó-ayudas-humanitarias-ante-la-emergencia-generada-por-la-actividad-del-volcán-Puracé.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca intensifica acciones para proteger la salud de las comunidades tras la alerta naranja del volcán Puracé",
|
||||
dateStr: "sábado, 1 de agosto de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-intensifica-acciones-para-proteger-la-salud-de-las-comunidades-tras-la-alerta-naranja-del-volcán-Pura.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca unifica sus trámites tributarios en el portal web institucional en cumplimiento de la Ley de Transparencia",
|
||||
dateStr: "martes, 28 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-unifica-sus-trámites-tributarios-en-el-portal-web-institucional-en-cumplimiento-de-la-Ley-de-Transpar.aspx"
|
||||
},
|
||||
{
|
||||
title: "Invitación rueda de prensa sobre el estado actual de la Industria Licorera del Cauca",
|
||||
dateStr: "martes, 28 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Invitación-rueda-de-prensa-sobre-el-estado-actual-de-la-Industria-Licorera-del-Cauca.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobierno Departamental fortalece la defensa de los Derechos Humanos con la Mesa Territorial de Garantías",
|
||||
dateStr: "lunes, 27 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/-Gobierno-Departamental-fortalece-la-defensa-de-los-Derechos-Humanos-con-la-Mesa-Territorial-de-Garantías--En-el-marco-del-.aspx"
|
||||
},
|
||||
{
|
||||
title: "La Afrocolombianidad se vive y se fortalece en el Cauca con La Fuerza del Pueblo",
|
||||
dateStr: "viernes, 24 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/La-Afrocolombianidad-se-vive-y-se-fortalece-en-el-Cauca-con-La-Fuerza-del-Pueblo.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca hace un llamado urgente a entidades competentes para garantizar atención en salud a más de 400.000 afiliados de Asmet Salud",
|
||||
dateStr: "jueves, 23 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-hace-un-llamado-urgente-a-entidades-competentes-para-garantizar-atención-en-salud-a-más-de-400-000-af.aspx"
|
||||
},
|
||||
{
|
||||
title: "Golpe al contrabando en el Norte del Cauca: decomisadas 586 unidades de licor",
|
||||
dateStr: "martes, 21 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/-Golpe-al-contrabando-en-el-Norte-del-Cauca-decomisadas-586-unidades-de-licor.aspx"
|
||||
},
|
||||
{
|
||||
title: "Con Escuela y Café, la Gobernación del Cauca fortalece la formación de los jóvenes del departamento",
|
||||
dateStr: "martes, 21 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Con-Escuela-y-Café,-la-Gobernación-del-Cauca-fortalece-la-formación-de-los-jóvenes-del-departamento.aspx"
|
||||
},
|
||||
{
|
||||
title: "Vías para la Vida: el sector rural de Popayán, una prioridad para el Gobierno Departamental",
|
||||
dateStr: "sábado, 11 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Vías-para-la-Vida-el-sector-rural-de-Popayán,-una-prioridad-para-el-Gobierno-Departamental.aspx"
|
||||
},
|
||||
{
|
||||
title: "La paz se construye con acciones en los territorios: Gobernación del Cauca entrega ayudas integrales en El Carmelo, Cajibío",
|
||||
dateStr: "miércoles, 8 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/La-paz-se-construye-con-acciones-en-los-territorios-Gobernación-del-Cauca-entrega-ayudas-integrales-en-El-Carmelo,-Cajibío.aspx"
|
||||
},
|
||||
{
|
||||
title: "El Cauca se destaca por su gestión administrativa a nivel nacional",
|
||||
dateStr: "lunes, 6 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/El-Cauca-se-destaca-por-su-gestión-administrative-a-nivel-nacional.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca alerta sobre enlace fraudulento para el pago del impuesto vehicular",
|
||||
dateStr: "lunes, 6 de julio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-alerta-sobre-enlace-fraudulento-para-el-pago-del-impuesto-vehicular.aspx"
|
||||
},
|
||||
{
|
||||
title: "Cauca se consolida como referente mundial del café con la VI Feria \"Cauca Sabe a Café 2026\"",
|
||||
dateStr: "viernes, 26 de junio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/-Cauca-se-consolida-como-referente-mundial-del-café-con-la-VI-Feria-Cauca-Sabe-a-Café-2026.aspx"
|
||||
},
|
||||
{
|
||||
title: "Este 26 de junio Indeportes Cauca rinde cuentas a la ciudadanía",
|
||||
dateStr: "martes, 23 de junio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/-Este-26-de-junio-Indeportes-Cauca-rinde-cuentas-a-la-ciudadanía.aspx"
|
||||
},
|
||||
{
|
||||
title: "Participación histórica fortalece la democracia en el Cauca",
|
||||
dateStr: "domingo, 21 de junio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/-Participación-histórica-fortalece-la-democracia-en-el-Cauca.aspx"
|
||||
},
|
||||
{
|
||||
title: "Con normalidad y garantías, se realiza la segunda vuelta presidencial en los 42 municipios del Cauca",
|
||||
dateStr: "domingo, 21 de junio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Con-normalidad-y-garantías,-se-realiza-la-segunda-vuelta-presidencial-en-los-42-municipios-del-Cauca.aspx"
|
||||
},
|
||||
{
|
||||
title: "Cauca fortalece garantías para un proceso electoral seguro, transparente y participativo en la segunda vuelta presidencial",
|
||||
dateStr: "miércoles, 10 de junio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Cauca-fortalece-garantías-para-un-proceso-electoral-seguro,-transparente-y-participativo-en-la-segunda-vuelta-presidencial.aspx"
|
||||
},
|
||||
{
|
||||
title: "La Industria Licorera del Cauca se vistió de oro en París",
|
||||
dateStr: "martes, 9 de junio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/La-Industria-Licorera-del-Cauca-se-vistió-de-oro-en-París.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca fortalece acciones de vigilancia sanitaria en establecimientos de estética",
|
||||
dateStr: "sábado, 6 de junio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-fortalece-acciones-de-vigilancia-sanitaria-en-establecimientos-de-estética.aspx"
|
||||
},
|
||||
{
|
||||
title: "Gobernación del Cauca fortalece el bienestar animal, la movilidad rural y la productividad campesina en Timbío",
|
||||
dateStr: "viernes, 5 de junio de 2026",
|
||||
link: "/Prensa/SaladePrensa/Paginas/Gobernación-del-Cauca-fortalece-el-bienestar-animal,-la-movilidad-rural-y-la-productividad-campesina-en-Timbío.aspx"
|
||||
}
|
||||
];
|
||||
|
||||
function slugify(text) {
|
||||
return text
|
||||
.toString()
|
||||
.toLowerCase()
|
||||
.normalize('NFD')
|
||||
.replace(/[\u0300-\u036f]/g, '') // remove accents
|
||||
.replace(/[^a-z0-9 -]/g, '') // remove invalid chars
|
||||
.replace(/\s+/g, '-') // collapse whitespace and replace by -
|
||||
.replace(/-+/g, '-') // collapse dashes
|
||||
.trim();
|
||||
}
|
||||
|
||||
function parseSpanishDate(dateStr) {
|
||||
const cleaned = dateStr.replace(/^[-\s]+/, '').trim().toLowerCase();
|
||||
const months = {
|
||||
enero: 0, febrero: 1, marzo: 2, abril: 3, mayo: 4, junio: 5,
|
||||
julio: 6, agosto: 7, septiembre: 8, octubre: 9, noviembre: 10, diciembre: 11
|
||||
};
|
||||
|
||||
const match = cleaned.match(/(?:[a-z]+,\s+)?(\d{1,2})\s+de\s+([a-z]+)\s+de\s+(\d{4})/i);
|
||||
if (match) {
|
||||
const day = parseInt(match[1], 10);
|
||||
const monthName = match[2];
|
||||
const year = parseInt(match[3], 10);
|
||||
const month = months[monthName];
|
||||
if (month !== undefined) {
|
||||
return new Date(year, month, day, 12, 0, 0);
|
||||
}
|
||||
}
|
||||
return new Date();
|
||||
}
|
||||
|
||||
function cleanContentHtml(html) {
|
||||
if (!html) return '';
|
||||
const $ = cheerio.load(html, null, false);
|
||||
|
||||
$('style, script, link, iframe, object, embed, video, audio, source, noscript').remove();
|
||||
|
||||
const allowedTags = ['p', 'b', 'i', 'strong', 'em', 'ul', 'ol', 'li', 'br', 'a'];
|
||||
|
||||
function processNode(node) {
|
||||
$(node).children().each((i, child) => {
|
||||
processNode(child);
|
||||
});
|
||||
|
||||
const tagName = node.tagName ? node.tagName.toLowerCase() : null;
|
||||
if (tagName) {
|
||||
const attribs = Object.keys(node.attribs || {});
|
||||
attribs.forEach(attr => {
|
||||
if (tagName !== 'a' || attr !== 'href') {
|
||||
$(node).removeAttr(attr);
|
||||
}
|
||||
});
|
||||
|
||||
if (!allowedTags.includes(tagName)) {
|
||||
$(node).replaceWith($(node).html() || '');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$.root().children().each((i, child) => {
|
||||
processNode(child);
|
||||
});
|
||||
|
||||
return $.html().trim();
|
||||
}
|
||||
|
||||
async function getCollectionIds() {
|
||||
const list = await databases.listCollections(DB_ID);
|
||||
const map = {};
|
||||
list.collections.forEach(c => { map[c.name.toLowerCase()] = c.$id; });
|
||||
return map;
|
||||
}
|
||||
|
||||
// Delay helper
|
||||
const sleep = ms => new Promise(res => setTimeout(res, ms));
|
||||
|
||||
async function run() {
|
||||
console.log("Starting scraping and publishing Cauca Government web news...");
|
||||
try {
|
||||
const collections = await getCollectionIds();
|
||||
const postsCollectionId = collections['posts'] || 'posts';
|
||||
const mediaCollectionId = collections['media'] || 'media';
|
||||
const pubTypesCollectionId = collections['publication types'] || collections['publication_types'] || 'publication_types';
|
||||
|
||||
// Get NOTICIA type ID
|
||||
const pubTypesRes = await databases.listDocuments(DB_ID, pubTypesCollectionId);
|
||||
const typeMap = {};
|
||||
pubTypesRes.documents.forEach(d => {
|
||||
typeMap[d.codigo] = d.$id;
|
||||
});
|
||||
const typeId = typeMap['NOTICIA'];
|
||||
if (!typeId) {
|
||||
console.error("Error: Could not find 'NOTICIA' publication type in database.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Using Publication Type ID: ${typeId}`);
|
||||
|
||||
// Fetch existing slugs to avoid duplicate publications
|
||||
console.log("Fetching existing slugs from Appwrite...");
|
||||
const existingSlugs = new Set();
|
||||
let cursor = null;
|
||||
while (true) {
|
||||
const queries = [Query.limit(100), Query.select(['slug'])];
|
||||
if (cursor) queries.push(Query.cursorAfter(cursor));
|
||||
const existingRes = await databases.listDocuments(DB_ID, postsCollectionId, queries);
|
||||
|
||||
existingRes.documents.forEach(d => {
|
||||
if (d.slug) existingSlugs.add(d.slug);
|
||||
});
|
||||
|
||||
if (existingRes.documents.length < 100) break;
|
||||
cursor = existingRes.documents[existingRes.documents.length - 1].$id;
|
||||
}
|
||||
console.log(`Found ${existingSlugs.size} existing slugs in posts collection.`);
|
||||
|
||||
let processedCount = 0;
|
||||
let publishedCount = 0;
|
||||
|
||||
for (let i = 0; i < scrapedNews.length; i++) {
|
||||
const item = scrapedNews[i];
|
||||
processedCount++;
|
||||
console.log(`\n[${processedCount}/${scrapedNews.length}] Processing: "${item.title}"`);
|
||||
|
||||
const slug = `gobernacion-web-${slugify(item.title)}`;
|
||||
if (existingSlugs.has(slug)) {
|
||||
console.log(`-> Skipped (already published with slug: ${slug})`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = 'https://cauca.gov.co' + item.link;
|
||||
console.log(`-> Fetching detail page: ${url}`);
|
||||
|
||||
try {
|
||||
const res = await axios.get(encodeURI(url), {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'
|
||||
},
|
||||
timeout: 10000
|
||||
});
|
||||
|
||||
const $ = cheerio.load(res.data);
|
||||
|
||||
// 1. Get featured image
|
||||
let image = null;
|
||||
// Check first inside main body content
|
||||
$('.ms-rtestate-field img').each((i, el) => {
|
||||
const src = $(el).attr('src');
|
||||
if (src && !src.includes('logo') && !src.includes('icon')) {
|
||||
image = src.startsWith('http') ? src : 'https://cauca.gov.co' + src;
|
||||
return false; // break loop
|
||||
}
|
||||
});
|
||||
|
||||
// Fallback to any image containing /Prensa/PublishingImages/
|
||||
if (!image) {
|
||||
$('img').each((i, el) => {
|
||||
const src = $(el).attr('src');
|
||||
if (src && src.includes('/Prensa/PublishingImages/')) {
|
||||
image = src.startsWith('http') ? src : 'https://cauca.gov.co' + src;
|
||||
return false; // break loop
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Final fallback: any image that is likely the article image
|
||||
if (!image) {
|
||||
$('img').each((i, el) => {
|
||||
const src = $(el).attr('src');
|
||||
if (src && !src.includes('logo') && !src.includes('icon') && !src.includes('menu') && src.length > 10) {
|
||||
image = src.startsWith('http') ? src : 'https://cauca.gov.co' + src;
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 2. Get body content
|
||||
let contentHtml = null;
|
||||
let maxLen = 0;
|
||||
$('.ms-rtestate-field').each((idx, el) => {
|
||||
const textLen = $(el).text().trim().length;
|
||||
if (textLen > maxLen) {
|
||||
maxLen = textLen;
|
||||
contentHtml = $(el).html();
|
||||
}
|
||||
});
|
||||
|
||||
if (!contentHtml) {
|
||||
// Fallback to container
|
||||
contentHtml = $('.lg\\:container.mx-auto.flex.items-center.justify-center.my-\\[80px\\]').html();
|
||||
}
|
||||
|
||||
if (!contentHtml || contentHtml.trim().length === 0) {
|
||||
console.error("-> Failed: Empty or missing body content!");
|
||||
continue;
|
||||
}
|
||||
|
||||
const cleanedContent = cleanContentHtml(contentHtml);
|
||||
const publicationDate = parseSpanishDate(item.dateStr);
|
||||
|
||||
// 3. Upload featured image if found
|
||||
let mediaId = null;
|
||||
if (image) {
|
||||
try {
|
||||
console.log(`-> Downloading image: ${image}`);
|
||||
const imgRes = await axios.get(encodeURI(image), { responseType: 'arraybuffer', timeout: 10000 });
|
||||
const buffer = Buffer.from(imgRes.data, 'binary');
|
||||
|
||||
// Extract filename from URL
|
||||
const urlParts = image.split('/');
|
||||
const filename = urlParts[urlParts.length - 1] || 'image.jpg';
|
||||
const filePath = `/tmp/${ID.unique()}_${filename}`;
|
||||
fs.writeFileSync(filePath, buffer);
|
||||
|
||||
let fileParam;
|
||||
if (InputFile && InputFile.fromPath) fileParam = InputFile.fromPath(filePath, filename);
|
||||
else fileParam = fs.createReadStream(filePath);
|
||||
|
||||
console.log("-> Uploading image to Appwrite storage...");
|
||||
const file = await storage.createFile(BUCKET_ID, ID.unique(), fileParam);
|
||||
|
||||
console.log("-> Creating media document in database...");
|
||||
const mediaDoc = await databases.createDocument(DB_ID, mediaCollectionId, ID.unique(), {
|
||||
file_id: file.$id,
|
||||
bucket_id: BUCKET_ID,
|
||||
titulo: item.title,
|
||||
tipo: 'imagen'
|
||||
});
|
||||
mediaId = mediaDoc.$id;
|
||||
|
||||
// Cleanup temp file
|
||||
try { fs.unlinkSync(filePath); } catch (e) {}
|
||||
} catch (err) {
|
||||
console.error('-> Error uploading image:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Create post document
|
||||
const plainText = cheerio.load(cleanedContent).text().replace(/\s+/g, ' ').trim();
|
||||
let summary = plainText.substring(0, 150);
|
||||
if (plainText.length > 150) summary += '...';
|
||||
|
||||
const doc = {
|
||||
titulo: item.title.substring(0, 245),
|
||||
slug: slug.substring(0, 95),
|
||||
tipo: 'noticia',
|
||||
tipo_publicacion_id: typeId,
|
||||
estado: 'publicado',
|
||||
fecha_publicacion: publicationDate.toISOString(),
|
||||
contenido: cleanedContent,
|
||||
resumen_redes: summary,
|
||||
seo_descripcion: summary.substring(0, 155),
|
||||
seo_titulo: item.title.substring(0, 58),
|
||||
imagen_destacada_id: mediaId
|
||||
};
|
||||
|
||||
console.log(`-> Publishing post: "${doc.titulo}" (${doc.slug})`);
|
||||
const publishedDoc = await databases.createDocument(DB_ID, postsCollectionId, ID.unique(), doc);
|
||||
console.log(`[Success] Published! ID: ${publishedDoc.$id}`);
|
||||
publishedCount++;
|
||||
|
||||
// Small delay to be polite
|
||||
await sleep(1500);
|
||||
|
||||
} catch (e) {
|
||||
console.error(`-> Error processing article:`, e.message);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nAll done! Processed: ${processedCount}/${scrapedNews.length} articles. Successfully published: ${publishedCount} new articles.`);
|
||||
|
||||
} catch (e) {
|
||||
console.error("Fatal Error:", e.message);
|
||||
}
|
||||
}
|
||||
|
||||
run();
|
||||
@ -2,7 +2,7 @@ import { APPWRITE_ALMEIRA_CONFIG } from '@/lib/appwrite';
|
||||
import { createAdminClient } from '@/lib/appwrite.server';
|
||||
import { Query } from 'node-appwrite';
|
||||
import { GlobalSidebar, hasSidebarContent } from '@/components/AdWrappers';
|
||||
import { Calendar, User, ArrowLeft } from 'lucide-react';
|
||||
import { Calendar, User, ArrowLeft, Newspaper } from 'lucide-react';
|
||||
import Link from 'next/link';
|
||||
import { decodeHtmlEntities } from '@/lib/utils';
|
||||
import DOMPurify from 'isomorphic-dompurify';
|
||||
@ -228,7 +228,7 @@ export default async function NoticiaDetail({ params }: { params: Promise<{ slug
|
||||
const client = await createAdminClient();
|
||||
const mediaRes = await client.databases.getDocument(APPWRITE_ALMEIRA_CONFIG.databaseId, 'media', post.imagen_destacada_id);
|
||||
if (mediaRes) {
|
||||
imagenUrl = `${ENDPOINT}/storage/buckets/${mediaRes.bucket_id}/files/${mediaRes.file_id}/view?project=${PROJECT_ID}&width=1200&height=600&quality=90`;
|
||||
imagenUrl = `${ENDPOINT}/storage/buckets/${mediaRes.bucket_id}/files/${mediaRes.file_id}/preview?project=${PROJECT_ID}&width=640&height=360&quality=90`;
|
||||
}
|
||||
} catch(e) {}
|
||||
}
|
||||
@ -283,11 +283,17 @@ export default async function NoticiaDetail({ params }: { params: Promise<{ slug
|
||||
}} />
|
||||
</div>
|
||||
) : imagenUrl ? (
|
||||
<div className="relative aspect-[21/9] w-full rounded-3xl overflow-hidden mb-12 border border-white/10 shadow-2xl">
|
||||
<div className="relative aspect-video w-full rounded-3xl overflow-hidden mb-12 border border-white/10 shadow-2xl bg-black">
|
||||
<img src={imagenUrl} alt={post.titulo} className="w-full h-full object-cover" />
|
||||
<div className="absolute inset-0 bg-gradient-to-t from-slate-950/50 to-transparent"></div>
|
||||
</div>
|
||||
) : null}
|
||||
) : (
|
||||
<div className="relative aspect-video w-full rounded-3xl overflow-hidden mb-12 border border-white/10 shadow-2xl bg-gradient-to-br from-slate-900 to-slate-950 flex flex-col items-center justify-center gap-4">
|
||||
<div className="absolute inset-0 bg-[radial-gradient(circle_at_center,_var(--accent-glow))] opacity-5 pointer-events-none"></div>
|
||||
<Newspaper size={48} className="text-cyan-500/40 animate-pulse" />
|
||||
<span className="text-slate-500 font-bold uppercase tracking-widest text-xs font-outfit">Almeira Web</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div
|
||||
className="prose prose-invert prose-cyan max-w-none text-slate-300 leading-relaxed text-lg whitespace-pre-wrap break-words
|
||||
|
||||
@ -140,7 +140,7 @@ export default async function ClasificadosPage() {
|
||||
const mediaObj = post.resolved_media || (typeof post.imagen_destacada_id === 'object' ? post.imagen_destacada_id : null) as MediaItem | null;
|
||||
|
||||
if (mediaObj) {
|
||||
imagenUrl = `${ENDPOINT}/storage/buckets/${mediaObj.bucket_id}/files/${mediaObj.file_id}/preview?project=${PROJECT_ID}&width=600&height=400&quality=80`;
|
||||
imagenUrl = `${ENDPOINT}/storage/buckets/${mediaObj.bucket_id}/files/${mediaObj.file_id}/preview?project=${PROJECT_ID}&width=640&height=360&quality=80`;
|
||||
}
|
||||
|
||||
return (
|
||||
|
||||
@ -140,7 +140,7 @@ export default async function EventosPage() {
|
||||
const mediaObj = post.resolved_media || (typeof post.imagen_destacada_id === 'object' ? post.imagen_destacada_id : null) as MediaItem | null;
|
||||
|
||||
if (mediaObj) {
|
||||
imagenUrl = `${ENDPOINT}/storage/buckets/${mediaObj.bucket_id}/files/${mediaObj.file_id}/preview?project=${PROJECT_ID}&width=600&height=400&quality=80`;
|
||||
imagenUrl = `${ENDPOINT}/storage/buckets/${mediaObj.bucket_id}/files/${mediaObj.file_id}/preview?project=${PROJECT_ID}&width=640&height=360&quality=80`;
|
||||
}
|
||||
|
||||
return (
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
@import "tailwindcss";
|
||||
@plugin "@tailwindcss/typography";
|
||||
|
||||
|
||||
@layer base {
|
||||
:root {
|
||||
|
||||
@ -220,7 +220,7 @@ export default async function Home(props: { searchParams: Promise<{ [key: string
|
||||
const mediaObj = post.resolved_media || (typeof post.imagen_destacada_id === 'object' ? post.imagen_destacada_id : null) as MediaItem | null;
|
||||
|
||||
if (mediaObj) {
|
||||
imagenUrl = `${ENDPOINT}/storage/buckets/${mediaObj.bucket_id}/files/${mediaObj.file_id}/preview?project=${PROJECT_ID}&width=800&height=500&quality=85`;
|
||||
imagenUrl = `${ENDPOINT}/storage/buckets/${mediaObj.bucket_id}/files/${mediaObj.file_id}/preview?project=${PROJECT_ID}&width=640&height=360&quality=85`;
|
||||
}
|
||||
|
||||
return (
|
||||
|
||||
@ -31,7 +31,7 @@ export const NewsCard = ({
|
||||
showFooterButton = false,
|
||||
footerText,
|
||||
isAdmin = false,
|
||||
imageAspect = 'aspect-[16/10]'
|
||||
imageAspect = 'aspect-video'
|
||||
}: NewsCardProps) => {
|
||||
const titulo = decodeHtmlEntities(rawTitulo);
|
||||
|
||||
@ -74,8 +74,9 @@ export const NewsCard = ({
|
||||
<img src={imagenUrl} alt={titulo} className="relative z-10 w-full h-full object-contain transition-transform duration-700 group-hover:scale-105" />
|
||||
</>
|
||||
) : (
|
||||
<div className="w-full h-full flex items-center justify-center bg-[radial-gradient(ellipse_at_center,_var(--tw-gradient-stops))] from-slate-800 to-slate-900">
|
||||
{/* No placeholder text as per user request */}
|
||||
<div className="w-full h-full flex items-center justify-center bg-gradient-to-br from-slate-900 to-slate-950 relative">
|
||||
<div className="absolute inset-0 bg-[radial-gradient(circle_at_center,_var(--accent-glow))] opacity-5 pointer-events-none"></div>
|
||||
<span className="text-[10px] font-black uppercase tracking-widest text-slate-700">Almeira Web</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
|
||||
@ -36,7 +36,7 @@ function resolveImageUrl(
|
||||
|
||||
if (typeof post.imagen_destacada_id === 'object') {
|
||||
const { bucket_id, file_id } = post.imagen_destacada_id;
|
||||
return `${endpoint}/storage/buckets/${bucket_id}/files/${file_id}/preview?project=${projectId}&width=600&height=400&quality=80`;
|
||||
return `${endpoint}/storage/buckets/${bucket_id}/files/${file_id}/preview?project=${projectId}&width=640&height=360&quality=80`;
|
||||
}
|
||||
|
||||
// If it's a string, we can't generate a valid URL because it's a document ID, not a file ID.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user