5c76ae18bb
Extrae texto del PDF/DOCX en Fuentes del kit de marca para no depender de pegar el contenido a mano. Co-authored-by: Cursor <cursoragent@cursor.com>
271 lines
8.0 KiB
TypeScript
271 lines
8.0 KiB
TypeScript
/**
|
|
* Extract plain text from brandbook source uploads (txt/md/csv/json/pdf/docx).
|
|
* No heavy native deps — PDF via FlateDecode streams; DOCX via zip + document.xml.
|
|
*/
|
|
import { inflateRawSync, inflateSync } from 'zlib';
|
|
|
|
const MAX_CHARS = 200_000;
|
|
|
|
function decodePdfStringLiteral(raw: string): string {
|
|
// PDF literal strings: \( \) \\ \n \r \t \b \f \ddd octal
|
|
let out = '';
|
|
for (let i = 0; i < raw.length; i++) {
|
|
const c = raw[i];
|
|
if (c !== '\\') {
|
|
out += c;
|
|
continue;
|
|
}
|
|
const n = raw[i + 1];
|
|
if (n === 'n') {
|
|
out += '\n';
|
|
i++;
|
|
} else if (n === 'r') {
|
|
out += '\r';
|
|
i++;
|
|
} else if (n === 't') {
|
|
out += '\t';
|
|
i++;
|
|
} else if (n === 'b' || n === 'f') {
|
|
i++;
|
|
} else if (n === '(' || n === ')' || n === '\\') {
|
|
out += n;
|
|
i++;
|
|
} else if (n && /[0-7]/.test(n)) {
|
|
let oct = n;
|
|
i++;
|
|
if (raw[i + 1] && /[0-7]/.test(raw[i + 1])) {
|
|
oct += raw[i + 1];
|
|
i++;
|
|
}
|
|
if (raw[i + 1] && /[0-7]/.test(raw[i + 1])) {
|
|
oct += raw[i + 1];
|
|
i++;
|
|
}
|
|
out += String.fromCharCode(parseInt(oct, 8));
|
|
} else if (n) {
|
|
out += n;
|
|
i++;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function decodePdfHexString(hex: string): string {
|
|
const clean = hex.replace(/[^0-9a-fA-F]/g, '');
|
|
let out = '';
|
|
for (let i = 0; i + 1 < clean.length; i += 2) {
|
|
const code = parseInt(clean.slice(i, i + 2), 16);
|
|
if (code >= 32 && code !== 127) out += String.fromCharCode(code);
|
|
else if (code === 10 || code === 13) out += '\n';
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function extractTextFromPdfContent(content: string): string {
|
|
const parts: string[] = [];
|
|
// Tj / ' / " operators with literal or hex strings
|
|
const re =
|
|
/(\((?:\\.|[^\\)])*\))\s*Tj|(\((?:\\.|[^\\)])*\))\s*'|\[((?:[^\[\]]|\[[^\]]*\])*)\]\s*TJ|<([0-9A-Fa-f\s]+)>\s*Tj/g;
|
|
let m: RegExpExecArray | null;
|
|
while ((m = re.exec(content))) {
|
|
if (m[1] || m[2]) {
|
|
const lit = (m[1] || m[2]).slice(1, -1);
|
|
parts.push(decodePdfStringLiteral(lit));
|
|
} else if (m[3]) {
|
|
const arr = m[3];
|
|
const inner = /\((?:\\.|[^\\)])*\)|<([0-9A-Fa-f\s]+)>/g;
|
|
let im: RegExpExecArray | null;
|
|
while ((im = inner.exec(arr))) {
|
|
if (im[0].startsWith('(')) {
|
|
parts.push(decodePdfStringLiteral(im[0].slice(1, -1)));
|
|
} else if (im[1]) {
|
|
parts.push(decodePdfHexString(im[1]));
|
|
}
|
|
}
|
|
parts.push(' ');
|
|
} else if (m[4]) {
|
|
parts.push(decodePdfHexString(m[4]));
|
|
}
|
|
}
|
|
return parts.join('');
|
|
}
|
|
|
|
function inflatePdfStream(data: Buffer): Buffer | null {
|
|
try {
|
|
return inflateSync(data);
|
|
} catch {
|
|
try {
|
|
return inflateRawSync(data);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Best-effort text extraction from PDF bytes. */
|
|
export function extractTextFromPdf(buffer: Buffer): string {
|
|
const latin = buffer.toString('latin1');
|
|
const chunks: string[] = [];
|
|
|
|
// Stream objects: ... stream\nDATA\nendstream
|
|
const streamRe = /stream\r?\n([\s\S]*?)endstream/g;
|
|
let sm: RegExpExecArray | null;
|
|
while ((sm = streamRe.exec(latin))) {
|
|
const before = latin.slice(Math.max(0, sm.index - 400), sm.index);
|
|
const isFlate = /\/Filter\s*\/FlateDecode|\/Filter\s*\[\s*\/FlateDecode/i.test(
|
|
before
|
|
);
|
|
const raw = Buffer.from(sm[1], 'latin1');
|
|
// Trim possible trailing \r\n that belongs to endstream delimiter
|
|
let payload = raw;
|
|
if (payload.length >= 2 && payload[payload.length - 2] === 0x0d) {
|
|
payload = payload.subarray(0, payload.length - 2);
|
|
} else if (payload.length >= 1 && payload[payload.length - 1] === 0x0a) {
|
|
payload = payload.subarray(0, payload.length - 1);
|
|
}
|
|
|
|
let content = '';
|
|
if (isFlate) {
|
|
const inflated = inflatePdfStream(payload);
|
|
if (!inflated) continue;
|
|
content = inflated.toString('latin1');
|
|
} else {
|
|
content = payload.toString('latin1');
|
|
}
|
|
const text = extractTextFromPdfContent(content);
|
|
if (text.trim()) chunks.push(text);
|
|
}
|
|
|
|
// Also scan uncompressed page content outside streams (rare)
|
|
const direct = extractTextFromPdfContent(latin);
|
|
if (direct.trim().length > 40) chunks.push(direct);
|
|
|
|
const joined = chunks
|
|
.join('\n')
|
|
.replace(/\r/g, '\n')
|
|
.replace(/[ \t]+\n/g, '\n')
|
|
.replace(/\n{3,}/g, '\n\n')
|
|
.replace(/[^\S\n]{2,}/g, ' ')
|
|
.trim();
|
|
|
|
return joined.slice(0, MAX_CHARS);
|
|
}
|
|
|
|
/** Minimal ZIP local-file reader (enough for DOCX document.xml). */
|
|
function zipReadFile(buffer: Buffer, pathInZip: string): Buffer | null {
|
|
let offset = 0;
|
|
while (offset + 30 <= buffer.length) {
|
|
if (buffer.readUInt32LE(offset) !== 0x04034b50) break;
|
|
const method = buffer.readUInt16LE(offset + 8);
|
|
const compSize = buffer.readUInt32LE(offset + 18);
|
|
const nameLen = buffer.readUInt16LE(offset + 26);
|
|
const extraLen = buffer.readUInt16LE(offset + 28);
|
|
const name = buffer
|
|
.subarray(offset + 30, offset + 30 + nameLen)
|
|
.toString('utf8');
|
|
const dataStart = offset + 30 + nameLen + extraLen;
|
|
const data = buffer.subarray(dataStart, dataStart + compSize);
|
|
offset = dataStart + compSize;
|
|
if (name !== pathInZip) continue;
|
|
if (method === 0) return Buffer.from(data);
|
|
if (method === 8) {
|
|
try {
|
|
return inflateRawSync(data);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function extractTextFromDocx(buffer: Buffer): string {
|
|
const xml = zipReadFile(buffer, 'word/document.xml');
|
|
if (!xml) {
|
|
throw new Error(
|
|
'No se pudo leer el DOCX (¿archivo corrupto o protegido?). Prueba a pegar el texto.'
|
|
);
|
|
}
|
|
const raw = xml.toString('utf8');
|
|
const text = raw
|
|
.replace(/<w:tab\/>/g, '\t')
|
|
.replace(/<w:br\/>/g, '\n')
|
|
.replace(/<\/w:p>/g, '\n\n')
|
|
.replace(/<[^>]+>/g, '')
|
|
.replace(/&/g, '&')
|
|
.replace(/</g, '<')
|
|
.replace(/>/g, '>')
|
|
.replace(/"/g, '"')
|
|
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
|
.replace(/[ \t]+\n/g, '\n')
|
|
.replace(/\n{3,}/g, '\n\n')
|
|
.trim();
|
|
return text.slice(0, MAX_CHARS);
|
|
}
|
|
|
|
export type ExtractableExt = 'txt' | 'md' | 'csv' | 'json' | 'pdf' | 'docx';
|
|
|
|
export function detectSourceExt(filename: string, mime?: string): ExtractableExt | null {
|
|
const lower = filename.toLowerCase();
|
|
if (lower.endsWith('.pdf') || mime === 'application/pdf') return 'pdf';
|
|
if (
|
|
lower.endsWith('.docx') ||
|
|
mime ===
|
|
'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
|
)
|
|
return 'docx';
|
|
if (lower.endsWith('.txt') || mime === 'text/plain') return 'txt';
|
|
if (lower.endsWith('.md') || mime === 'text/markdown') return 'md';
|
|
if (lower.endsWith('.csv') || mime === 'text/csv') return 'csv';
|
|
if (lower.endsWith('.json') || mime === 'application/json') return 'json';
|
|
return null;
|
|
}
|
|
|
|
export async function extractSourceText(opts: {
|
|
filename: string;
|
|
mime?: string;
|
|
buffer?: Buffer;
|
|
textFallback?: string;
|
|
}): Promise<{ text: string; filename: string }> {
|
|
const ext = detectSourceExt(opts.filename, opts.mime);
|
|
const filename = opts.filename || `fuente.${ext || 'txt'}`;
|
|
|
|
if (!ext || ext === 'txt' || ext === 'md' || ext === 'csv' || ext === 'json') {
|
|
if (opts.textFallback != null) {
|
|
return { text: opts.textFallback.slice(0, MAX_CHARS), filename };
|
|
}
|
|
if (opts.buffer) {
|
|
return {
|
|
text: opts.buffer.toString('utf8').slice(0, MAX_CHARS),
|
|
filename,
|
|
};
|
|
}
|
|
throw new Error('Sin texto');
|
|
}
|
|
|
|
if (!opts.buffer) throw new Error('Falta el archivo');
|
|
|
|
if (ext === 'pdf') {
|
|
const text = extractTextFromPdf(opts.buffer);
|
|
if (!text.trim() || text.trim().length < 20) {
|
|
throw new Error(
|
|
'No se pudo extraer texto del PDF (puede ser escaneado o protegido). Copia el texto y pégalo en el campo.'
|
|
);
|
|
}
|
|
return { text, filename };
|
|
}
|
|
|
|
if (ext === 'docx') {
|
|
const text = extractTextFromDocx(opts.buffer);
|
|
if (!text.trim()) {
|
|
throw new Error(
|
|
'El DOCX no contiene texto legible. Copia el contenido y pégalo en el campo.'
|
|
);
|
|
}
|
|
return { text, filename };
|
|
}
|
|
|
|
throw new Error('Formato no soportado');
|
|
}
|