/** * Function Node — Normalize & Dedupe (multi-item, no require) * Memproses SEMUA items dari IMAP/HTML node dan mengeluarkan jumlah item yang sama. * Output per item: * { title, snippet, bodyText, slug, messageId, sentAt, fromAddress, sourceUrl, debugFields } */ /* ---------- Helpers (dipakai di tiap item) ---------- */ function toISO(d) { const dt = d ? new Date(d) : new Date(); return isNaN(dt) ? new Date().toISOString() : dt.toISOString(); } function slugify(s) { return String(s || '') .trim().toLowerCase() .replace(/\s+/g, '-') // spasi -> - .replace(/[^a-z0-9-]/g, '') // buang non URL-safe .replace(/-+/g, '-') // rapikan multiple - .replace(/^-+|-+$/g, ''); // trim - di awal/akhir } function limit(str, max) { const s = String(str || '').replace(/\s+/g, ' ').trim(); return s.length <= max ? s : s.slice(0, max - 1).trim() + '…'; } function firstUrlFrom(text) { const s = String(text || ''); const m = s.match(/https?:\/\/[^\s<>"'`]+/i); return m ? m[0] : ''; } function extractFromAddress(from) { if (!from) return ''; if (typeof from === 'string') { const m = from.match(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i); return m ? m[0] : from.trim(); } if (from.value && Array.isArray(from.value) && from.value[0]?.address) { return from.value[0].address; } if (from.text) { const m = String(from.text).match(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i); return m ? m[0] : String(from.text).trim(); } if (from.address) return String(from.address); return ''; } function stripHtmlToText(html, fallbackText = '') { let h = String(html || ''); if (!h) return String(fallbackText || ''); h = h.replace(//gi, '') .replace(//gi, ''); let t = h.replace(/<\/?[^>]+>/g, ' '); return t.replace(/\s+/g, ' ').trim(); } function simpleHash(str) { let hash = 0; const s = String(str || ''); for (let i = 0; i < s.length; i++) { hash = (hash << 5) - hash + s.charCodeAt(i); hash |= 0; // 32-bit } return Math.abs(hash).toString(16); } /* ---------- PROSES SEMUA ITEM ---------- */ const titleMax = 140; const snippetMax = 260; return items.map((item) => { const j = item.json || {}; // 1) SUBJECT (multi-jalur) const subject = j.subject ?? j.headers?.subject ?? j.message?.subject ?? '(no subject)'; // 2) BODY (multi-jalur) — dukung hasil HTML node juga (mis. bodyText) const htmlCandidates = [ j.bodyTextHtml, // beberapa node j.textHtml, j.html, j.body?.html, j.message?.html, ]; const textCandidates = [ j.bodyText, // kalau HTML node menaruh hasil disini j.textPlain, j.text, j.body?.text, j.message?.text, ]; const htmlSrc = htmlCandidates.find(v => v && String(v).trim().length > 0); let text = textCandidates.find(v => v && String(v).trim().length > 0) || ''; if (htmlSrc) { text = stripHtmlToText(htmlSrc, text); } else { text = String(text).replace(/\s+/g, ' ').trim(); } // 3) DATE (multi-jalur) const rawDate = j.date ?? j.internalDate ?? j.headers?.date ?? j.message?.date ?? new Date().toISOString(); const sentAt = toISO(rawDate); // 4) FROM (multi-jalur) const fromCandidate = j.from ?? j.headers?.from ?? j.message?.from ?? ''; const fromAddress = extractFromAddress(fromCandidate); // 5) Title/Slug/Snippet const title = limit(String(subject || '(no subject)').trim(), titleMax); const slug = slugify(title); const snippet = limit(text, snippetMax); // 6) URL pertama (opsional) const sourceUrl = firstUrlFrom(text) || ''; // 7) messageId (fallback hash) let messageId = (j.messageId ?? j.headers?.['message-id'] ?? j.message?.messageId ?? '').toString().trim(); if (!messageId) { messageId = simpleHash(`${title}__${sentAt}`); } // 8) Debug asal field (opsional, boleh dihapus nanti) const debugFields = { subjectPickedFrom: subject === j.subject ? 'subject' : subject === j.headers?.subject ? 'headers.subject' : subject === j.message?.subject ? 'message.subject' : '(default)', bodyPickedFrom: htmlSrc ? 'HTML→text' : (textCandidates.find(v => v && String(v).trim().length > 0) ? 'plain text' : '(none)'), datePickedFrom: rawDate === j.date ? 'date' : rawDate === j.internalDate ? 'internalDate' : rawDate === j.headers?.date ? 'headers.date' : rawDate === j.message?.date ? 'message.date' : '(default now)', fromPickedFrom: typeof fromCandidate === 'string' ? '(string)' : fromCandidate?.value ? 'from.value[0]' : fromCandidate?.text ? 'from.text' : '(unknown)', }; return { json: { title, snippet, bodyText: text, slug, messageId, sentAt, fromAddress, sourceUrl, debugFields, // hapus kalau sudah yakin } }; });

Ariyanto Catur Pamungkas

Email to Notion Knowledge Base with IMAP, Postgres Dedup and Telegram Alert