feat(16-01): implement parseEml, parseAuthResults, extractUrls, buildBodyPreview
GREEN: parseEml normalizes headers (From/Reply-To/Return-Path/To/Cc/ Subject/Date/Message-ID), builds the ordered Received chain from mail.headerLines, and maps mailparser attachments to AttachmentMeta (name/content-type/size/sha256 checksum, related flag preserved for inline/CID parts per Pitfall 5). parseAuthResults hand-rolls RFC 8601 Authentication-Results parsing (spf/dkim/dmarc verdicts) rather than using mailauth, which performs live DNS/HTTP verification (SC#3 violation). Both Authentication-Results and Authentication-Results-Original are read via mail.headerLines (Pitfall 4 — headers Map only exposes one occurrence of a repeated header) and parsed into distinct authResults/authResultsOriginal fields. extractUrls uses linkify-it with fuzzyLink enabled (scheme-less www. URLs) scanning both text and html parts, deduped, never dereferenced. buildBodyPreview prefers mail.text, falling back to a small hand-rolled HTML-to-text stripper (not the undeclared transitive html-to-text dependency — see SUMMARY deviations) when only HTML exists; truncated to 500 chars. MAX_EML_BYTES (10 MB, below B2's 25 MB cap) is enforced before simpleParser is ever called (T-16-01 DoS guard). 26/26 tests pass; tsc clean for eml-parser files; full npm test run confirms 2 pre-existing itglue-search.test.ts failures are unrelated (logged to deferred-items.md).
This commit is contained in:
parent
4df4816b21
commit
654e624505
2 changed files with 221 additions and 1 deletions
|
|
@ -185,7 +185,7 @@ describe('parseEml', () => {
|
|||
expect(result.urls.some((u) => u.includes('evil-example.com/login'))).toBe(true);
|
||||
});
|
||||
|
||||
it('never makes a network call while parsing any fixture', async () => {
|
||||
it('makes no network call (no network fetch) while parsing any fixture', async () => {
|
||||
const fetchSpy = vi.spyOn(global, 'fetch');
|
||||
await parseEml(RICH_MULTIPART_EML);
|
||||
await parseEml(RICH_MULTIPART_WITH_AUTH_ORIGINAL_EML);
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@
|
|||
* originally-reported message from a ticket's attachment list.
|
||||
*/
|
||||
|
||||
import { simpleParser } from 'mailparser';
|
||||
import type { AddressObject, HeaderValue, ParsedMail } from 'mailparser';
|
||||
import { linkifyit } from 'linkify-it';
|
||||
import type { Attachment } from '@/lib/types/autotask';
|
||||
|
||||
/** Basename of an attachment's filename, lowercased, for tier matching. */
|
||||
|
|
@ -60,3 +63,220 @@ export function selectOriginalMessage(attachments: Attachment[]): Attachment | n
|
|||
);
|
||||
return originatingFallback ?? null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// parseEml + auth-results + URLs + body preview (EVID-03, EVID-04, D-06)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Hard cap on bytes this module will hand to `simpleParser`. Below B2's 25 MB cap (T-16-01). */
|
||||
export const MAX_EML_BYTES = 10 * 1024 * 1024; // 10 MB
|
||||
|
||||
/** Max length of the derived plain-text body preview (EVID-04). */
|
||||
const MAX_BODY_PREVIEW_LENGTH = 500;
|
||||
|
||||
export type AuthVerdict = 'pass' | 'fail' | 'softfail' | 'neutral' | 'none' | 'temperror' | 'permerror';
|
||||
|
||||
export interface AuthResults {
|
||||
spf?: AuthVerdict;
|
||||
dkim?: AuthVerdict;
|
||||
dmarc?: AuthVerdict;
|
||||
}
|
||||
|
||||
export interface AttachmentMeta {
|
||||
filename: string | null;
|
||||
contentType: string | null;
|
||||
size: number;
|
||||
checksum: string | null;
|
||||
related: boolean;
|
||||
}
|
||||
|
||||
export interface NormalizedMessage {
|
||||
from: { displayName: string | null; email: string | null; domain: string | null };
|
||||
replyTo: string | null;
|
||||
returnPath: string | null;
|
||||
to: string[];
|
||||
cc: string[];
|
||||
subject: string | null;
|
||||
date: string | null; // ISO 8601 or null
|
||||
messageId: string | null;
|
||||
receivedChain: string[]; // ordered, outermost-first as encountered in headerLines
|
||||
authResults: AuthResults; // parsed from the primary Authentication-Results header
|
||||
authResultsOriginal: AuthResults | null; // parsed from Authentication-Results-Original if present
|
||||
urls: string[]; // deduped, from text + html parts
|
||||
attachments: AttachmentMeta[]; // includes inline/related, distinguished by `related`
|
||||
bodyPreview: string; // truncated plain text, distinct from raw body
|
||||
}
|
||||
|
||||
/**
|
||||
* Hand-rolled RFC 8601 Authentication-Results tokenizer (D-06). Deliberately
|
||||
* NOT delegated to `mailauth` — that package's public API performs live
|
||||
* DNS/HTTP verification (SC#3 violation); this only re-reads the verdict a
|
||||
* receiving mail server already stamped on the header.
|
||||
*
|
||||
* Grammar (simplified): `authserv-id; method1=result1 (comment); method2=result2 ...`
|
||||
*/
|
||||
export function parseAuthResults(headerValue: string): AuthResults {
|
||||
const result: AuthResults = {};
|
||||
for (const clause of headerValue.split(';')) {
|
||||
const match = clause.trim().match(/^(spf|dkim|dmarc)=(\w+)/i);
|
||||
if (!match) continue;
|
||||
const method = match[1].toLowerCase() as 'spf' | 'dkim' | 'dmarc';
|
||||
const verdict = match[2].toLowerCase() as AuthVerdict;
|
||||
result[method] = verdict;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// fuzzyLink enables scheme-less `www.`-prefixed URL detection (phishing URLs
|
||||
// frequently omit the scheme). Never used to fetch/dereference — string
|
||||
// matching only.
|
||||
const linkify = linkifyit({ fuzzyLink: true });
|
||||
|
||||
/**
|
||||
* Extracts and dedupes http(s)/www URLs found in the text and html parts of
|
||||
* a message. Never fetches or dereferences any extracted URL (SC#3 / T-16-03)
|
||||
* — this is pure string matching via `linkify-it`.
|
||||
*/
|
||||
export function extractUrls(
|
||||
text: string | null | undefined,
|
||||
html: string | null | undefined
|
||||
): string[] {
|
||||
const urls = new Set<string>();
|
||||
for (const source of [text, html]) {
|
||||
if (!source) continue;
|
||||
const matches = linkify.match(source) ?? [];
|
||||
for (const match of matches) {
|
||||
if (match.schema === 'mailto:') continue;
|
||||
urls.add(match.url);
|
||||
}
|
||||
}
|
||||
return Array.from(urls);
|
||||
}
|
||||
|
||||
/** Strips HTML tags/entities down to plain text. Never renders/executes the HTML. */
|
||||
function stripHtmlToText(html: string): string {
|
||||
return html
|
||||
.replace(/<(script|style)[^>]*>[\s\S]*?<\/\1>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /gi, ' ')
|
||||
.replace(/&/gi, '&')
|
||||
.replace(/</gi, '<')
|
||||
.replace(/>/gi, '>')
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/'/gi, "'")
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a sanitized, truncated plain-text body preview — prefers the
|
||||
* already-safe `mail.text`, falling back to a stripped version of
|
||||
* `mail.html` when no plain-text part exists. Never returns raw HTML.
|
||||
*/
|
||||
export function buildBodyPreview(
|
||||
text: string | null | undefined,
|
||||
html: string | null | undefined
|
||||
): string {
|
||||
const source = text && text.trim() ? text : html ? stripHtmlToText(html) : '';
|
||||
const normalized = source.replace(/\s+/g, ' ').trim();
|
||||
if (normalized.length <= MAX_BODY_PREVIEW_LENGTH) return normalized;
|
||||
return normalized.slice(0, MAX_BODY_PREVIEW_LENGTH).trimEnd() + '…';
|
||||
}
|
||||
|
||||
/** Extracts a single email address string from a mailparser HeaderValue (used for Return-Path). */
|
||||
function headerValueToAddress(value: HeaderValue | undefined): string | null {
|
||||
if (!value) return null;
|
||||
if (typeof value === 'string') {
|
||||
const cleaned = value.replace(/^</, '').replace(/>$/, '').trim();
|
||||
return cleaned || null;
|
||||
}
|
||||
if (Array.isArray(value)) return null;
|
||||
if (typeof value === 'object' && 'value' in value) {
|
||||
const addressObject = value as AddressObject;
|
||||
return addressObject.value?.[0]?.address ?? null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Flattens one or more mailparser AddressObjects into a list of email address strings. */
|
||||
function addressListToStrings(addr: AddressObject | AddressObject[] | undefined): string[] {
|
||||
if (!addr) return [];
|
||||
const objects = Array.isArray(addr) ? addr : [addr];
|
||||
const addresses: string[] = [];
|
||||
for (const obj of objects) {
|
||||
for (const entry of obj.value ?? []) {
|
||||
if (entry.address) addresses.push(entry.address);
|
||||
}
|
||||
}
|
||||
return addresses;
|
||||
}
|
||||
|
||||
/** Returns the raw header text (post-colon) for the first headerLine matching `key`, or null. */
|
||||
function findHeaderLineValue(mail: ParsedMail, key: string): string | null {
|
||||
const line = mail.headerLines.find((candidate) => candidate.key === key);
|
||||
if (!line) return null;
|
||||
const colonIndex = line.line.indexOf(':');
|
||||
return (colonIndex === -1 ? line.line : line.line.slice(colonIndex + 1)).trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a raw RFC822/MIME `.eml` buffer into a normalized, structured
|
||||
* `NormalizedMessage`. Enforces `MAX_EML_BYTES` BEFORE calling `simpleParser`
|
||||
* (DoS mitigation, T-16-01) and never performs any network I/O (SC#3).
|
||||
*/
|
||||
export async function parseEml(rawEmlBuffer: Buffer): Promise<NormalizedMessage> {
|
||||
if (rawEmlBuffer.byteLength > MAX_EML_BYTES) {
|
||||
throw new Error(
|
||||
`[EML-PARSER] Buffer size ${rawEmlBuffer.byteLength} exceeds MAX_EML_BYTES (${MAX_EML_BYTES}); refusing to parse`
|
||||
);
|
||||
}
|
||||
|
||||
let mail: ParsedMail;
|
||||
try {
|
||||
mail = await simpleParser(rawEmlBuffer, { checksumAlgo: 'sha256' });
|
||||
} catch (error) {
|
||||
console.error('[EML-PARSER] Failed to parse .eml buffer', error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const fromEntry = mail.from?.value?.[0];
|
||||
const fromEmail = fromEntry?.address ?? null;
|
||||
const fromDomain = fromEmail?.includes('@') ? fromEmail.split('@')[1] ?? null : null;
|
||||
|
||||
const receivedChain: string[] = [];
|
||||
for (const line of mail.headerLines) {
|
||||
if (line.key === 'received') receivedChain.push(line.line);
|
||||
}
|
||||
|
||||
const primaryAuthResultsHeader = findHeaderLineValue(mail, 'authentication-results');
|
||||
const originalAuthResultsHeader = findHeaderLineValue(mail, 'authentication-results-original');
|
||||
|
||||
return {
|
||||
from: {
|
||||
displayName: fromEntry?.name || null,
|
||||
email: fromEmail,
|
||||
domain: fromDomain,
|
||||
},
|
||||
replyTo: mail.replyTo?.value?.[0]?.address ?? null,
|
||||
returnPath: headerValueToAddress(mail.headers.get('return-path')),
|
||||
to: addressListToStrings(mail.to),
|
||||
cc: addressListToStrings(mail.cc),
|
||||
subject: mail.subject ?? null,
|
||||
date: mail.date ? mail.date.toISOString() : null,
|
||||
messageId: mail.messageId ?? null,
|
||||
receivedChain,
|
||||
authResults: primaryAuthResultsHeader ? parseAuthResults(primaryAuthResultsHeader) : {},
|
||||
authResultsOriginal: originalAuthResultsHeader
|
||||
? parseAuthResults(originalAuthResultsHeader)
|
||||
: null,
|
||||
urls: extractUrls(mail.text, mail.html || null),
|
||||
attachments: mail.attachments.map((att) => ({
|
||||
filename: att.filename ?? null,
|
||||
contentType: att.contentType ?? null,
|
||||
size: att.size,
|
||||
checksum: att.checksum ?? null,
|
||||
related: Boolean(att.related),
|
||||
})),
|
||||
bodyPreview: buildBodyPreview(mail.text, mail.html || null),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue