feat(16-03): phishing-eml-service.ts orchestration (list->select->fetch->B2->parse->persist)
- parseAndStoreMessage lists ticket attachments, selects the original message via selectOriginalMessage
- fetches full content via getAttachmentContent, size-guards the decoded buffer against MAX_EML_BYTES
- uploads raw bytes to B2 under phishing/{reportId}/{attachmentId}.eml when isB2Configured(), gracefully degrading rawRef=null on failure/absence
- parses via parseEml and persists one messages row (headers incl. D-06 auth verdicts, urls, attachments, body_preview, raw_ref) plus indicators rows (attachment_hash/url/sender) carrying D-07 metadata JSONB
- never fetches any URL extracted from the message; the only outbound calls are the Autotask attachment GET and the B2 presigned PUT
This commit is contained in:
parent
36ef9e7ce3
commit
5088d8d511
1 changed files with 224 additions and 0 deletions
224
lib/services/phishing-eml-service.ts
Normal file
224
lib/services/phishing-eml-service.ts
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
/**
|
||||
* Phishing EML/MIME evidence orchestration service (Phase 16, Plan 03).
|
||||
*
|
||||
* Turns a detected phishing report into persisted, normalized message
|
||||
* evidence: lists a ticket's attachments, selects the original reported
|
||||
* message (Plan 01's `selectOriginalMessage`), fetches its full base64
|
||||
* content (Plan 02's `AutotaskClient.getAttachmentContent`), size-guards the
|
||||
* decoded buffer, stores the raw bytes in B2 under a dedicated `.eml` key
|
||||
* when configured (D-05), parses it (Plan 01's `parseEml`), and persists one
|
||||
* `messages` row plus `indicators` rows carrying per-indicator context in
|
||||
* the D-07 `metadata` JSONB column.
|
||||
*
|
||||
* Hard invariant (SC#3 / T-16-03): this service never fetches or executes
|
||||
* anything found in the parsed message. The only outbound network calls are
|
||||
* the Autotask attachment GET (via `AutotaskClient`) and the B2 presigned
|
||||
* PUT of the raw bytes we already hold — never a URL/host derived from the
|
||||
* message content itself.
|
||||
*
|
||||
* The live on-demand trigger (`POST /api/phishing/tickets/{id}/analyze`)
|
||||
* arrives in Phase 18 (DETECT-03); this module is callable and fully tested
|
||||
* here.
|
||||
*/
|
||||
|
||||
import { postgresClient } from './postgres-client';
|
||||
import { getAutotaskClient } from './autotask-factory';
|
||||
import { presignUpload, isB2Configured, EML_OBJECT_KEY_REGEX } from './b2/client';
|
||||
import { parseEml, selectOriginalMessage, MAX_EML_BYTES, type NormalizedMessage } from './eml-parser';
|
||||
|
||||
export interface ParseAndStoreInput {
|
||||
reportId: string;
|
||||
ticketId: number;
|
||||
}
|
||||
|
||||
export interface ParseAndStoreResult {
|
||||
stored: boolean;
|
||||
messageId?: string;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrates list -> select -> fetch -> (B2 gated) -> parse -> persist for
|
||||
* a single report/ticket.
|
||||
*
|
||||
* Returns `{ stored: false, reason: ... }` (never throws) when there is
|
||||
* nothing to persist yet (no `.eml` attachment found, no content returned,
|
||||
* or the decoded buffer is oversized). Rethrows on any other failure so a
|
||||
* future caller (Phase 18) can decide fail-vs-degrade (T-16-08).
|
||||
*/
|
||||
export async function parseAndStoreMessage(
|
||||
input: ParseAndStoreInput
|
||||
): Promise<ParseAndStoreResult> {
|
||||
const { reportId, ticketId } = input;
|
||||
|
||||
try {
|
||||
// reports.evidence only stores fullPath/title/contentType (no attachment
|
||||
// id) — list live so we have the attachment id needed for the
|
||||
// per-attachment content fetch below.
|
||||
const attachments = await getAutotaskClient().getAttachments('Tickets', ticketId);
|
||||
const selected = selectOriginalMessage(attachments);
|
||||
|
||||
if (!selected) {
|
||||
return { stored: false, reason: 'no-eml-attachment' };
|
||||
}
|
||||
|
||||
const fullAttachment = await getAutotaskClient().getAttachmentContent(
|
||||
'Tickets',
|
||||
ticketId,
|
||||
selected.id
|
||||
);
|
||||
|
||||
if (!fullAttachment?.data) {
|
||||
console.error(
|
||||
'[PHISHING-EML] Selected attachment had no content for report',
|
||||
reportId,
|
||||
'ticket',
|
||||
ticketId,
|
||||
'attachment',
|
||||
selected.id
|
||||
);
|
||||
return { stored: false, reason: 'no-attachment-content' };
|
||||
}
|
||||
|
||||
const rawEmlBuffer = Buffer.from(fullAttachment.data, 'base64');
|
||||
|
||||
// Size guard BEFORE parseEml (T-16-01 — DoS mitigation): degrade rather
|
||||
// than let parseEml's own internal guard throw and crash the caller.
|
||||
if (rawEmlBuffer.byteLength > MAX_EML_BYTES) {
|
||||
console.error(
|
||||
'[PHISHING-EML] Decoded attachment exceeds MAX_EML_BYTES, skipping report',
|
||||
reportId,
|
||||
'size',
|
||||
rawEmlBuffer.byteLength
|
||||
);
|
||||
return { stored: false, reason: 'oversized-attachment' };
|
||||
}
|
||||
|
||||
let rawRef: string | null = null;
|
||||
if (isB2Configured()) {
|
||||
const objectKey = `phishing/${reportId}/${selected.id}.eml`;
|
||||
try {
|
||||
const uploadUrl = presignUpload(objectKey, 1800, undefined, EML_OBJECT_KEY_REGEX);
|
||||
const putResponse = await fetch(uploadUrl, { method: 'PUT', body: rawEmlBuffer });
|
||||
if (!putResponse.ok) {
|
||||
throw new Error(`B2 PUT ${objectKey} failed: ${putResponse.status}`);
|
||||
}
|
||||
rawRef = objectKey;
|
||||
} catch (error) {
|
||||
// Graceful degrade (RESEARCH.md Pitfall 3) — a failed/absent B2 PUT
|
||||
// must not abort parsing/persistence.
|
||||
console.error(
|
||||
'[PHISHING-EML] Failed to upload raw .eml to B2 for report',
|
||||
reportId,
|
||||
error
|
||||
);
|
||||
rawRef = null;
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
'[PHISHING-EML] B2 not configured; skipping raw .eml upload for report',
|
||||
reportId
|
||||
);
|
||||
}
|
||||
|
||||
const normalized: NormalizedMessage = await parseEml(rawEmlBuffer);
|
||||
|
||||
// Full normalized header block, including structured SPF/DKIM/DMARC
|
||||
// verdicts (D-06), persisted into messages.headers JSONB.
|
||||
const headersPayload = {
|
||||
from: normalized.from,
|
||||
replyTo: normalized.replyTo,
|
||||
returnPath: normalized.returnPath,
|
||||
to: normalized.to,
|
||||
cc: normalized.cc,
|
||||
subject: normalized.subject,
|
||||
date: normalized.date,
|
||||
messageId: normalized.messageId,
|
||||
receivedChain: normalized.receivedChain,
|
||||
authResults: normalized.authResults,
|
||||
authResultsOriginal: normalized.authResultsOriginal,
|
||||
};
|
||||
|
||||
const messageInsert = await postgresClient.query<{ id: string }>(
|
||||
`INSERT INTO messages (
|
||||
report_id, message_id, headers, urls, attachments, body_preview, raw_ref
|
||||
)
|
||||
VALUES ($1, $2, $3::jsonb, $4::jsonb, $5::jsonb, $6, $7)
|
||||
RETURNING id::text AS id`,
|
||||
[
|
||||
reportId,
|
||||
normalized.messageId,
|
||||
JSON.stringify(headersPayload),
|
||||
JSON.stringify(normalized.urls),
|
||||
JSON.stringify(normalized.attachments),
|
||||
normalized.bodyPreview,
|
||||
rawRef,
|
||||
]
|
||||
);
|
||||
|
||||
const messageId = messageInsert.rows[0].id;
|
||||
|
||||
// One indicator per attachment checksum (D-07 metadata carries
|
||||
// filename/contentType/size/related so Phase 19 can weight inline parts
|
||||
// differently without a second lookup).
|
||||
for (const attachment of normalized.attachments) {
|
||||
if (!attachment.checksum) continue;
|
||||
await postgresClient.query(
|
||||
`INSERT INTO indicators (message_id, indicator_type, value, metadata)
|
||||
VALUES ($1, $2, $3, $4::jsonb)
|
||||
RETURNING id::text AS id`,
|
||||
[
|
||||
messageId,
|
||||
'attachment_hash',
|
||||
attachment.checksum,
|
||||
JSON.stringify({
|
||||
filename: attachment.filename,
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
related: attachment.related,
|
||||
}),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
// One indicator per extracted URL — never dereferenced, only persisted
|
||||
// as a string value (SC#3 / T-16-03).
|
||||
for (const url of normalized.urls) {
|
||||
await postgresClient.query(
|
||||
`INSERT INTO indicators (message_id, indicator_type, value, metadata)
|
||||
VALUES ($1, $2, $3, $4::jsonb)
|
||||
RETURNING id::text AS id`,
|
||||
[messageId, 'url', url, JSON.stringify({ part: 'body' })]
|
||||
);
|
||||
}
|
||||
|
||||
// One indicator for the sender, if present.
|
||||
if (normalized.from.email) {
|
||||
await postgresClient.query(
|
||||
`INSERT INTO indicators (message_id, indicator_type, value, metadata)
|
||||
VALUES ($1, $2, $3, $4::jsonb)
|
||||
RETURNING id::text AS id`,
|
||||
[
|
||||
messageId,
|
||||
'sender',
|
||||
normalized.from.email,
|
||||
JSON.stringify({
|
||||
displayName: normalized.from.displayName,
|
||||
domain: normalized.from.domain,
|
||||
}),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
return { stored: true, messageId };
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'[PHISHING-EML] Failed to parse/store message for report',
|
||||
reportId,
|
||||
'ticket',
|
||||
ticketId,
|
||||
error
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue