lorentz/howl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial commit: Howl M365 email management daemon

Browse source 
Full implementation of the Howl email triage system:
- Microsoft Graph API integration with MSAL auth (client-credentials and delegated modes)
- Claude LLM classification via tool use for structured output
- PostgreSQL database with customers, vendors, whitelist, and email_log tables
- Alembic migration for full schema
- APScheduler daemon with graceful shutdown
- Typer CLI (run, dry-run, status commands)
- Business rule classifier with overrides (whitelist protection, low-confidence fallback)
- Action executor (move to folders, flag, escalate with webhook)
- 35 passing unit tests
- README, SETUP, and NEXT_STEPS documentation

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
lorentz 2026-04-01 16:02:24 -04:00
commit 3bfda9e585
39 changed files with 3656 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

0
tests/__init__.py Normal file
View file

157
tests/conftest.py Normal file
View file

@ -0,0 +1,157 @@
from __future__ import annotations
import json
import uuid
from datetime import datetime, timezone
from pathlib import Path
from typing import Generator
from unittest.mock import MagicMock, patch
import pytest
from howl.config import Settings
from howl.db.queries import SenderMatch
from howl.graph.client import GraphClient, Message
from howl.llm.schemas import EmailClassification
FIXTURES_DIR = Path(__file__).parent / "fixtures"
# ---------------------------------------------------------------------------
# Settings fixture (no real credentials needed)
# ---------------------------------------------------------------------------
@pytest.fixture
def settings() -> Settings:
return Settings(
azure_tenant_id="test-tenant",
azure_client_id="test-client-id",
azure_client_secret="test-secret",
graph_auth_mode="client_credentials",
graph_mailbox="test@example.com",
anthropic_api_key="test-anthropic-key",
database_url="postgresql+asyncpg://test:test@localhost:5432/howl_test",
dry_run=True,
log_level="WARNING",
log_format="text",
)
# ---------------------------------------------------------------------------
# Sample email messages
# ---------------------------------------------------------------------------
@pytest.fixture
def sample_emails() -> list[dict]:
with open(FIXTURES_DIR / "sample_emails.json") as f:
return json.load(f)
@pytest.fixture
def llm_responses() -> dict:
with open(FIXTURES_DIR / "llm_responses.json") as f:
return json.load(f)
def make_message(raw: dict) -> Message:
sender = raw.get("sender", {}).get("emailAddress", {})
received_str = raw.get("receivedDateTime")
received_at = datetime.fromisoformat(received_str.replace("Z", "+00:00")) if received_str else None
return Message(
id=raw["id"],
conversation_id=raw.get("conversationId"),
subject=raw.get("subject"),
sender_address=sender.get("address", ""),
sender_name=sender.get("name"),
received_at=received_at,
has_attachments=raw.get("hasAttachments", False),
importance=raw.get("importance", "normal"),
body_preview=raw.get("bodyPreview"),
)
@pytest.fixture
def customer_message(sample_emails) -> Message:
return make_message(next(e for e in sample_emails if e["id"] == "msg-001-customer-inquiry"))
@pytest.fixture
def spam_message(sample_emails) -> Message:
return make_message(next(e for e in sample_emails if e["id"] == "msg-004-spam"))
@pytest.fixture
def urgent_message(sample_emails) -> Message:
return make_message(next(e for e in sample_emails if e["id"] == "msg-007-urgent"))
# ---------------------------------------------------------------------------
# Mock Graph client
# ---------------------------------------------------------------------------
@pytest.fixture
def mock_graph(sample_emails) -> MagicMock:
graph = MagicMock(spec=GraphClient)
raw_messages = [make_message(e) for e in sample_emails]
graph.list_unread_messages.return_value = (raw_messages[:3], None)
graph.get_message_body.return_value = "This is the email body text."
graph.get_or_create_folder.return_value = str(uuid.uuid4())
graph.move_message.return_value = None
graph.flag_message.return_value = None
graph.mark_as_read.return_value = None
return graph
# ---------------------------------------------------------------------------
# Mock LLM client
# ---------------------------------------------------------------------------
@pytest.fixture
def mock_llm_classification(llm_responses) -> EmailClassification:
return EmailClassification.model_validate(llm_responses["customer_inquiry"])
@pytest.fixture
def mock_llm(mock_llm_classification) -> MagicMock:
from howl.llm.client import LLMClient
llm = MagicMock(spec=LLMClient)
llm.classify.return_value = (
mock_llm_classification,
{
"model": "claude-sonnet-4-6",
"stop_reason": "tool_use",
"usage": {"input_tokens": 350, "output_tokens": 80},
"tool_input": mock_llm_classification.model_dump(),
},
)
return llm
# ---------------------------------------------------------------------------
# Sender matches
# ---------------------------------------------------------------------------
@pytest.fixture
def customer_match() -> SenderMatch:
return SenderMatch(
sender_type="customer",
entity_id=uuid.uuid4(),
entity_table="customers",
entity_name="Acme Corp",
notes="Key account - handle with priority",
)
@pytest.fixture
def unknown_match() -> SenderMatch:
return SenderMatch(sender_type="unknown")
@pytest.fixture
def whitelist_match() -> SenderMatch:
return SenderMatch(
sender_type="whitelist",
entity_id=uuid.uuid4(),
entity_table="whitelist",
entity_name="Trusted partner domain",
)

65
tests/fixtures/llm_responses.json vendored Normal file
View file

@ -0,0 +1,65 @@
{
"customer_inquiry": {
"classification": "customer_inquiry",
"confidence": 0.95,
"action": "move_customer",
"reasoning": "This is a known customer following up on an order. Should be routed to the Customers folder for response.",
"priority": "normal",
"requires_human_review": false,
"tags": ["order-followup", "customer-service"]
},
"vendor_invoice": {
"classification": "vendor_invoice",
"confidence": 0.97,
"action": "move_vendor",
"reasoning": "Invoice from a known vendor with a specific invoice number and payment details. Route to Vendors folder.",
"priority": "normal",
"requires_human_review": true,
"tags": ["invoice", "payment-due"]
},
"spam": {
"classification": "spam",
"confidence": 0.98,
"action": "move_spam",
"reasoning": "Classic spam characteristics: gift card offer, urgency language, unknown sender domain.",
"priority": "low",
"requires_human_review": false,
"tags": ["gift-card-scam", "promotional"]
},
"phishing": {
"classification": "phishing",
"confidence": 0.96,
"action": "move_spam",
"reasoning": "Phishing attempt: impersonates Microsoft using a suspicious domain, requests credential verification.",
"priority": "high",
"requires_human_review": true,
"tags": ["phishing", "credential-theft", "impersonation"]
},
"urgent": {
"classification": "urgent_action_required",
"confidence": 0.93,
"action": "escalate",
"reasoning": "Known customer reporting production system outage. Requires immediate human attention.",
"priority": "urgent",
"requires_human_review": true,
"tags": ["production-outage", "customer-escalation"]
},
"low_confidence": {
"classification": "unknown",
"confidence": 0.45,
"action": "move_review",
"reasoning": "Cannot determine the nature of this email with sufficient confidence. Human review recommended.",
"priority": "normal",
"requires_human_review": true,
"tags": []
},
"newsletter": {
"classification": "newsletter",
"confidence": 0.92,
"action": "move_spam",
"reasoning": "Unsolicited newsletter from an unknown sender. Contains unsubscribe link.",
"priority": "low",
"requires_human_review": false,
"tags": ["newsletter", "marketing"]
}
}

142
tests/fixtures/sample_emails.json vendored Normal file
View file

@ -0,0 +1,142 @@
[
{
"id": "msg-001-customer-inquiry",
"conversationId": "conv-001",
"subject": "Question about my order #45231",
"sender": {"emailAddress": {"address": "alice@acme.com", "name": "Alice Johnson"}},
"receivedDateTime": "2026-04-01T09:15:00Z",
"hasAttachments": false,
"importance": "normal",
"bodyPreview": "Hi, I wanted to check on the status of my order...",
"body": {
"contentType": "text",
"content": "Hi,\n\nI wanted to check on the status of my order #45231 that was placed on March 28th. I haven't received any shipping notification yet and was hoping you could provide an update.\n\nThank you,\nAlice Johnson\nAcme Corp"
}
},
{
"id": "msg-002-vendor-invoice",
"conversationId": "conv-002",
"subject": "Invoice INV-2026-0342 from SupplyPro",
"sender": {"emailAddress": {"address": "billing@supplypro.com", "name": "SupplyPro Billing"}},
"receivedDateTime": "2026-04-01T10:30:00Z",
"hasAttachments": true,
"importance": "normal",
"bodyPreview": "Please find attached Invoice INV-2026-0342...",
"body": {
"contentType": "text",
"content": "Dear Customer,\n\nPlease find attached Invoice INV-2026-0342 for $4,250.00 due April 30, 2026.\n\nPayment terms: Net 30\nBank transfer details are included in the attached PDF.\n\nRegards,\nSupplyPro Billing Team"
}
},
{
"id": "msg-003-whitelist-notification",
"conversationId": "conv-003",
"subject": "Quarterly audit reminder - action required",
"sender": {"emailAddress": {"address": "audit@partnerfirm.com", "name": "Partner Firm Audit"}},
"receivedDateTime": "2026-04-01T11:00:00Z",
"hasAttachments": false,
"importance": "high",
"bodyPreview": "This is a reminder about the upcoming quarterly audit...",
"body": {
"contentType": "text",
"content": "Hello,\n\nThis is a reminder that the Q1 2026 quarterly audit is scheduled for April 15th. Please ensure all financial records from January through March are compiled and available for review.\n\nPlease confirm receipt of this notice.\n\nBest regards,\nPartner Firm Audit Team"
}
},
{
"id": "msg-004-spam",
"conversationId": "conv-004",
"subject": "You've been selected! Claim your $500 gift card NOW",
"sender": {"emailAddress": {"address": "promo@random-offers.xyz", "name": "Special Rewards"}},
"receivedDateTime": "2026-04-01T08:00:00Z",
"hasAttachments": false,
"importance": "normal",
"bodyPreview": "Congratulations! You have been specially selected...",
"body": {
"contentType": "text",
"content": "CONGRATULATIONS!! You have been specially selected to receive a $500 gift card. Click here to claim your reward within 24 hours. Limited time offer! Act NOW!!!"
}
},
{
"id": "msg-005-phishing",
"conversationId": "conv-005",
"subject": "Urgent: Your account requires verification",
"sender": {"emailAddress": {"address": "security@microsoft-support-helpdesk.com", "name": "Microsoft Security"}},
"receivedDateTime": "2026-04-01T07:45:00Z",
"hasAttachments": false,
"importance": "high",
"bodyPreview": "Your account has been flagged for suspicious activity...",
"body": {
"contentType": "text",
"content": "Dear User,\n\nYour Microsoft account has been flagged for suspicious activity. You must verify your identity within 24 hours or your account will be suspended.\n\nClick here to verify: http://microsoft-verify.malicious-site.com/verify\n\nMicrosoft Security Team"
}
},
{
"id": "msg-006-newsletter",
"conversationId": "conv-006",
"subject": "This week in SaaS: Top 10 tools for 2026",
"sender": {"emailAddress": {"address": "newsletter@techdigest.io", "name": "Tech Digest Weekly"}},
"receivedDateTime": "2026-04-01T06:00:00Z",
"hasAttachments": false,
"importance": "low",
"bodyPreview": "Welcome to this week's edition of Tech Digest...",
"body": {
"contentType": "text",
"content": "Welcome to Tech Digest Weekly!\n\nThis week we cover:\n- Top 10 SaaS tools for productivity in 2026\n- AI coding assistants reviewed\n- Cloud cost optimization tips\n\nRead more at techdigest.io\n\nUnsubscribe | Manage preferences"
}
},
{
"id": "msg-007-urgent",
"conversationId": "conv-007",
"subject": "URGENT: Production system down - need immediate help",
"sender": {"emailAddress": {"address": "bob@bigclient.com", "name": "Bob Smith"}},
"receivedDateTime": "2026-04-01T14:22:00Z",
"hasAttachments": false,
"importance": "high",
"bodyPreview": "Our production system went down 20 minutes ago...",
"body": {
"contentType": "text",
"content": "Hi Team,\n\nOur production system went down about 20 minutes ago and we're losing orders. We believe it's related to the API changes deployed this morning. We need immediate assistance.\n\nPlease call me ASAP at 555-0199.\n\nBob Smith\nVP Engineering, BigClient Inc"
}
},
{
"id": "msg-008-vendor-notification",
"conversationId": "conv-008",
"subject": "Your subscription renewal confirmation - CloudTools Pro",
"sender": {"emailAddress": {"address": "noreply@cloudtools.io", "name": "CloudTools"}},
"receivedDateTime": "2026-04-01T12:00:00Z",
"hasAttachments": false,
"importance": "normal",
"bodyPreview": "Your CloudTools Pro subscription has been renewed...",
"body": {
"contentType": "text",
"content": "Thank you for renewing your CloudTools Pro subscription.\n\nPlan: Pro Annual\nRenewal date: April 1, 2026\nNext renewal: April 1, 2027\nAmount charged: $1,199.00\n\nManage your subscription at app.cloudtools.io/billing"
}
},
{
"id": "msg-009-unknown-legit",
"conversationId": "conv-009",
"subject": "Partnership inquiry - integration opportunity",
"sender": {"emailAddress": {"address": "partnerships@newcompany.com", "name": "New Company Partnerships"}},
"receivedDateTime": "2026-04-01T13:15:00Z",
"hasAttachments": false,
"importance": "normal",
"bodyPreview": "Hello, I'm reaching out regarding a potential partnership...",
"body": {
"contentType": "text",
"content": "Hello,\n\nI'm reaching out regarding a potential partnership opportunity. Our company provides complementary services to yours and we believe there could be significant mutual benefit in exploring an integration.\n\nWould you be open to a 30-minute call this week?\n\nBest,\nJen Williams\nHead of Partnerships, New Company"
}
},
{
"id": "msg-010-domain-whitelist",
"conversationId": "conv-010",
"subject": "Updated compliance documentation",
"sender": {"emailAddress": {"address": "compliance@microsoft.com", "name": "Microsoft Compliance"}},
"receivedDateTime": "2026-04-01T15:00:00Z",
"hasAttachments": true,
"importance": "normal",
"bodyPreview": "Please review the attached updated compliance documentation...",
"body": {
"contentType": "text",
"content": "Dear Customer,\n\nPlease review the attached updated Microsoft compliance documentation relevant to your enterprise subscription. These updates reflect changes in data processing terms effective May 1, 2026.\n\nMicrosoft Compliance Team"
}
}
]

91
tests/test_classifier.py Normal file
View file

@ -0,0 +1,91 @@
from __future__ import annotations
import pytest
from howl.db.queries import SenderMatch
from howl.llm.schemas import EmailClassification
from howl.pipeline import classifier
def make_classification(**kwargs) -> EmailClassification:
defaults = {
"classification": "customer_inquiry",
"confidence": 0.90,
"action": "move_customer",
"reasoning": "Test classification",
"priority": "normal",
"requires_human_review": False,
"tags": [],
}
defaults.update(kwargs)
return EmailClassification.model_validate(defaults)
def test_whitelist_sender_not_moved_to_spam(settings, whitelist_match):
classification = make_classification(action="move_spam", confidence=0.85)
final_action, overridden = classifier.decide(whitelist_match, classification, settings)
assert final_action == "inbox_keep"
assert overridden is True
def test_customer_sender_not_moved_to_spam(settings, customer_match):
classification = make_classification(action="move_spam", confidence=0.85)
final_action, overridden = classifier.decide(customer_match, classification, settings)
assert final_action == "move_review"
assert overridden is True
def test_requires_human_review_forces_move_review(settings, unknown_match):
classification = make_classification(
action="inbox_keep",
confidence=0.75,
requires_human_review=True,
)
final_action, overridden = classifier.decide(unknown_match, classification, settings)
assert final_action == "move_review"
assert overridden is True
def test_low_confidence_falls_back_to_inbox_keep(settings, unknown_match):
# Settings has llm_confidence_threshold=0.60, this is 0.45
classification = make_classification(
action="move_spam",
confidence=0.45,
requires_human_review=False,
)
final_action, overridden = classifier.decide(unknown_match, classification, settings)
assert final_action == "inbox_keep"
assert overridden is True
def test_high_confidence_customer_passes_through(settings, customer_match):
classification = make_classification(
action="move_customer",
confidence=0.95,
requires_human_review=False,
)
final_action, overridden = classifier.decide(customer_match, classification, settings)
assert final_action == "move_customer"
assert overridden is False
def test_escalate_not_overridden_by_low_confidence(settings, customer_match):
"""Escalate action should survive even low confidence — it's already going to human review."""
classification = make_classification(
action="escalate",
confidence=0.40,
requires_human_review=True,
)
final_action, overridden = classifier.decide(customer_match, classification, settings)
assert final_action == "escalate"
def test_flag_follow_up_survives_requires_human_review(settings, unknown_match):
"""flag_follow_up is an acceptable requires_human_review action — should not be overridden."""
classification = make_classification(
action="flag_follow_up",
confidence=0.80,
requires_human_review=True,
)
final_action, overridden = classifier.decide(unknown_match, classification, settings)
assert final_action == "flag_follow_up"

183
tests/test_graph_client.py Normal file
View file

@ -0,0 +1,183 @@
from __future__ import annotations
from datetime import datetime, timezone
from unittest.mock import MagicMock, patch
import pytest
import respx
import httpx
from howl.graph.client import GraphClient, _parse_message, _extract_skip_token, _strip_html
GRAPH_BASE = "https://graph.microsoft.com/v1.0"
def make_graph_client() -> GraphClient:
return GraphClient(token_provider=lambda: "test-bearer-token")
# ---------------------------------------------------------------------------
# Unit tests (no HTTP)
# ---------------------------------------------------------------------------
def test_parse_message_extracts_fields(sample_emails):
raw = sample_emails[0] # customer inquiry
msg = _parse_message(raw)
assert msg.id == "msg-001-customer-inquiry"
assert msg.sender_address == "alice@acme.com"
assert msg.sender_name == "Alice Johnson"
assert msg.subject == "Question about my order #45231"
assert msg.has_attachments is False
assert isinstance(msg.received_at, datetime)
def test_parse_message_with_no_sender():
raw = {"id": "x", "conversationId": None}
msg = _parse_message(raw)
assert msg.sender_address == ""
assert msg.received_at is None
def test_extract_skip_token_from_next_link():
link = "https://graph.microsoft.com/v1.0/users/mb/messages?$skipToken=abc123&$top=50"
assert _extract_skip_token(link) == "abc123"
def test_extract_skip_token_returns_none_when_absent():
assert _extract_skip_token("https://graph.microsoft.com/v1.0/users/mb/messages") is None
def test_strip_html_removes_tags():
html = "<html><body><p>Hello <b>World</b></p></body></html>"
text = _strip_html(html)
assert "<" not in text
assert "Hello" in text
assert "World" in text
# ---------------------------------------------------------------------------
# HTTP-level tests with respx
# ---------------------------------------------------------------------------
@respx.mock
def test_list_unread_messages_single_page():
client = make_graph_client()
mailbox = "test@example.com"
url = f"{GRAPH_BASE}/users/{mailbox}/mailFolders/Inbox/messages"
respx.get(url).mock(return_value=httpx.Response(200, json={
"value": [
{
"id": "msg-001",
"conversationId": "conv-001",
"subject": "Test",
"sender": {"emailAddress": {"address": "a@b.com", "name": "A B"}},
"receivedDateTime": "2026-04-01T10:00:00Z",
"hasAttachments": False,
"importance": "normal",
}
]
}))
messages, next_skip = client.list_unread_messages(mailbox, top=50)
assert len(messages) == 1
assert messages[0].id == "msg-001"
assert next_skip is None
@respx.mock
def test_list_unread_messages_pagination():
client = make_graph_client()
mailbox = "test@example.com"
url = f"{GRAPH_BASE}/users/{mailbox}/mailFolders/Inbox/messages"
respx.get(url).mock(return_value=httpx.Response(200, json={
"value": [{"id": "msg-001", "conversationId": None, "subject": "S",
"sender": {"emailAddress": {"address": "a@b.com"}},
"receivedDateTime": "2026-04-01T10:00:00Z",
"hasAttachments": False, "importance": "normal"}],
"@odata.nextLink": f"{url}?$skipToken=NEXT_TOKEN_ABC"
}))
messages, next_skip = client.list_unread_messages(mailbox, top=1)
assert len(messages) == 1
assert next_skip == "NEXT_TOKEN_ABC"
@respx.mock
def test_get_message_body_plain_text():
client = make_graph_client()
mailbox = "test@example.com"
message_id = "msg-001"
url = f"{GRAPH_BASE}/users/{mailbox}/messages/{message_id}"
respx.get(url).mock(return_value=httpx.Response(200, json={
"body": {"contentType": "text", "content": "Hello, this is the body."}
}))
body = client.get_message_body(mailbox, message_id)
assert body == "Hello, this is the body."
@respx.mock
def test_get_message_body_html_stripped():
client = make_graph_client()
mailbox = "test@example.com"
message_id = "msg-002"
url = f"{GRAPH_BASE}/users/{mailbox}/messages/{message_id}"
respx.get(url).mock(return_value=httpx.Response(200, json={
"body": {"contentType": "html", "content": "<html><body><p>Hello</p></body></html>"}
}))
body = client.get_message_body(mailbox, message_id)
assert "<" not in body
assert "Hello" in body
@respx.mock
def test_get_or_create_folder_creates_when_not_found():
client = make_graph_client()
mailbox = "test@example.com"
list_url = f"{GRAPH_BASE}/users/{mailbox}/mailFolders"
create_url = f"{GRAPH_BASE}/users/{mailbox}/mailFolders"
# First call: list returns empty
respx.get(list_url).mock(return_value=httpx.Response(200, json={"value": []}))
# Second call: create returns new folder
respx.post(create_url).mock(return_value=httpx.Response(201, json={
"id": "folder-id-123",
"displayName": "Customers"
}))
folder_id = client.get_or_create_folder(mailbox, "Customers")
assert folder_id == "folder-id-123"
# Second call should use cache, no additional HTTP requests
folder_id_cached = client.get_or_create_folder(mailbox, "Customers")
assert folder_id_cached == folder_id
@respx.mock
def test_move_message():
client = make_graph_client()
mailbox = "test@example.com"
message_id = "msg-001"
folder_id = "folder-abc"
url = f"{GRAPH_BASE}/users/{mailbox}/messages/{message_id}/move"
respx.post(url).mock(return_value=httpx.Response(201, json={"id": "msg-001-moved"}))
client.move_message(mailbox, message_id, folder_id) # Should not raise
@respx.mock
def test_raises_on_401():
client = make_graph_client()
mailbox = "test@example.com"
url = f"{GRAPH_BASE}/users/{mailbox}/mailFolders/Inbox/messages"
respx.get(url).mock(return_value=httpx.Response(401, json={"error": {"code": "Unauthorized"}}))
with pytest.raises(httpx.HTTPStatusError):
client.list_unread_messages(mailbox, top=10)

133
tests/test_llm_client.py Normal file
View file

@ -0,0 +1,133 @@
from __future__ import annotations
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from howl.llm.schemas import CLASSIFY_EMAIL_TOOL, EmailClassification
from howl.llm.prompts import build_prompt, SYSTEM_PROMPT
FIXTURES_DIR = Path(__file__).parent / "fixtures"
def test_email_classification_validates_valid_data(llm_responses):
cls = EmailClassification.model_validate(llm_responses["customer_inquiry"])
assert cls.classification == "customer_inquiry"
assert cls.action == "move_customer"
assert 0.0 <= cls.confidence <= 1.0
assert cls.reasoning
def test_email_classification_rejects_invalid_confidence():
with pytest.raises(Exception):
EmailClassification.model_validate({
"classification": "unknown",
"confidence": 1.5, # Out of range
"action": "inbox_keep",
"reasoning": "test",
"priority": "normal",
"requires_human_review": False,
})
def test_email_classification_requires_human_review_for_phishing(llm_responses):
cls = EmailClassification.model_validate(llm_responses["phishing"])
assert cls.requires_human_review is True
def test_email_classification_low_confidence(llm_responses):
cls = EmailClassification.model_validate(llm_responses["low_confidence"])
assert cls.confidence < 0.60
assert cls.requires_human_review is True
def test_build_prompt_includes_sender_info(customer_message, customer_match):
prompt = build_prompt(customer_message, "Hello, checking on my order.", customer_match)
assert customer_message.sender_address in prompt
assert "customer" in prompt
assert "Acme Corp" in prompt
assert "Key account" in prompt
def test_build_prompt_truncates_body(customer_message, unknown_match):
long_body = "x" * 10000
prompt = build_prompt(customer_message, long_body, unknown_match, max_body_chars=100)
assert "truncated" in prompt
# Body in prompt should be 100 chars, not 10000
assert "x" * 101 not in prompt
def test_build_prompt_unknown_sender(spam_message, unknown_match):
prompt = build_prompt(spam_message, "Claim your prize!", unknown_match)
assert "unknown" in prompt
# No entity name should appear
assert "Known as:" not in prompt
def test_classify_email_tool_schema_is_valid():
"""Tool definition should have all required fields for Anthropic tool use."""
assert CLASSIFY_EMAIL_TOOL["name"] == "classify_email"
schema = CLASSIFY_EMAIL_TOOL["input_schema"]
assert schema["type"] == "object"
required = schema["required"]
assert "classification" in required
assert "confidence" in required
assert "action" in required
assert "reasoning" in required
assert "requires_human_review" in required
def test_llm_client_calls_anthropic_and_parses_response(
settings, customer_message, customer_match
):
"""Test that LLMClient correctly calls the Anthropic SDK and parses tool use output."""
from howl.llm.client import LLMClient
mock_tool_use = MagicMock()
mock_tool_use.type = "tool_use"
mock_tool_use.input = {
"classification": "customer_inquiry",
"confidence": 0.92,
"action": "move_customer",
"reasoning": "Known customer inquiry.",
"priority": "normal",
"requires_human_review": False,
"tags": [],
}
mock_response = MagicMock()
mock_response.content = [mock_tool_use]
mock_response.model = "claude-sonnet-4-6"
mock_response.stop_reason = "tool_use"
mock_response.usage.input_tokens = 300
mock_response.usage.output_tokens = 75
with patch("howl.llm.client.Anthropic") as MockAnthropic:
MockAnthropic.return_value.messages.create.return_value = mock_response
client = LLMClient(settings)
classification, raw = client.classify(
customer_message, "Hello, checking my order status.", customer_match
)
assert classification.classification == "customer_inquiry"
assert classification.action == "move_customer"
assert raw["usage"]["input_tokens"] == 300
def test_llm_client_raises_if_no_tool_use(settings, customer_message, customer_match):
"""LLMClient should raise ValueError if Claude doesn't call the tool."""
from howl.llm.client import LLMClient
mock_text_block = MagicMock()
mock_text_block.type = "text"
mock_response = MagicMock()
mock_response.content = [mock_text_block]
with patch("howl.llm.client.Anthropic") as MockAnthropic:
MockAnthropic.return_value.messages.create.return_value = mock_response
client = LLMClient(settings)
with pytest.raises(ValueError, match="classify_email"):
client.classify(customer_message, "body", customer_match)

187
tests/test_pipeline.py Normal file
View file

@ -0,0 +1,187 @@
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock, patch
from datetime import datetime, timezone
import pytest
from howl.db.queries import SenderMatch
from howl.llm.schemas import EmailClassification
from howl.pipeline import classifier
from howl.pipeline.actions import ActionExecutor
from howl.pipeline.processor import EmailProcessor
# ---------------------------------------------------------------------------
# ActionExecutor tests (dry_run mode — no actual HTTP calls)
# ---------------------------------------------------------------------------
def test_action_executor_dry_run_does_not_call_graph(settings, mock_graph):
from howl.db.models import EmailLog
assert settings.dry_run is True # conftest sets dry_run=True
executor = ActionExecutor(mock_graph, settings)
log_entry = MagicMock(spec=EmailLog)
log_entry.mailbox = "test@example.com"
log_entry.graph_message_id = "msg-001"
log_entry.sender_name = "Alice"
log_entry.sender_address = "alice@test.com"
log_entry.subject = "Test"
executor.execute(log_entry, "move_customer")
mock_graph.move_message.assert_not_called()
mock_graph.get_or_create_folder.assert_not_called()
def test_action_executor_move_customer_live(settings, mock_graph):
settings = settings.model_copy(update={"dry_run": False})
from howl.db.models import EmailLog
executor = ActionExecutor(mock_graph, settings)
log_entry = MagicMock(spec=EmailLog)
log_entry.mailbox = "test@example.com"
log_entry.graph_message_id = "msg-001"
log_entry.sender_name = "Alice"
log_entry.sender_address = "alice@test.com"
log_entry.subject = "Test"
executor.execute(log_entry, "move_customer")
mock_graph.get_or_create_folder.assert_called_once_with("test@example.com", settings.folder_customers)
mock_graph.move_message.assert_called_once()
def test_action_executor_escalate_sends_notification(settings, mock_graph):
settings = settings.model_copy(update={
"dry_run": False,
"notification_webhook_url": "https://hooks.example.com/notify",
})
from howl.db.models import EmailLog
executor = ActionExecutor(mock_graph, settings)
log_entry = MagicMock(spec=EmailLog)
log_entry.mailbox = "test@example.com"
log_entry.graph_message_id = "msg-escalate"
log_entry.sender_name = "Bob"
log_entry.sender_address = "bob@client.com"
log_entry.subject = "URGENT"
with patch("httpx.post") as mock_post:
mock_post.return_value = MagicMock(status_code=200, raise_for_status=MagicMock())
executor.execute(log_entry, "escalate")
mock_graph.flag_message.assert_called_once()
mock_post.assert_called_once()
call_kwargs = mock_post.call_args
assert call_kwargs[0][0] == "https://hooks.example.com/notify"
assert "text" in call_kwargs[1]["json"]
assert call_kwargs[1]["timeout"] == 10
# ---------------------------------------------------------------------------
# Full pipeline integration (all external calls mocked)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_processor_process_message_happy_path(
settings, mock_graph, mock_llm, customer_message, customer_match
):
"""Full pipeline: message → DB lookup → LLM → classify → action → log."""
processor = EmailProcessor(mock_graph, mock_llm, settings)
inserted_log = MagicMock()
inserted_log.id = "log-uuid-001"
inserted_log.retry_count = 0
with (
patch("howl.pipeline.processor.db_engine") as mock_db_engine,
patch("howl.pipeline.processor.queries") as mock_queries,
):
# Set up async session context manager
mock_session = AsyncMock()
mock_db_engine.async_session.return_value.__aenter__ = AsyncMock(return_value=mock_session)
mock_db_engine.async_session.return_value.__aexit__ = AsyncMock(return_value=False)
mock_queries.is_already_processed = AsyncMock(return_value=False)
mock_queries.insert_email_log = AsyncMock(return_value=inserted_log)
mock_queries.lookup_sender = AsyncMock(return_value=customer_match)
mock_queries.update_email_log = AsyncMock()
await processor.process_message(customer_message)
# Verify DB was consulted for idempotency
mock_queries.is_already_processed.assert_called_once_with(mock_session, customer_message.id)
# Verify email log was created
mock_queries.insert_email_log.assert_called_once()
# Verify sender was looked up
mock_queries.lookup_sender.assert_called_once_with(mock_session, customer_message.sender_address)
# Verify LLM was called
mock_llm.classify.assert_called_once()
# Verify log was updated with completed status
final_update_calls = [
c for c in mock_queries.update_email_log.call_args_list
if c.kwargs.get("status") == "completed" or (len(c.args) > 2 and "completed" in str(c))
]
# At minimum update_email_log should have been called
assert mock_queries.update_email_log.call_count >= 2
@pytest.mark.asyncio
async def test_processor_skips_already_processed_message(
settings, mock_graph, mock_llm, customer_message
):
processor = EmailProcessor(mock_graph, mock_llm, settings)
with (
patch("howl.pipeline.processor.db_engine") as mock_db_engine,
patch("howl.pipeline.processor.queries") as mock_queries,
):
mock_session = AsyncMock()
mock_db_engine.async_session.return_value.__aenter__ = AsyncMock(return_value=mock_session)
mock_db_engine.async_session.return_value.__aexit__ = AsyncMock(return_value=False)
mock_queries.is_already_processed = AsyncMock(return_value=True)
await processor.process_message(customer_message)
# Should not proceed past idempotency check
mock_llm.classify.assert_not_called()
mock_graph.move_message.assert_not_called()
@pytest.mark.asyncio
async def test_processor_handles_llm_failure_gracefully(
settings, mock_graph, mock_llm, customer_message, customer_match
):
"""If LLM throws, the pipeline should catch it and log failure — not crash."""
mock_llm.classify.side_effect = RuntimeError("Anthropic API timeout")
processor = EmailProcessor(mock_graph, mock_llm, settings)
inserted_log = MagicMock()
inserted_log.id = "log-uuid-002"
inserted_log.retry_count = 0
with (
patch("howl.pipeline.processor.db_engine") as mock_db_engine,
patch("howl.pipeline.processor.queries") as mock_queries,
):
mock_session = AsyncMock()
mock_db_engine.async_session.return_value.__aenter__ = AsyncMock(return_value=mock_session)
mock_db_engine.async_session.return_value.__aexit__ = AsyncMock(return_value=False)
mock_queries.is_already_processed = AsyncMock(return_value=False)
mock_queries.insert_email_log = AsyncMock(return_value=inserted_log)
mock_queries.lookup_sender = AsyncMock(return_value=customer_match)
mock_queries.update_email_log = AsyncMock()
# Should NOT raise
await processor.process_message(customer_message)
# Should have logged failure
failed_updates = [
c for c in mock_queries.update_email_log.call_args_list
if c.kwargs.get("status") == "failed"
]
assert len(failed_updates) >= 1