92 lines
3.2 KiB
Python
92 lines
3.2 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from howl.db.queries import SenderMatch
|
||
|
|
from howl.llm.schemas import EmailClassification
|
||
|
|
from howl.pipeline import classifier
|
||
|
|
|
||
|
|
|
||
|
|
def make_classification(**kwargs) -> EmailClassification:
|
||
|
|
defaults = {
|
||
|
|
"classification": "customer_inquiry",
|
||
|
|
"confidence": 0.90,
|
||
|
|
"action": "move_customer",
|
||
|
|
"reasoning": "Test classification",
|
||
|
|
"priority": "normal",
|
||
|
|
"requires_human_review": False,
|
||
|
|
"tags": [],
|
||
|
|
}
|
||
|
|
defaults.update(kwargs)
|
||
|
|
return EmailClassification.model_validate(defaults)
|
||
|
|
|
||
|
|
|
||
|
|
def test_whitelist_sender_not_moved_to_spam(settings, whitelist_match):
|
||
|
|
classification = make_classification(action="move_spam", confidence=0.85)
|
||
|
|
final_action, overridden = classifier.decide(whitelist_match, classification, settings)
|
||
|
|
assert final_action == "inbox_keep"
|
||
|
|
assert overridden is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_customer_sender_not_moved_to_spam(settings, customer_match):
|
||
|
|
classification = make_classification(action="move_spam", confidence=0.85)
|
||
|
|
final_action, overridden = classifier.decide(customer_match, classification, settings)
|
||
|
|
assert final_action == "move_review"
|
||
|
|
assert overridden is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_requires_human_review_forces_move_review(settings, unknown_match):
|
||
|
|
classification = make_classification(
|
||
|
|
action="inbox_keep",
|
||
|
|
confidence=0.75,
|
||
|
|
requires_human_review=True,
|
||
|
|
)
|
||
|
|
final_action, overridden = classifier.decide(unknown_match, classification, settings)
|
||
|
|
assert final_action == "move_review"
|
||
|
|
assert overridden is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_low_confidence_falls_back_to_inbox_keep(settings, unknown_match):
|
||
|
|
# Settings has llm_confidence_threshold=0.60, this is 0.45
|
||
|
|
classification = make_classification(
|
||
|
|
action="move_spam",
|
||
|
|
confidence=0.45,
|
||
|
|
requires_human_review=False,
|
||
|
|
)
|
||
|
|
final_action, overridden = classifier.decide(unknown_match, classification, settings)
|
||
|
|
assert final_action == "inbox_keep"
|
||
|
|
assert overridden is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_high_confidence_customer_passes_through(settings, customer_match):
|
||
|
|
classification = make_classification(
|
||
|
|
action="move_customer",
|
||
|
|
confidence=0.95,
|
||
|
|
requires_human_review=False,
|
||
|
|
)
|
||
|
|
final_action, overridden = classifier.decide(customer_match, classification, settings)
|
||
|
|
assert final_action == "move_customer"
|
||
|
|
assert overridden is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_escalate_not_overridden_by_low_confidence(settings, customer_match):
|
||
|
|
"""Escalate action should survive even low confidence — it's already going to human review."""
|
||
|
|
classification = make_classification(
|
||
|
|
action="escalate",
|
||
|
|
confidence=0.40,
|
||
|
|
requires_human_review=True,
|
||
|
|
)
|
||
|
|
final_action, overridden = classifier.decide(customer_match, classification, settings)
|
||
|
|
assert final_action == "escalate"
|
||
|
|
|
||
|
|
|
||
|
|
def test_flag_follow_up_survives_requires_human_review(settings, unknown_match):
|
||
|
|
"""flag_follow_up is an acceptable requires_human_review action — should not be overridden."""
|
||
|
|
classification = make_classification(
|
||
|
|
action="flag_follow_up",
|
||
|
|
confidence=0.80,
|
||
|
|
requires_human_review=True,
|
||
|
|
)
|
||
|
|
final_action, overridden = classifier.decide(unknown_match, classification, settings)
|
||
|
|
assert final_action == "flag_follow_up"
|