Fix Verdict:Error from prose-wrapped AI responses

This commit is contained in:
2026-04-12 10:08:57 +01:00
parent 5c801cef4b
commit 8aa35469fc

View File

@@ -8,38 +8,43 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using SpamGuard.Configuration; using SpamGuard.Configuration;
using SpamGuard.Models; using SpamGuard.Models;
using SpamGuard.State;
public sealed partial class EmailClassifier public sealed partial class EmailClassifier
{ {
private readonly SpamGuardOptions _options; private readonly SpamGuardOptions _options;
private readonly ILogger<EmailClassifier> _logger; private readonly ILogger<EmailClassifier> _logger;
private readonly HttpClient _httpClient; private readonly HttpClient _httpClient;
private readonly OverrideStore _overrideStore;
private const string SystemPrompt = """ private const string BaseSystemPrompt = """
You are an email spam classifier. Analyze the following email and determine if it is spam or legitimate. You are an email spam classifier. Analyze the following email and determine if it is spam or legitimate.
Spam includes: Spam includes:
- Unsolicited marketing or promotional emails the recipient never signed up for - Unsolicited marketing or promotional emails the recipient never signed up for
- AI-generated emails designed to look like legitimate correspondence - AI-generated emails designed to look like legitimate correspondence
- Newsletter signups the recipient didn't request - Newsletter signups the recipient did not request
Legitimate includes: Legitimate includes:
- Emails from known contacts or businesses the recipient has a relationship with - Emails from known contacts or businesses the recipient has a relationship with
- Transactional emails (receipts, shipping notifications, password resets) - Transactional emails (receipts, shipping notifications, password resets)
- Emails the recipient would expect to receive - Emails the recipient would expect to receive
Respond with JSON only: YOU MUST respond with ONLY a JSON object. No explanation, no preamble, no markdown fencing.
Output exactly this structure and nothing else:
{"classification": "spam" | "legitimate", "confidence": 0.0-1.0, "reason": "brief explanation"} {"classification": "spam" | "legitimate", "confidence": 0.0-1.0, "reason": "brief explanation"}
"""; """;
public EmailClassifier( public EmailClassifier(
IOptions<SpamGuardOptions> options, IOptions<SpamGuardOptions> options,
ILogger<EmailClassifier> logger, ILogger<EmailClassifier> logger,
HttpClient httpClient) HttpClient httpClient,
OverrideStore overrideStore)
{ {
_options = options.Value; _options = options.Value;
_logger = logger; _logger = logger;
_httpClient = httpClient; _httpClient = httpClient;
_overrideStore = overrideStore;
} }
public string BuildPrompt(EmailSummary email) public string BuildPrompt(EmailSummary email)
@@ -62,11 +67,13 @@ public sealed partial class EmailClassifier
_logger.LogDebug("Classifying email UID={Uid} from {From}", email.Uid, email.From); _logger.LogDebug("Classifying email UID={Uid} from {From}", email.Uid, email.From);
var systemPrompt = BaseSystemPrompt + _overrideStore.BuildFewShotText(10);
var requestBody = new var requestBody = new
{ {
model = _options.Claude.Model, model = _options.Claude.Model,
max_tokens = 256, max_tokens = 256,
system = SystemPrompt, system = systemPrompt,
messages = new[] messages = new[]
{ {
new { role = "user", content = userMessage } new { role = "user", content = userMessage }
@@ -78,11 +85,27 @@ public sealed partial class EmailClassifier
{ {
Content = new StringContent(json, Encoding.UTF8, "application/json") Content = new StringContent(json, Encoding.UTF8, "application/json")
}; };
var isAnthropic = _options.Claude.BaseUrl.Contains("anthropic.com", StringComparison.OrdinalIgnoreCase);
if (isAnthropic)
{
request.Headers.Add("x-api-key", _options.Claude.ApiKey); request.Headers.Add("x-api-key", _options.Claude.ApiKey);
request.Headers.Add("anthropic-version", "2023-06-01"); request.Headers.Add("anthropic-version", "2023-06-01");
}
else
{
request.Headers.Authorization =
new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", _options.Claude.ApiKey);
}
var response = await _httpClient.SendAsync(request, ct); var response = await _httpClient.SendAsync(request, ct);
response.EnsureSuccessStatusCode();
if (!response.IsSuccessStatusCode)
{
var errorBody = await response.Content.ReadAsStringAsync(ct);
_logger.LogError("Claude API returned {StatusCode}: {Error}", (int)response.StatusCode, errorBody);
return null;
}
var responseJson = await response.Content.ReadAsStringAsync(ct); var responseJson = await response.Content.ReadAsStringAsync(ct);
using var doc = JsonDocument.Parse(responseJson); using var doc = JsonDocument.Parse(responseJson);
@@ -108,18 +131,32 @@ public sealed partial class EmailClassifier
// Strip markdown code fencing if present // Strip markdown code fencing if present
var cleaned = StripMarkdownFencing().Replace(text, "$1").Trim(); var cleaned = StripMarkdownFencing().Replace(text, "$1").Trim();
// Try direct parse first (happy path)
var result = TryParseJson(cleaned);
if (result != null) return result;
// Fall back: extract first {...} block from prose-wrapped responses
var start = cleaned.IndexOf('{');
var end = cleaned.LastIndexOf('}');
if (start >= 0 && end > start)
return TryParseJson(cleaned[start..(end + 1)]);
return null;
}
private static ClassificationResult? TryParseJson(string json)
{
try try
{ {
using var doc = JsonDocument.Parse(cleaned); using var doc = JsonDocument.Parse(json);
var root = doc.RootElement; var root = doc.RootElement;
return new ClassificationResult( return new ClassificationResult(
Classification: root.GetProperty("classification").GetString() ?? "unknown", Classification: root.GetProperty("classification").GetString() ?? "unknown",
Confidence: root.GetProperty("confidence").GetDouble(), Confidence: root.GetProperty("confidence").GetDouble(),
Reason: root.GetProperty("reason").GetString() ?? "" Reason: root.GetProperty("reason").GetString() ?? ""
); );
} }
catch (Exception) catch
{ {
return null; return null;
} }