using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using OFACScraper.Configuration; namespace OFACScraper; public class Application { private readonly OFACScraper _scraper; private readonly Exporter _exporter; private readonly CheckpointStore _checkpoint; private readonly OFACOptions _options; private readonly ILogger _logger; public Application( OFACScraper scraper, Exporter exporter, CheckpointStore checkpoint, IOptions options, ILogger logger) { _scraper = scraper; _exporter = exporter; _checkpoint = checkpoint; _options = options.Value; _logger = logger; } /// /// Full historical scrape: all years from StartYear to current year. /// Skips records already in checkpoint. /// public async Task RunFullAsync(CancellationToken ct = default) { var currentYear = DateTime.UtcNow.Year; _logger.LogInformation("Starting full scrape {StartYear}–{EndYear}", _options.StartYear, currentYear); var total = 0; for (var year = _options.StartYear; year <= currentYear; year++) { total += await ProcessYearAsync(year, ct); if (ct.IsCancellationRequested) break; } _logger.LogInformation("Full scrape complete. {Total} new records exported. DB total: {DbTotal}", total, _checkpoint.GetTotalCount()); return 0; } /// /// Daily/incremental run: scrapes current year only, exports any new records. /// public async Task RunDailyAsync(CancellationToken ct = default) { var currentYear = DateTime.UtcNow.Year; _logger.LogInformation("Starting daily scrape for {Year}", currentYear); var newRecords = await ProcessYearAsync(currentYear, ct); _logger.LogInformation("Daily scrape complete. {New} new records exported.", newRecords); return 0; } private async Task ProcessYearAsync(int year, CancellationToken ct) { var records = await _scraper.GetYearRecordsAsync(year, ct); var newCount = 0; foreach (var record in records) { if (ct.IsCancellationRequested) break; if (_checkpoint.HasRecord(record.TextId)) { _logger.LogDebug("Skipping {TextId} (already processed)", record.TextId); continue; } var success = await _exporter.ExportRecordAsync(record, ct); if (success) { _checkpoint.MarkProcessed( record.TextId, record.Date, record.Name, record.PenaltyTotalUsd, record.DocumentUrl, record.FileName, record.Year); newCount++; } } if (newCount > 0) _logger.LogInformation("Year {Year}: exported {Count} new records", year, newCount); return newCount; } }