using System.Text.Json;
using FuzzySharp;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using RealCV.Application.Data;
using RealCV.Application.DTOs;
using RealCV.Application.Helpers;
using RealCV.Application.Interfaces;
using RealCV.Application.Models;
using RealCV.Domain.Entities;
using RealCV.Infrastructure.Data;
using RealCV.Infrastructure.ExternalApis;

namespace RealCV.Infrastructure.Services;

public sealed class CompanyVerifierService : ICompanyVerifierService
{
    private readonly CompaniesHouseClient _companiesHouseClient;
    private readonly IDbContextFactory<ApplicationDbContext> _dbContextFactory;
    private readonly ICompanyNameMatcherService _aiMatcher;
    private readonly ILogger<CompanyVerifierService> _logger;

    private const int FuzzyMatchThreshold = 85;
    private const int CacheExpirationDays = 30;

    // Non-employment entity patterns organized by category
    // These are entities that exist in Companies House but are not typical employers
    private static readonly Dictionary<string, string[]> NonEmploymentEntityPatterns = new()
    {
        ["Clubs"] = new[] { "club", "fan club", "owners club", "car club", "supporters", "enthusiast", "aficionados" },
        ["Associations"] = new[] { "association", "society", "federation", "institute", "institution", "guild", "chamber of commerce" },
        ["Trusts"] = new[] { "benefit trust", "pension", "retirement", "employee trust", "share trust", "employee benefit", "superannuation", "provident" },
        ["Charities"] = new[] { "charity", "charitable", "foundation", "relief fund", "benevolent", "philanthropic" },
        ["Investment"] = new[] { "nominee", "custodian", "trustee", "investment trust", "unit trust", "investment fund", "capital partners" },
        ["Property"] = new[] { "freehold", "leasehold", "property management", "residents association", "management company rtm", "commonhold" },
        ["Religious"] = new[] { "church", "chapel", "mosque", "synagogue", "temple", "parish", "diocese", "ministry" },
        ["Sports"] = new[] { "football club", "cricket club", "rugby club", "golf club", "tennis club", "sports club", "athletic club" },
        ["Educational"] = new[] { "old boys", "old girls", "alumni", "school association", "pta", "parent teacher" },
        ["Professional"] = new[] { "chartered institute", "royal college", "professional body", "trade body", "regulatory body" }
    };

    // SIC codes that indicate non-trading or non-employment entities
    private static readonly HashSet<string> NonTradingSicCodes = new()
    {
        "99999", // Dormant company
        "64209", // Activities of holding companies (shell companies)
        "68100", // Buying and selling of own real estate (often shell)
    };

    // Words that indicate a subsidiary rather than the main trading company
    // When someone says they worked for "ASDA", they likely mean ASDA STORES LIMITED,
    // not ASDA DELIVERY LIMITED or ASDA PROPERTY HOLDINGS LIMITED
    private static readonly HashSet<string> SubsidiaryIndicators = new(StringComparer.OrdinalIgnoreCase)
    {
        // Logistics/Operations subsidiaries
        "delivery", "distribution", "logistics", "transport", "fleet", "haulage", "warehousing", "fulfilment",
        // Property subsidiaries
        "property", "properties", "estates", "land", "real estate", "developments",
        // Financial/Holding subsidiaries
        "holdings", "holding", "investments", "capital", "finance", "financial", "treasury",
        // Administrative subsidiaries
        "nominees", "nominee", "trustees", "trustee", "secretarial", "registrars",
        // Insurance subsidiaries
        "insurance", "assurance", "underwriting",
        // Specific function subsidiaries
        "leasing", "rentals", "procurement", "sourcing"
    };

    // Words that indicate a main trading/employer company (prefer these)
    private static readonly HashSet<string> MainCompanyIndicators = new(StringComparer.OrdinalIgnoreCase)
    {
        "stores", "retail", "supermarkets", "superstores", "hypermarkets",
        "manufacturing", "operations", "trading"
    };

    // Words that are "safe expansions" - they don't change company identity when added
    // "Boots" -> "BOOTS UK LIMITED" is safe (UK + LIMITED are standard suffixes)
    // "Boots" -> "THE BOOTS COMPANY PLC" is safe (THE + COMPANY + PLC are standard)
    // These are NOT counted as "extra meaningful words" in matching
    private static readonly HashSet<string> SafeExpansionWords = new(StringComparer.OrdinalIgnoreCase)
    {
        // Legal structures (already in SkipWords, but explicit here for clarity)
        "limited", "ltd", "plc", "llp", "llc", "inc", "incorporated", "corporation", "corp",
        "company", "co", "partners", "partnership",

        // Corporate structure words - these expand but don't change identity
        "group", "holdings", "holding", "the",

        // Geographic qualifiers (already in SkipWords)
        "uk", "u.k.", "gb", "britain", "british", "england", "europe", "european",
        "international", "global", "worldwide",

        // Common corporate suffixes
        "services", "solutions",  // "Boots" vs "BOOTS SERVICES LIMITED" - likely same company
        "retail", "stores",       // "ASDA" vs "ASDA STORES LIMITED" - same company
    };

    // Well-known UK trading names/brands mapped to their expected company name patterns
    // This provides fast-path recognition for major employers without full AI evaluation
    // Key: Trading name (how people commonly refer to the company)
    // Value: Tuple of (primary company number, list of acceptable name patterns)
    private static readonly Dictionary<string, WellKnownBrand> WellKnownBrands =
        new(StringComparer.OrdinalIgnoreCase)
    {
        // Retail
        ["Boots"] = new("00928555", "pharmacy/healthcare retail", new[] { "BOOTS UK", "BOOTS COMPANY", "BOOTS PLC", "BOOTS LIMITED" }),
        ["ASDA"] = new("00464777", "supermarket retail", new[] { "ASDA STORES", "ASDA GROUP", "ASDA PLC" }),
        ["Tesco"] = new("00445790", "supermarket retail", new[] { "TESCO PLC", "TESCO STORES", "TESCO UK" }),
        ["Sainsbury"] = new("00185647", "supermarket retail", new[] { "SAINSBURY", "J SAINSBURY" }),
        ["Sainsbury's"] = new("00185647", "supermarket retail", new[] { "SAINSBURY", "J SAINSBURY" }),
        ["Morrisons"] = new("00358949", "supermarket retail", new[] { "WM MORRISON", "MORRISON SUPERMARKETS" }),
        ["Waitrose"] = new("00099405", "supermarket retail", new[] { "WAITROSE", "JOHN LEWIS" }),
        ["Marks & Spencer"] = new("00214436", "retail", new[] { "MARKS AND SPENCER", "MARKS & SPENCER" }),
        ["M&S"] = new("00214436", "retail", new[] { "MARKS AND SPENCER", "MARKS & SPENCER" }),
        ["John Lewis"] = new("00233462", "retail", new[] { "JOHN LEWIS", "JOHN LEWIS PARTNERSHIP" }),
        ["Next"] = new("04425340", "retail", new[] { "NEXT PLC", "NEXT RETAIL", "NEXT GROUP" }),
        ["Primark"] = new("NI016270", "retail", new[] { "PRIMARK", "PENNEYS", "ASSOCIATED BRITISH FOODS" }),
        ["Argos"] = new("01081551", "retail", new[] { "ARGOS", "SAINSBURY'S ARGOS" }),

        // Finance & Insurance
        ["Legal & General"] = new("01417162", "insurance/financial services", new[] { "LEGAL AND GENERAL", "LEGAL & GENERAL", "L&G" }),
        ["Aviva"] = new("02468686", "insurance", new[] { "AVIVA", "NORWICH UNION" }),
        ["Prudential"] = new("01397169", "insurance/financial services", new[] { "PRUDENTIAL" }),
        ["AXA"] = new("01878835", "insurance", new[] { "AXA UK", "AXA INSURANCE" }),
        ["Lloyds Banking Group"] = new("00002065", "banking", new[] { "LLOYDS BANK", "LLOYDS BANKING" }),
        ["Barclays"] = new("01026167", "banking", new[] { "BARCLAYS BANK", "BARCLAYS PLC" }),
        ["HSBC"] = new("00014259", "banking", new[] { "HSBC BANK", "HSBC UK", "HSBC HOLDINGS" }),
        ["NatWest"] = new("00929027", "banking", new[] { "NATWEST", "NATIONAL WESTMINSTER", "NATWEST GROUP" }),

        // Professional Services
        ["EY"] = new("OC300001", "accounting/professional services", new[] { "ERNST & YOUNG", "EY LLP", "ERNST AND YOUNG" }),
        ["EY UK"] = new("OC300001", "accounting/professional services", new[] { "ERNST & YOUNG", "EY LLP" }),
        ["Ernst & Young"] = new("OC300001", "accounting/professional services", new[] { "ERNST & YOUNG", "EY LLP" }),
        ["PwC"] = new("OC303525", "accounting/professional services", new[] { "PRICEWATERHOUSECOOPERS", "PWC" }),
        ["Deloitte"] = new("OC303675", "accounting/professional services", new[] { "DELOITTE LLP", "DELOITTE" }),
        ["KPMG"] = new("OC301540", "accounting/professional services", new[] { "KPMG LLP", "KPMG" }),
        ["Accenture"] = new("04abortedt6", "consulting", new[] { "ACCENTURE UK", "ACCENTURE" }),
        ["McKinsey"] = new("03883888", "consulting", new[] { "MCKINSEY", "MCKINSEY & COMPANY" }),

        // Technology
        ["Checkout.com"] = new("09131987", "fintech/payments", new[] { "CHECKOUT.COM", "CHECKOUT LTD", "CHECKOUT PAYMENTS" }),
        ["Revolut"] = new("08804411", "fintech", new[] { "REVOLUT LTD", "REVOLUT" }),
        ["Monzo"] = new("09446231", "fintech", new[] { "MONZO BANK", "MONZO" }),
        ["Wise"] = new("07209813", "fintech", new[] { "WISE", "TRANSFERWISE" }),
        ["TransferWise"] = new("07209813", "fintech", new[] { "WISE", "TRANSFERWISE" }),
        ["Deliveroo"] = new("08167130", "food delivery", new[] { "DELIVEROO", "ROO" }),
        ["Just Eat"] = new("02465307", "food delivery", new[] { "JUST EAT", "JUST-EAT" }),
        ["IBM"] = new("00741598", "technology", new[] { "IBM", "IBM UK", "INTERNATIONAL BUSINESS MACHINES" }),
        ["IBM UK"] = new("00741598", "technology", new[] { "IBM", "IBM UK", "INTERNATIONAL BUSINESS MACHINES" }),
        ["JCB"] = new("00561597", "manufacturing/machinery", new[] { "JCB", "J C BAMFORD", "BAMFORD EXCAVATORS" }),
        ["Brewdog"] = new("SC311560", "brewing/hospitality", new[] { "BREWDOG", "BREW DOG" }),
        ["BrewDog"] = new("SC311560", "brewing/hospitality", new[] { "BREWDOG", "BREW DOG" }),
        ["Cazoo"] = new("11043737", "automotive/retail", new[] { "CAZOO" }),
        ["Gymshark"] = new("08396100", "retail/fitness", new[] { "GYMSHARK", "GYM SHARK" }),

        // Telecoms & Media
        ["BT"] = new("01800000", "telecoms", new[] { "BT GROUP", "BT PLC", "BRITISH TELECOM" }),
        ["Vodafone"] = new("01471587", "telecoms", new[] { "VODAFONE", "VODAFONE UK", "VODAFONE GROUP" }),
        ["Sky"] = new("02247735", "media/telecoms", new[] { "SKY UK", "SKY LIMITED", "BSkyB" }),
        ["Virgin Media"] = new("02591237", "telecoms", new[] { "VIRGIN MEDIA", "VIRGIN MEDIA O2" }),

        // Airlines & Travel
        ["British Airways"] = new("01777777", "airline", new[] { "BRITISH AIRWAYS", "BA PLC" }),
        ["BA"] = new("01777777", "airline", new[] { "BRITISH AIRWAYS", "BA PLC" }),
        ["easyJet"] = new("03959649", "airline", new[] { "EASYJET", "EASY JET" }),
        ["Ryanair"] = new("01914abortedt", "airline", new[] { "RYANAIR UK", "RYANAIR" }),

        // Energy
        ["BP"] = new("00102498", "oil & gas", new[] { "BP P.L.C.", "BP PLC", "BRITISH PETROLEUM" }),
        ["Shell"] = new("04366849", "oil & gas", new[] { "SHELL UK", "SHELL PLC", "ROYAL DUTCH SHELL" }),
        ["National Grid"] = new("04031152", "utilities", new[] { "NATIONAL GRID", "NATIONAL GRID PLC" }),
        ["SSE"] = new("SC117119", "utilities", new[] { "SSE PLC", "SSE ENERGY", "SCOTTISH AND SOUTHERN" }),
        ["Centrica"] = new("03033654", "utilities", new[] { "CENTRICA", "BRITISH GAS" }),

        // Pharma & Healthcare
        ["GSK"] = new("03888792", "pharmaceuticals", new[] { "GLAXOSMITHKLINE", "GSK PLC" }),
        ["GlaxoSmithKline"] = new("03888792", "pharmaceuticals", new[] { "GLAXOSMITHKLINE", "GSK" }),
        ["AstraZeneca"] = new("02723534", "pharmaceuticals", new[] { "ASTRAZENECA", "ASTRA ZENECA" }),

        // Manufacturing & Industrial
        ["Rolls-Royce"] = new("01003142", "aerospace/engineering", new[] { "ROLLS-ROYCE", "ROLLS ROYCE" }),
        ["BAE Systems"] = new("01470151", "defence/aerospace", new[] { "BAE SYSTEMS" }),
        ["Dyson"] = new("02023199", "manufacturing/technology", new[] { "DYSON", "DYSON TECHNOLOGY" }),

        // Automotive
        ["Jaguar Land Rover"] = new("01672070", "automotive", new[] { "JAGUAR LAND ROVER", "JLR" }),
        ["JLR"] = new("01672070", "automotive", new[] { "JAGUAR LAND ROVER" }),
    };

    // Record to hold well-known brand information
    private sealed record WellKnownBrand(
        string PrimaryCompanyNumber,
        string Industry,
        string[] AcceptablePatterns
    );


    public CompanyVerifierService(
        CompaniesHouseClient companiesHouseClient,
        IDbContextFactory<ApplicationDbContext> dbContextFactory,
        ICompanyNameMatcherService aiMatcher,
        ILogger<CompanyVerifierService> logger)
    {
        _companiesHouseClient = companiesHouseClient;
        _dbContextFactory = dbContextFactory;
        _aiMatcher = aiMatcher;
        _logger = logger;
    }

    public async Task<CompanyVerificationResult> VerifyCompanyAsync(
        string companyName,
        DateOnly? startDate,
        DateOnly? endDate,
        string? jobTitle = null)
    {
        ArgumentException.ThrowIfNullOrWhiteSpace(companyName);

        // Normalize company name - strip trailing punctuation that causes matching issues
        var normalizedName = NormalizeCompanyName(companyName);
        _logger.LogDebug("Verifying company: {CompanyName} (normalized: {NormalizedName})", companyName, normalizedName);
        var flags = new List<CompanyVerificationFlag>();

        // Check 1a: Is this a public sector employer?
        if (UKHistoricalEmployers.IsPublicSectorEmployer(normalizedName))
        {
            _logger.LogInformation("Recognised public sector employer: {CompanyName}", companyName);
            return new CompanyVerificationResult
            {
                ClaimedCompany = companyName,
                MatchedCompanyName = companyName,
                MatchedCompanyNumber = null,
                MatchScore = 100,
                IsVerified = true,
                VerificationNotes = "Public sector employer - not registered at Companies House",
                ClaimedStartDate = startDate,
                ClaimedEndDate = endDate,
                CompanyType = "public-sector",
                CompanyStatus = "active",
                ClaimedJobTitle = jobTitle,
                Flags = flags
            };
        }

        // Check 1b: Is this a charity or non-profit organisation?
        if (UKHistoricalEmployers.IsCharityEmployer(normalizedName))
        {
            _logger.LogInformation("Recognised charity employer: {CompanyName}", companyName);
            return new CompanyVerificationResult
            {
                ClaimedCompany = companyName,
                MatchedCompanyName = companyName,
                MatchedCompanyNumber = null,
                MatchScore = 100,
                IsVerified = true,
                VerificationNotes = "Charity/non-profit organisation",
                ClaimedStartDate = startDate,
                ClaimedEndDate = endDate,
                CompanyType = "charity",
                CompanyStatus = "active",
                ClaimedJobTitle = jobTitle,
                Flags = flags
            };
        }

        // Check 1c: Is this self-employment?
        if (IsSelfEmployment(normalizedName))
        {
            _logger.LogInformation("Recognised self-employment: {CompanyName}", companyName);
            return new CompanyVerificationResult
            {
                ClaimedCompany = companyName,
                MatchedCompanyName = companyName,
                MatchedCompanyNumber = null,
                MatchScore = 100,
                IsVerified = true,
                VerificationNotes = "Self-employed / freelance - not a registered company",
                ClaimedStartDate = startDate,
                ClaimedEndDate = endDate,
                CompanyType = "self-employed",
                CompanyStatus = "active",
                ClaimedJobTitle = jobTitle,
                Flags = flags
            };
        }

        // Check 1d: Is this an overseas/international company?
        var overseasInfo = DetectOverseasCompany(normalizedName);
        if (overseasInfo is not null)
        {
            _logger.LogInformation("Recognised overseas company: {CompanyName} ({Country})", companyName, overseasInfo.Value.Country);
            return new CompanyVerificationResult
            {
                ClaimedCompany = companyName,
                MatchedCompanyName = companyName,
                MatchedCompanyNumber = null,
                MatchScore = 100,
                IsVerified = true,
                VerificationNotes = $"Overseas company ({overseasInfo.Value.Country}) - not registered at UK Companies House",
                ClaimedStartDate = startDate,
                ClaimedEndDate = endDate,
                CompanyType = "overseas",
                CompanyStatus = "active",
                ClaimedJobTitle = jobTitle,
                Flags = flags
            };
        }

        // Check 1e: Is this a well-known brand we can fast-track verify?
        var knownBrand = GetWellKnownBrand(normalizedName);
        if (knownBrand != null && !string.IsNullOrEmpty(knownBrand.PrimaryCompanyNumber))
        {
            _logger.LogInformation("Fast-track verifying well-known brand '{CompanyName}' -> company #{CompanyNumber}",
                companyName, knownBrand.PrimaryCompanyNumber);

            // Look up the company directly from Companies House
            try
            {
                var companyDetails = await _companiesHouseClient.GetCompanyAsync(knownBrand.PrimaryCompanyNumber);
                if (companyDetails != null)
                {
                    DateOnly? incorporationDate = null;
                    if (!string.IsNullOrEmpty(companyDetails.DateOfCreation) &&
                        DateOnly.TryParse(companyDetails.DateOfCreation, out var parsedDate))
                    {
                        incorporationDate = parsedDate;
                    }

                    return new CompanyVerificationResult
                    {
                        ClaimedCompany = companyName,
                        MatchedCompanyName = companyDetails.CompanyName,
                        MatchedCompanyNumber = knownBrand.PrimaryCompanyNumber,
                        MatchScore = 100,
                        IsVerified = true,
                        VerificationNotes = $"Well-known brand ({knownBrand.Industry})",
                        ClaimedStartDate = startDate,
                        ClaimedEndDate = endDate,
                        CompanyType = companyDetails.Type,
                        CompanyStatus = companyDetails.CompanyStatus,
                        IncorporationDate = incorporationDate,
                        ClaimedJobTitle = jobTitle,
                        Flags = flags
                    };
                }
            }
            catch (Exception ex)
            {
                _logger.LogWarning(ex, "Failed to fast-track verify well-known brand '{CompanyName}', falling back to search", companyName);
                // Fall through to normal search
            }
        }

        // Check 2: Is this an internal division of a larger company?
        var parentCompany = UKHistoricalEmployers.GetParentCompanyForDivision(normalizedName);
        if (parentCompany != null)
        {
            _logger.LogInformation("Recognised division '{CompanyName}' of parent company '{ParentCompany}'", companyName, parentCompany);
            // Try to verify the parent company instead
            var parentResult = await VerifyCompanyAsync(parentCompany, startDate, endDate, jobTitle);
            if (parentResult.IsVerified)
            {
                return parentResult with
                {
                    ClaimedCompany = companyName,
                    VerificationNotes = $"Internal division of {parentResult.MatchedCompanyName}"
                };
            }
            // If parent verification failed, return a partial match
            return new CompanyVerificationResult
            {
                ClaimedCompany = companyName,
                MatchedCompanyName = parentCompany,
                MatchedCompanyNumber = null,
                MatchScore = 85,
                IsVerified = true,
                VerificationNotes = $"Recognised as division of {parentCompany}",
                ClaimedStartDate = startDate,
                ClaimedEndDate = endDate,
                ClaimedJobTitle = jobTitle,
                Flags = flags
            };
        }

        // Check 3: Is this a known historical employer?
        var historicalInfo = UKHistoricalEmployers.GetHistoricalEmployerInfo(normalizedName);
        if (historicalInfo != null)
        {
            _logger.LogInformation("Recognised historical employer: {CompanyName} -> {Successor}", companyName, historicalInfo.SuccessorName);

            // If we have a company number for the successor, try to get current details
            if (!string.IsNullOrEmpty(historicalInfo.CompanyNumber))
            {
                try
                {
                    var successorDetails = await _companiesHouseClient.GetCompanyAsync(historicalInfo.CompanyNumber);
                    if (successorDetails != null)
                    {
                        return new CompanyVerificationResult
                        {
                            ClaimedCompany = companyName,
                            MatchedCompanyName = $"{companyName} (now {successorDetails.CompanyName})",
                            MatchedCompanyNumber = historicalInfo.CompanyNumber,
                            MatchScore = 90,
                            IsVerified = true,
                            VerificationNotes = $"Historical company. {historicalInfo.Notes}",
                            ClaimedStartDate = startDate,
                            ClaimedEndDate = endDate,
                            CompanyType = successorDetails.Type,
                            CompanyStatus = "historical",
                            ClaimedJobTitle = jobTitle,
                            Flags = flags
                        };
                    }
                }
                catch (Exception ex)
                {
                    _logger.LogWarning(ex, "Failed to fetch successor company details for {CompanyNumber}", historicalInfo.CompanyNumber);
                }
            }

            // Return historical match without successor details
            return new CompanyVerificationResult
            {
                ClaimedCompany = companyName,
                MatchedCompanyName = $"{companyName} (now {historicalInfo.SuccessorName})",
                MatchedCompanyNumber = historicalInfo.CompanyNumber,
                MatchScore = 90,
                IsVerified = true,
                VerificationNotes = $"Historical company. {historicalInfo.Notes}",
                ClaimedStartDate = startDate,
                ClaimedEndDate = endDate,
                CompanyStatus = "historical",
                ClaimedJobTitle = jobTitle,
                Flags = flags
            };
        }

        // Check 4: Is this a compound company name (e.g., "ASDA/WALMART", "Corus & Laura Ashley Hotels")?
        // Try to verify each part individually
        var compoundResult = await TryVerifyCompoundNameAsync(normalizedName, companyName, startDate, endDate, jobTitle, flags);
        if (compoundResult is not null)
        {
            return compoundResult;
        }

        // Try to find a cached match first (but only if it existed at claimed start date)
        var cachedMatch = await FindCachedMatchAsync(normalizedName);
        if (cachedMatch is not null)
        {
            // Check if cached company existed at the claimed start date
            var cacheValid = !startDate.HasValue ||
                             cachedMatch.IncorporationDate == null ||
                             cachedMatch.IncorporationDate <= startDate.Value;

            if (cacheValid)
            {
                _logger.LogDebug("Found cached company match for: {CompanyName}", companyName);
                return CreateResultFromCache(cachedMatch, companyName, startDate, endDate, jobTitle, flags);
            }
            else
            {
                _logger.LogDebug("Cached company {CachedName} was incorporated after claimed start date, searching for alternatives", cachedMatch.CompanyName);
            }
        }

        // Search Companies House with fallback queries
        try
        {
            var searchQueries = GenerateSearchQueries(normalizedName);
            _logger.LogDebug("Generated {Count} search queries for '{CompanyName}': {Queries}",
                searchQueries.Count, normalizedName, string.Join(", ", searchQueries.Select(q => $"'{q}'")));

            // Collect all candidates from all search queries for AI matching
            var allCandidates = new Dictionary<string, CompaniesHouseSearchItem>();
            var fuzzyMatches = new List<(CompaniesHouseSearchItem Item, int Score)>();

            foreach (var query in searchQueries)
            {
                _logger.LogDebug("Searching Companies House with query: {Query}", query);
                var searchResponse = await _companiesHouseClient.SearchCompaniesAsync(query);

                if (searchResponse?.Items is null || searchResponse.Items.Count == 0)
                {
                    continue;
                }

                // Collect unique candidates
                foreach (var item in searchResponse.Items)
                {
                    if (!string.IsNullOrWhiteSpace(item.CompanyNumber) &&
                        !allCandidates.ContainsKey(item.CompanyNumber))
                    {
                        allCandidates[item.CompanyNumber] = item;
                    }
                }

                // Find fuzzy matches (as before) for fallback
                var fuzzyMatch = FindBestMatch(normalizedName, query, searchResponse.Items, startDate);
                if (fuzzyMatch is not null)
                {
                    fuzzyMatches.Add(fuzzyMatch.Value);
                }
            }

            if (allCandidates.Count == 0)
            {
                _logger.LogDebug("No candidates found for: {CompanyName} after trying {Count} queries", normalizedName, searchQueries.Count);
                return CreateUnverifiedResult(companyName, startDate, endDate, jobTitle,
                    "Company name could not be verified against official records");
            }

            // Use AI to find the best semantic match from all candidates
            _logger.LogDebug("Using AI to match '{CompanyName}' against {Count} candidates", normalizedName, allCandidates.Count);

            // Check if this is a well-known brand - if so, we can be more lenient with matching
            var wellKnownBrand = GetWellKnownBrand(normalizedName);
            if (wellKnownBrand != null)
            {
                _logger.LogDebug("Recognised well-known brand '{CompanyName}' (industry: {Industry})",
                    normalizedName, wellKnownBrand.Industry);
            }

            // Extract core identifiers from the original company name
            var originalCoreWords = ExtractCoreIdentifiers(normalizedName);
            var normalizedUpper = normalizedName.ToUpperInvariant();

            // Pre-filter candidates: reject those missing significant core words from the original
            // This prevents "Northwick Industries Limited" matching "NORTHWICK LIMITED" (missing INDUSTRIES)
            // BUT: for well-known brands or candidates with only safe expansions, we're more lenient
            var filteredCandidates = allCandidates.Values
                .Where(c =>
                {
                    // All original core words must appear in the candidate
                    var candidateTitle = c.Title.ToUpperInvariant();
                    var hasAllCores = originalCoreWords.Count == 0 ||
                        originalCoreWords.All(w => candidateTitle.Contains(w));

                    if (!hasAllCores)
                    {
                        // For well-known brands, check if candidate matches acceptable patterns
                        if (wellKnownBrand != null && MatchesWellKnownBrandPatterns(c.Title, wellKnownBrand))
                        {
                            _logger.LogDebug("Pre-filter allowing '{Candidate}' - matches well-known brand pattern for '{Original}'",
                                c.Title, normalizedName);
                            return true;
                        }

                        _logger.LogDebug("Pre-filter rejected '{Candidate}' - missing core words from '{Original}'. " +
                            "Required: [{Required}]", c.Title, normalizedName, string.Join(", ", originalCoreWords));
                        return false;
                    }

                    // For candidates that have all core words, check if extra words are safe
                    // This is a RELAXED filter for AI candidates - we let the AI make the final call
                    // But we still log for debugging
                    var candidateCores = ExtractCoreIdentifiers(c.Title);
                    var meaningfulExtras = CountMeaningfulExtraWords(originalCoreWords, candidateCores);
                    if (meaningfulExtras > 0)
                    {
                        _logger.LogDebug("Pre-filter note: '{Candidate}' has {ExtraCount} meaningful extra words vs '{Original}'",
                            c.Title, meaningfulExtras, normalizedName);
                    }

                    return true; // Let AI evaluate candidates with extra words
                })
                .ToList();

            _logger.LogDebug("Pre-filtered {Original} candidates to {Filtered} candidates",
                allCandidates.Count, filteredCandidates.Count);

            // Sort remaining candidates by fuzzy relevance to the search term before taking top 10
            // This ensures the most likely matches are sent to the AI, not just arbitrary entries
            var candidatesForAI = filteredCandidates
                .Select(c => new
                {
                    Item = c,
                    Score = Fuzz.TokenSetRatio(normalizedUpper, c.Title.ToUpperInvariant())
                })
                .OrderByDescending(x => x.Score)
                .Take(10)
                .Select(x => new CompanyCandidate
                {
                    CompanyName = x.Item.Title,
                    CompanyNumber = x.Item.CompanyNumber,
                    CompanyStatus = x.Item.CompanyStatus,
                    DateOfCreation = x.Item.DateOfCreation
                })
                .ToList();

            _logger.LogDebug("Top candidates for AI matching (sorted by relevance): {Candidates}",
                string.Join(", ", candidatesForAI.Select(c => $"{c.CompanyName} [{c.CompanyNumber}]")));

            // Pass industry context if this is a well-known brand
            var industryHint = wellKnownBrand?.Industry;
            var aiResult = await _aiMatcher.FindBestMatchAsync(normalizedName, candidatesForAI, industryHint);

            CompaniesHouseSearchItem? matchedItem = null;
            int matchScore;

            // Get best fuzzy match for potential fallback
            var bestFuzzy = fuzzyMatches.Count > 0
                ? fuzzyMatches.OrderByDescending(m => m.Score).First()
                : ((CompaniesHouseSearchItem Item, int Score)?)null;

            if (aiResult is not null && aiResult.IsMatch)
            {
                // AI found a valid match
                matchedItem = allCandidates.GetValueOrDefault(aiResult.CandidateCompanyNumber);
                matchScore = aiResult.ConfidenceScore;
                _logger.LogInformation(
                    "AI matched '{ClaimedName}' to '{MatchedName}' with {Score}% confidence. Reasoning: {Reasoning}",
                    companyName, aiResult.CandidateCompanyName, aiResult.ConfidenceScore, aiResult.Reasoning);
            }
            else if (fuzzyMatches.Count > 0)
            {
                // AI didn't find a match - check if it explicitly rejected or just failed
                if (aiResult?.MatchType == "NoMatch")
                {
                    // AI explicitly rejected. Only override if fuzzy match passes STRICT validation.
                    // We trust the AI's judgment - only override in clear-cut cases where
                    // the fuzzy match is essentially identical to the original OR has only safe expansions.
                    if (bestFuzzy.HasValue && bestFuzzy.Value.Score >= 90)
                    {
                        var originalCores = ExtractCoreIdentifiers(normalizedName);
                        var matchCores = ExtractCoreIdentifiers(bestFuzzy.Value.Item.Title);

                        // All original core words must appear in the match
                        var allCoresPresent = originalCores.Count == 0 ||
                            originalCores.All(c => bestFuzzy.Value.Item.Title.Contains(c, StringComparison.OrdinalIgnoreCase));

                        // Count MEANINGFUL extra core words (excluding safe expansions like UK, LIMITED, COMPANY)
                        var meaningfulExtras = CountMeaningfulExtraWords(originalCores, matchCores);

                        // For short company names (1-2 core words), no meaningful extras allowed
                        // For longer names, allow up to 1 meaningful extra
                        // But if it's a well-known brand matching an acceptable pattern, allow more
                        var isShortName = originalCores.Count <= 2;
                        var maxAllowedExtras = isShortName ? 0 : 1;

                        // Well-known brand override: if the match fits acceptable patterns, allow it
                        var brandOverride = wellKnownBrand != null &&
                            MatchesWellKnownBrandPatterns(bestFuzzy.Value.Item.Title, wellKnownBrand);

                        var reasonableExtras = meaningfulExtras <= maxAllowedExtras || brandOverride;

                        // Additional check: if match has significantly MORE core words than original,
                        // it's likely a different company entirely
                        var coreDifference = matchCores.Count - originalCores.Count;
                        var acceptableCoreDifference = coreDifference <= 2; // Allow 2 extra total (could be safe expansions)

                        if (allCoresPresent && reasonableExtras && acceptableCoreDifference)
                        {
                            _logger.LogInformation(
                                "AI rejected '{CompanyName}' but fuzzy match '{MatchedName}' ({Score}%) passes validation. " +
                                "Original cores: [{OriginalCores}], Match cores: [{MatchCores}], MeaningfulExtras: {Extra}, BrandOverride: {Override}",
                                normalizedName, bestFuzzy.Value.Item.Title, bestFuzzy.Value.Score,
                                string.Join(", ", originalCores), string.Join(", ", matchCores), meaningfulExtras, brandOverride);
                            matchedItem = bestFuzzy.Value.Item;
                            matchScore = bestFuzzy.Value.Score;
                        }
                        else
                        {
                            _logger.LogDebug(
                                "AI rejected '{CompanyName}' and fuzzy match '{MatchedName}' fails validation. " +
                                "AllCoresPresent: {AllCores}, MeaningfulExtras: {Extra} (max: {MaxAllowed}), CoreDiff: {CoreDiff}, BrandOverride: {Override}",
                                normalizedName, bestFuzzy.Value.Item.Title, allCoresPresent, meaningfulExtras, maxAllowedExtras, coreDifference, brandOverride);
                            return CreateUnverifiedResult(companyName, startDate, endDate, jobTitle,
                                "Company name could not be verified - no matching company found in official records");
                        }
                    }
                    else
                    {
                        _logger.LogDebug("AI explicitly rejected all candidates for '{CompanyName}'. Reasoning: {Reasoning}",
                            normalizedName, aiResult?.Reasoning ?? "No match found");
                        return CreateUnverifiedResult(companyName, startDate, endDate, jobTitle,
                            "Company name could not be verified - no matching company found in official records");
                    }
                }
                else
                {
                    // AI failed (API error, etc.) - fall back to fuzzy matching
                    _logger.LogWarning("AI matching failed for '{CompanyName}', falling back to fuzzy matching", normalizedName);
                    matchedItem = bestFuzzy!.Value.Item;
                    matchScore = bestFuzzy!.Value.Score;
                }
            }
            else
            {
                _logger.LogDebug("No valid match found for: {CompanyName}", normalizedName);
                return CreateUnverifiedResult(companyName, startDate, endDate, jobTitle,
                    "Company name could not be verified against official records");
            }

            if (matchedItem is null)
            {
                _logger.LogDebug("No valid match found for: {CompanyName}", companyName);
                return CreateUnverifiedResult(companyName, startDate, endDate, jobTitle,
                    "Company name could not be verified against official records");
            }

            var match = (Item: matchedItem, Score: matchScore);

            // Fetch full company details for additional data
            var companyDetails = await _companiesHouseClient.GetCompanyAsync(match.Item.CompanyNumber);

            // Cache the matched company with full details
            await CacheCompanyAsync(match.Item, companyDetails);

            _logger.LogInformation(
                "Verified company {ClaimedName} matched to {MatchedName} with score {Score}%",
                companyName, match.Item.Title, match.Score);

            // Run all verification checks
            var incorporationDate = DateHelpers.ParseDate(match.Item.DateOfCreation);
            var dissolutionDate = DateHelpers.ParseDate(match.Item.DateOfCessation);
            var companyStatus = match.Item.CompanyStatus;
            var companyType = match.Item.CompanyType;
            var sicCodes = companyDetails?.SicCodes ?? match.Item.SicCodes;
            var accountsCategory = companyDetails?.Accounts?.LastAccounts?.Type;

            // Check 1: Employment before company incorporation
            CheckIncorporationDate(flags, startDate, incorporationDate, match.Item.Title);

            // Check 2: Employment at dissolved company
            CheckDissolutionDate(flags, endDate, dissolutionDate, companyStatus, match.Item.Title);

            // Check 3: Dormant company check
            CheckDormantCompany(flags, accountsCategory, jobTitle, match.Item.Title);

            // Check 4: Company size vs job title
            CheckCompanySizeVsRole(flags, accountsCategory, jobTitle, match.Item.Title);

            // Check 5: Job title plausibility for PLCs
            var (jobPlausible, jobNotes) = CheckJobTitlePlausibility(jobTitle, companyType);
            if (jobPlausible == false)
            {
                flags.Add(new CompanyVerificationFlag
                {
                    Type = "ImplausibleJobTitle",
                    Severity = "Critical",
                    Message = jobNotes ?? "Job title requires verification",
                    ScoreImpact = -15
                });
            }

            return new CompanyVerificationResult
            {
                ClaimedCompany = companyName,
                MatchedCompanyName = match.Item.Title,
                MatchedCompanyNumber = match.Item.CompanyNumber,
                MatchScore = match.Score,
                IsVerified = true,
                VerificationNotes = null,
                ClaimedStartDate = startDate,
                ClaimedEndDate = endDate,
                CompanyType = companyType,
                CompanyStatus = companyStatus,
                IncorporationDate = incorporationDate,
                DissolutionDate = dissolutionDate,
                AccountsCategory = accountsCategory,
                SicCodes = sicCodes,
                ClaimedJobTitle = jobTitle,
                JobTitlePlausible = jobPlausible,
                JobTitleNotes = jobNotes,
                Flags = flags
            };
        }
        catch (CompaniesHouseRateLimitException ex)
        {
            _logger.LogWarning(ex, "Rate limit hit while verifying company: {CompanyName}", companyName);
            return CreateUnverifiedResult(companyName, startDate, endDate, jobTitle,
                "Verification temporarily unavailable due to rate limiting");
        }
    }

    public async Task<List<CompanySearchResult>> SearchCompaniesAsync(string query)
    {
        ArgumentException.ThrowIfNullOrWhiteSpace(query);

        _logger.LogDebug("Searching companies for query: {Query}", query);

        var response = await _companiesHouseClient.SearchCompaniesAsync(query);

        if (response?.Items is null)
        {
            return [];
        }

        return response.Items.Select(item => new CompanySearchResult
        {
            CompanyNumber = item.CompanyNumber,
            CompanyName = item.Title,
            CompanyStatus = item.CompanyStatus ?? "Unknown",
            IncorporationDate = DateHelpers.ParseDate(item.DateOfCreation),
            AddressSnippet = item.AddressSnippet
        }).ToList();
    }

    public async Task<bool?> VerifyDirectorAsync(
        string companyNumber,
        string candidateName,
        DateOnly? startDate,
        DateOnly? endDate)
    {
        if (string.IsNullOrWhiteSpace(companyNumber) || string.IsNullOrWhiteSpace(candidateName))
        {
            return null;
        }

        try
        {
            var officers = await _companiesHouseClient.GetOfficersAsync(companyNumber);

            if (officers?.Items is null || officers.Items.Count == 0)
            {
                _logger.LogDebug("No officers found for company {CompanyNumber}", companyNumber);
                return null;
            }

            // Normalize candidate name for comparison
            var normalizedCandidate = NormalizeName(candidateName);

            foreach (var officer in officers.Items)
            {
                // Check if officer role is director-like
                var role = officer.OfficerRole?.ToLowerInvariant() ?? "";
                if (!role.Contains("director") && !role.Contains("secretary"))
                {
                    continue;
                }

                // Fuzzy match the name
                var normalizedOfficer = NormalizeName(officer.Name);
                var matchScore = Fuzz.Ratio(normalizedCandidate, normalizedOfficer);

                if (matchScore >= 80) // High threshold for name matching
                {
                    // Check date overlap
                    var appointedOn = DateHelpers.ParseDate(officer.AppointedOn);
                    var resignedOn = DateHelpers.ParseDate(officer.ResignedOn);

                    // If no claimed dates, just check if names match
                    if (!startDate.HasValue && !endDate.HasValue)
                    {
                        _logger.LogDebug(
                            "Found matching director {OfficerName} for candidate {CandidateName} at company {CompanyNumber}",
                            officer.Name, candidateName, companyNumber);
                        return true;
                    }

                    // Check if employment period overlaps with directorship
                    var datesOverlap = DatesOverlap(
                        startDate, endDate,
                        appointedOn, resignedOn);

                    if (datesOverlap)
                    {
                        _logger.LogDebug(
                            "Verified director {OfficerName} matches candidate {CandidateName} with overlapping dates",
                            officer.Name, candidateName);
                        return true;
                    }
                }
            }

            _logger.LogDebug(
                "No matching director found for candidate {CandidateName} at company {CompanyNumber}",
                candidateName, companyNumber);
            return false;
        }
        catch (CompaniesHouseRateLimitException)
        {
            _logger.LogWarning("Rate limit hit while verifying director for company {CompanyNumber}", companyNumber);
            return null;
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Error verifying director for company {CompanyNumber}", companyNumber);
            return null;
        }
    }

    private static string NormalizeName(string name)
    {
        if (string.IsNullOrWhiteSpace(name)) return "";

        // Companies House often stores names as "SURNAME, Firstname"
        // Convert to "Firstname Surname" format for comparison
        var normalized = name.ToUpperInvariant().Trim();

        if (normalized.Contains(','))
        {
            var parts = normalized.Split(',', 2);
            if (parts.Length == 2)
            {
                normalized = $"{parts[1].Trim()} {parts[0].Trim()}";
            }
        }

        return normalized;
    }

    private static bool DatesOverlap(DateOnly? start1, DateOnly? end1, DateOnly? start2, DateOnly? end2)
    {
        // If no dates, assume overlap
        if (!start1.HasValue && !end1.HasValue) return true;
        if (!start2.HasValue && !end2.HasValue) return true;

        // Use default dates for missing values
        var s1 = start1 ?? DateOnly.MinValue;
        var e1 = end1 ?? DateOnly.MaxValue;
        var s2 = start2 ?? DateOnly.MinValue;
        var e2 = end2 ?? DateOnly.MaxValue;

        // Check overlap: periods overlap if one starts before the other ends
        return s1 <= e2 && s2 <= e1;
    }

    #region Verification Checks

    private static void CheckIncorporationDate(
        List<CompanyVerificationFlag> flags,
        DateOnly? claimedStartDate,
        DateOnly? incorporationDate,
        string companyName)
    {
        if (claimedStartDate.HasValue && incorporationDate.HasValue)
        {
            if (claimedStartDate.Value < incorporationDate.Value)
            {
                flags.Add(new CompanyVerificationFlag
                {
                    Type = "EmploymentBeforeIncorporation",
                    Severity = "Critical",
                    Message = $"Claimed employment at '{companyName}' starting {claimedStartDate:MMM yyyy} is before company incorporation date {incorporationDate:MMM yyyy}",
                    ScoreImpact = -20
                });
            }
        }
    }

    private static void CheckDissolutionDate(
        List<CompanyVerificationFlag> flags,
        DateOnly? claimedEndDate,
        DateOnly? dissolutionDate,
        string? companyStatus,
        string companyName)
    {
        var isDissolvedStatus = companyStatus?.ToLowerInvariant() is "dissolved" or "liquidation" or "administration";

        if (dissolutionDate.HasValue && isDissolvedStatus)
        {
            // Allow 3 month buffer for wind-down
            var bufferDate = dissolutionDate.Value.AddMonths(3);

            if (claimedEndDate.HasValue && claimedEndDate.Value > bufferDate)
            {
                flags.Add(new CompanyVerificationFlag
                {
                    Type = "EmploymentAtDissolvedCompany",
                    Severity = "Critical",
                    Message = $"Claimed employment at '{companyName}' until {claimedEndDate:MMM yyyy} but company was dissolved on {dissolutionDate:MMM yyyy}",
                    ScoreImpact = -20
                });
            }
            else if (!claimedEndDate.HasValue) // Current employment at dissolved company
            {
                flags.Add(new CompanyVerificationFlag
                {
                    Type = "CurrentEmploymentAtDissolvedCompany",
                    Severity = "Critical",
                    Message = $"Claims current employment at '{companyName}' but company was dissolved on {dissolutionDate:MMM yyyy}",
                    ScoreImpact = -25
                });
            }
        }
    }

    private static void CheckDormantCompany(
        List<CompanyVerificationFlag> flags,
        string? accountsCategory,
        string? jobTitle,
        string companyName)
    {
        if (string.IsNullOrWhiteSpace(accountsCategory)) return;

        var isDormant = accountsCategory.ToLowerInvariant().Contains("dormant");
        if (!isDormant) return;

        // Directors can maintain dormant companies, but other roles are suspicious
        var title = jobTitle?.ToLowerInvariant() ?? "";
        var isDirectorRole = title.Contains("director") || title.Contains("company secretary");

        if (!isDirectorRole)
        {
            flags.Add(new CompanyVerificationFlag
            {
                Type = "EmploymentAtDormantCompany",
                Severity = "Warning",
                Message = $"Claimed active employment as '{jobTitle}' at '{companyName}' which files dormant accounts",
                ScoreImpact = -10
            });
        }
    }

    private static void CheckCompanySizeVsRole(
        List<CompanyVerificationFlag> flags,
        string? accountsCategory,
        string? jobTitle,
        string companyName)
    {
        if (string.IsNullOrWhiteSpace(accountsCategory) || string.IsNullOrWhiteSpace(jobTitle)) return;

        var category = accountsCategory.ToLowerInvariant();
        var title = jobTitle.ToLowerInvariant();

        // Micro-entity: < 10 employees, < £632k turnover
        var isMicroEntity = category.Contains("micro");

        // Check for senior management roles at micro companies
        var isSeniorRole = title.Contains("vp") ||
                          title.Contains("vice president") ||
                          title.Contains("head of") ||
                          title.Contains("chief") ||
                          title.Contains("director of") ||
                          title.Contains("senior director");

        // At micro companies, having many senior roles is suspicious
        if (isMicroEntity && isSeniorRole)
        {
            flags.Add(new CompanyVerificationFlag
            {
                Type = "SeniorRoleAtMicroCompany",
                Severity = "Warning",
                Message = $"Claimed senior role '{jobTitle}' at '{companyName}' which files micro-entity accounts (typically <10 employees)",
                ScoreImpact = -10
            });
        }
    }

    private static (bool? IsPlausible, string? Notes) CheckJobTitlePlausibility(string? jobTitle, string? companyType)
    {
        if (string.IsNullOrWhiteSpace(jobTitle) || string.IsNullOrWhiteSpace(companyType))
        {
            return (null, null);
        }

        var title = jobTitle.Trim().ToLowerInvariant();
        var type = companyType.Trim().ToLowerInvariant();

        // Check if this is a PLC (Public Limited Company) - these are large companies
        var isPlc = type.Contains("plc") || type.Contains("public limited");

        // Check for C-suite / very senior roles
        var isCsuiteRole = title.Contains("ceo") ||
                          title.Contains("chief executive") ||
                          title.Contains("cto") ||
                          title.Contains("chief technology") ||
                          title.Contains("cfo") ||
                          title.Contains("chief financial") ||
                          title.Contains("coo") ||
                          title.Contains("chief operating") ||
                          title.Contains("cio") ||
                          title.Contains("chief information") ||
                          title.Contains("managing director") ||
                          title == "md" ||
                          title.Contains("chairman") ||
                          title.Contains("chairwoman") ||
                          title.Contains("chairperson") ||
                          title.Contains("president");

        // Check for board-level roles
        var isBoardRole = title.Contains("board member") ||
                         title.Contains("non-executive director") ||
                         title.Contains("executive director") ||
                         (title == "director" && !title.Contains("of"));

        if (isPlc && (isCsuiteRole || isBoardRole))
        {
            return (false, $"Claimed senior role '{jobTitle}' at a PLC requires verification - C-suite positions at public companies are publicly disclosed");
        }

        // Check for VP/SVP at PLCs (also usually disclosed)
        var isVpRole = title.Contains("vice president") ||
                      title.Contains("vp ") ||
                      title.StartsWith("vp") ||
                      title.Contains("svp") ||
                      title.Contains("senior vice president") ||
                      title.Contains("evp") ||
                      title.Contains("executive vice president");

        if (isPlc && isVpRole)
        {
            return (false, $"Claimed VP-level role '{jobTitle}' at a PLC - senior positions at public companies should be verifiable");
        }

        return (true, null);
    }

    #endregion

    #region Helper Methods

    /// <summary>
    /// Normalizes a company name by removing trailing punctuation and cleaning up common issues.
    /// </summary>
    private static string NormalizeCompanyName(string companyName)
    {
        if (string.IsNullOrWhiteSpace(companyName))
            return companyName;

        var normalized = companyName.Trim();

        // Remove trailing punctuation (dots, commas, etc.) that cause matching issues
        // e.g., "Glaxo Research & Development Ltd." -> "Glaxo Research & Development Ltd"
        normalized = normalized.TrimEnd('.', ',', ';', ':', '!', '?');

        // Normalize multiple spaces to single space
        normalized = System.Text.RegularExpressions.Regex.Replace(normalized, @"\s+", " ");

        return normalized;
    }

    /// <summary>
    /// Detects if a company name indicates self-employment or freelance work.
    /// </summary>
    private static bool IsSelfEmployment(string companyName)
    {
        var lower = companyName.ToLowerInvariant().Trim();

        // Exact matches
        var selfEmployedTerms = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
        {
            "self-employed", "self employed", "selfemployed",
            "freelance", "freelancer", "free-lance", "free lance",
            "contractor", "independent contractor",
            "sole trader", "sole-trader", "soletrader",
            "consultant", "independent consultant",
            "self", "myself", "own business", "own company",
            "private practice", "private consultancy",
            "portfolio career", "various clients",
            "contract work", "contracting"
        };

        if (selfEmployedTerms.Contains(lower))
            return true;

        // Pattern matches
        if (lower.StartsWith("self-employed") || lower.StartsWith("self employed"))
            return true;
        if (lower.StartsWith("freelance") || lower.StartsWith("free-lance"))
            return true;
        if (lower.EndsWith("(self-employed)") || lower.EndsWith("(freelance)"))
            return true;
        if (lower.Contains("self-employed as") || lower.Contains("freelancing as"))
            return true;

        return false;
    }

    /// <summary>
    /// Detects if a company name indicates an overseas/international company not registered in the UK.
    /// Returns the detected country if found, null otherwise.
    /// </summary>
    private static (string Country, string BaseName)? DetectOverseasCompany(string companyName)
    {
        var lower = companyName.ToLowerInvariant();

        // Countries and their variations that indicate non-UK companies
        var overseasCountries = new Dictionary<string[], string>
        {
            // North America
            { new[] { "canada", "canadian" }, "Canada" },
            { new[] { "usa", "u.s.a.", "u.s.", "united states", "american", "america" }, "United States" },
            { new[] { "mexico", "mexican" }, "Mexico" },

            // Europe (non-UK)
            { new[] { "ireland", "irish", "eire", "dublin" }, "Ireland" },
            { new[] { "france", "french", "paris" }, "France" },
            { new[] { "germany", "german", "deutsche", "berlin", "munich" }, "Germany" },
            { new[] { "spain", "spanish", "madrid", "barcelona" }, "Spain" },
            { new[] { "italy", "italian", "milan", "rome" }, "Italy" },
            { new[] { "netherlands", "dutch", "holland", "amsterdam" }, "Netherlands" },
            { new[] { "belgium", "belgian", "brussels" }, "Belgium" },
            { new[] { "switzerland", "swiss", "zurich", "geneva" }, "Switzerland" },
            { new[] { "austria", "austrian", "vienna" }, "Austria" },
            { new[] { "sweden", "swedish", "stockholm" }, "Sweden" },
            { new[] { "norway", "norwegian", "oslo" }, "Norway" },
            { new[] { "denmark", "danish", "copenhagen" }, "Denmark" },
            { new[] { "finland", "finnish", "helsinki" }, "Finland" },
            { new[] { "poland", "polish", "warsaw" }, "Poland" },
            { new[] { "portugal", "portuguese", "lisbon" }, "Portugal" },
            { new[] { "greece", "greek", "athens" }, "Greece" },
            { new[] { "cyprus", "cypriot", "nicosia" }, "Cyprus" },
            { new[] { "czech", "prague" }, "Czech Republic" },
            { new[] { "hungary", "hungarian", "budapest" }, "Hungary" },
            { new[] { "romania", "romanian", "bucharest" }, "Romania" },

            // Asia Pacific
            { new[] { "australia", "australian", "sydney", "melbourne" }, "Australia" },
            { new[] { "new zealand", "nz", "auckland", "wellington" }, "New Zealand" },
            { new[] { "japan", "japanese", "tokyo" }, "Japan" },
            { new[] { "china", "chinese", "beijing", "shanghai", "hong kong" }, "China" },
            { new[] { "india", "indian", "mumbai", "delhi", "bangalore" }, "India" },
            { new[] { "singapore", "singaporean" }, "Singapore" },
            { new[] { "malaysia", "malaysian", "kuala lumpur" }, "Malaysia" },
            { new[] { "korea", "korean", "seoul" }, "South Korea" },
            { new[] { "taiwan", "taiwanese", "taipei" }, "Taiwan" },
            { new[] { "thailand", "thai", "bangkok" }, "Thailand" },
            { new[] { "philippines", "filipino", "manila" }, "Philippines" },
            { new[] { "indonesia", "indonesian", "jakarta" }, "Indonesia" },
            { new[] { "vietnam", "vietnamese", "hanoi", "ho chi minh" }, "Vietnam" },

            // Middle East & Africa
            { new[] { "uae", "u.a.e.", "dubai", "abu dhabi", "emirates" }, "UAE" },
            { new[] { "saudi", "riyadh", "jeddah" }, "Saudi Arabia" },
            { new[] { "qatar", "doha" }, "Qatar" },
            { new[] { "israel", "israeli", "tel aviv" }, "Israel" },
            { new[] { "south africa", "johannesburg", "cape town" }, "South Africa" },
            { new[] { "egypt", "egyptian", "cairo" }, "Egypt" },
            { new[] { "nigeria", "nigerian", "lagos" }, "Nigeria" },
            { new[] { "kenya", "kenyan", "nairobi" }, "Kenya" },

            // South America
            { new[] { "brazil", "brazilian", "sao paulo", "rio" }, "Brazil" },
            { new[] { "argentina", "argentine", "buenos aires" }, "Argentina" },
            { new[] { "chile", "chilean", "santiago" }, "Chile" },
            { new[] { "colombia", "colombian", "bogota" }, "Colombia" },
        };

        // Check for country indicators at the end or in the name
        foreach (var (terms, country) in overseasCountries)
        {
            foreach (var term in terms)
            {
                // Check if name ends with country (e.g., "BMW Group Canada")
                if (lower.EndsWith(" " + term))
                {
                    var baseName = companyName[..^(term.Length + 1)].Trim();
                    return (country, baseName);
                }

                // Check for patterns like "Company Name (Country)" or "Company Name - Country"
                if (lower.EndsWith($"({term})") || lower.EndsWith($"- {term}") || lower.EndsWith($", {term}"))
                {
                    var idx = lower.LastIndexOf(term);
                    var baseName = companyName[..(idx - 2)].Trim().TrimEnd('(', '-', ',').Trim();
                    return (country, baseName);
                }

                // Check for "Country Office" or "Country Branch" patterns
                if (lower.Contains($"{term} office") || lower.Contains($"{term} branch") ||
                    lower.Contains($"{term} division") || lower.Contains($"{term} operations"))
                {
                    return (country, companyName);
                }
            }
        }

        // Check for international organization patterns
        var internationalPatterns = new[]
        {
            "national guard", "armed forces", "military", "army", "navy", "air force",
            "embassy", "consulate", "foreign ministry",
            "max planck", "fraunhofer", "cnrs", "csiro", "nasa", "esa",
            "world bank", "imf", "united nations", "un ", "nato", "who ", "unesco"
        };

        foreach (var pattern in internationalPatterns)
        {
            if (lower.Contains(pattern))
            {
                return ("International", companyName);
            }
        }

        return null;
    }

    /// <summary>
    /// Attempts to verify compound company names by detecting if multiple companies are mentioned.
    /// Only triggers for names with potential separators (/, &amp;, "and") to avoid unnecessary AI calls.
    /// </summary>
    private async Task<CompanyVerificationResult?> TryVerifyCompoundNameAsync(
        string normalizedName,
        string originalName,
        DateOnly? startDate,
        DateOnly? endDate,
        string? jobTitle,
        List<CompanyVerificationFlag> flags)
    {
        // Quick check: only process names that might be compound
        // Look for separators that could indicate multiple companies
        var hasPotentialSeparator = normalizedName.Contains('/')
            || normalizedName.Contains(" & ")
            || normalizedName.Contains(" and ", StringComparison.OrdinalIgnoreCase);

        if (!hasPotentialSeparator)
        {
            return null;
        }

        // Use AI to determine if this is a compound name and extract parts
        var extractedParts = await _aiMatcher.ExtractCompanyNamesAsync(normalizedName);

        if (extractedParts is null || extractedParts.Count < 2)
        {
            // AI determined this is a single company (e.g., "Ernst & Young")
            return null;
        }

        _logger.LogDebug("AI detected compound company name '{Name}', extracted parts: {Parts}",
            originalName, string.Join(", ", extractedParts.Select(p => $"'{p}'")));

        // Try to verify each extracted part - return success on first match
        foreach (var part in extractedParts)
        {
            // Skip parts that are too short
            if (part.Length < 3) continue;

            _logger.LogDebug("Trying to verify compound part: '{Part}'", part);

            // Recursively verify this part
            var partResult = await VerifyCompanyAsync(part, startDate, endDate, jobTitle);

            if (partResult.IsVerified)
            {
                _logger.LogInformation("Compound name '{Original}' verified via part '{Part}' -> {Match}",
                    originalName, part, partResult.MatchedCompanyName);

                return partResult with
                {
                    ClaimedCompany = originalName,
                    VerificationNotes = $"Verified via '{part}': {partResult.VerificationNotes ?? partResult.MatchedCompanyName}"
                };
            }
        }

        // None of the parts could be verified
        _logger.LogDebug("No parts of compound name '{Name}' could be verified", originalName);
        return null;
    }

    private async Task<CompanyCache?> FindCachedMatchAsync(string companyName)
    {
        var cutoffDate = DateTime.UtcNow.AddDays(-CacheExpirationDays);

        await using var dbContext = await _dbContextFactory.CreateDbContextAsync();

        var cachedCompanies = await dbContext.CompanyCache
            .Where(c => c.CachedAt >= cutoffDate)
            .ToListAsync();

        if (cachedCompanies.Count == 0)
        {
            return null;
        }

        // Extract core identifiers from the original company name
        var originalCoreWords = ExtractCoreIdentifiers(companyName);

        var matches = cachedCompanies
            .Where(c => !string.IsNullOrWhiteSpace(c.CompanyName))
            .Where(c =>
            {
                // All original core words must appear in the cached company name
                var cachedTitle = c.CompanyName.ToUpperInvariant();
                return originalCoreWords.Count == 0 ||
                    originalCoreWords.All(w => cachedTitle.Contains(w));
            })
            .Select(c => new { Company = c, Score = Fuzz.TokenSetRatio(companyName.ToUpperInvariant(), c.CompanyName.ToUpperInvariant()) })
            .Where(m => m.Score >= FuzzyMatchThreshold)
            .OrderByDescending(m => m.Score)
            .FirstOrDefault();

        return matches?.Company;
    }

    private (CompaniesHouseSearchItem Item, int Score)? FindBestMatch(
        string companyName,
        string searchQuery,
        List<CompaniesHouseSearchItem> items,
        DateOnly? claimedStartDate)
    {
        var normalizedOriginal = companyName.ToUpperInvariant();
        var normalizedQuery = searchQuery.ToUpperInvariant();

        // Extract core identifying words that MUST appear in any valid match
        // This prevents "BMW Group Canada" matching "CANADA LIFE GROUP" just because of common words
        // and "Lloyds Bowmaker" matching "LLOYDS ALARMS" (missing "Bowmaker")
        var coreWords = ExtractCoreIdentifiers(companyName);
        var queryCoreWords = ExtractCoreIdentifiers(searchQuery);

        var originalLower = companyName.ToLowerInvariant();
        var queryLower = searchQuery.ToLowerInvariant();

        // Determine which entity types the search is explicitly looking for
        var searchEntityTypes = GetSearchEntityTypes(originalLower, queryLower);

        // Match against both the original company name AND the search query used
        // This handles cases like "Matthew Walker (Northern Foods Plc)" where we
        // search for "Northern Foods Plc" but need to match against it, not the full name
        var matches = items
            .Where(item => !string.IsNullOrWhiteSpace(item.Title))
            .Where(item =>
            {
                var itemTitle = item.Title.ToUpperInvariant();
                var itemTitleLower = item.Title.ToLowerInvariant();
                var itemCoreWords = ExtractCoreIdentifiers(item.Title);

                // Validate that ALL core identifiers from the ORIGINAL company name appear in the match
                // "Northwick Industries Limited" must have BOTH "NORTHWICK" and "INDUSTRIES" in the match
                // This prevents partial search queries (e.g., "Northwick") from bypassing validation
                var hasAllOriginalCores = coreWords.Count == 0 || coreWords.All(w => itemTitle.Contains(w));
                if (!hasAllOriginalCores) return false;

                // Additional check: ensure the match doesn't have too many EXTRA MEANINGFUL core words
                // "Families First" should NOT match "Families Against Conformity" because
                // "Against" and "Conformity" are extra significant words
                // BUT: safe expansions like "UK", "LIMITED", "COMPANY", "GROUP" don't count
                // So "Boots" -> "BOOTS UK LIMITED" is OK (no meaningful extras)
                // But "Boots" -> "BOOTS AND BEARDS" is NOT OK (BEARDS is meaningful extra)
                if (coreWords.Count > 0 && hasAllOriginalCores)
                {
                    // Count only MEANINGFUL extra words (not safe expansions)
                    var meaningfulExtras = CountMeaningfulExtraWords(coreWords, itemCoreWords);
                    var isShortName = coreWords.Count <= 2;

                    // For short names: no meaningful extras allowed (prevents "Boots" → "BOOTS AND BEARDS")
                    // For longer names: allow up to 1 meaningful extra
                    var maxAllowedExtras = isShortName ? 0 : 1;

                    if (meaningfulExtras > maxAllowedExtras)
                    {
                        return false;
                    }
                }

                // Filter out non-employment entities unless explicitly searching for that type
                if (!IsValidEmploymentEntity(itemTitleLower, searchEntityTypes))
                {
                    return false;
                }

                return true;
            })
            .Select(item =>
            {
                var itemTitle = item.Title.ToUpperInvariant();
                var itemTitleLower = item.Title.ToLowerInvariant();
                var scoreVsOriginal = Fuzz.TokenSetRatio(normalizedOriginal, itemTitle);
                var scoreVsQuery = Fuzz.TokenSetRatio(normalizedQuery, itemTitle);
                var baseScore = Math.Max(scoreVsOriginal, scoreVsQuery);

                // Calculate priority adjustment for main company vs subsidiary
                var priorityScore = CalculateCompanyPriorityScore(itemTitleLower, originalLower, queryLower);

                return (Item: item, Score: baseScore, PriorityScore: priorityScore);
            })
            .Where(m => m.Score >= FuzzyMatchThreshold)
            .ToList();

        _logger.LogDebug("Found {Count} matches above threshold for '{CompanyName}' (query: '{Query}')", matches.Count, companyName, searchQuery);
        foreach (var m in matches.Take(5))
        {
            _logger.LogDebug("  Match: {Title} ({Number}), Score: {Score}, Priority: {Priority}, DateOfCreation: {Date}",
                m.Item.Title, m.Item.CompanyNumber, m.Score, m.PriorityScore, m.Item.DateOfCreation ?? "null");
        }

        if (matches.Count == 0) return null;

        // If we have a claimed start date, prefer companies that existed at that time
        if (claimedStartDate.HasValue)
        {
            _logger.LogDebug("Filtering for companies that existed at claimed start date: {StartDate}", claimedStartDate.Value);

            var existedAtStartDate = matches
                .Where(m =>
                {
                    var incDate = DateHelpers.ParseDate(m.Item.DateOfCreation);
                    var existed = incDate == null || incDate <= claimedStartDate.Value;
                    _logger.LogDebug("  {Title}: IncDate={IncDate}, Existed={Existed}",
                        m.Item.Title, incDate?.ToString() ?? "null", existed);
                    return existed;
                })
                // Sort by priority first, then by fuzzy score
                .OrderByDescending(m => m.PriorityScore)
                .ThenByDescending(m => m.Score)
                .ToList();

            _logger.LogDebug("Companies that existed at start date: {Count}", existedAtStartDate.Count);

            // If any matches existed at the start date, prefer those
            if (existedAtStartDate.Count > 0)
            {
                var selected = existedAtStartDate[0];
                _logger.LogDebug("Selected: {Title} ({Number}), Priority: {Priority}", selected.Item.Title, selected.Item.CompanyNumber, selected.PriorityScore);
                return (selected.Item, selected.Score);
            }

            // No companies existed at the claimed start date - don't match a wrong company
            _logger.LogDebug("No companies found that existed at claimed start date {StartDate}, returning no match", claimedStartDate.Value);
            return null;
        }

        // No start date provided - sort by priority then score
        var fallback = matches
            .OrderByDescending(m => m.PriorityScore)
            .ThenByDescending(m => m.Score)
            .First();
        _logger.LogDebug("No start date filter, using highest priority: {Title} ({Number}), Priority: {Priority}", fallback.Item.Title, fallback.Item.CompanyNumber, fallback.PriorityScore);
        return (fallback.Item, fallback.Score);
    }

    /// <summary>
    /// Calculates a priority score for company matching.
    /// Higher scores = more likely to be the main employer company.
    /// Penalizes subsidiaries (delivery, property, holdings, etc.) unless explicitly searched for.
    /// Boosts main trading companies (stores, retail, etc.).
    /// </summary>
    private static int CalculateCompanyPriorityScore(string itemTitleLower, string originalLower, string queryLower)
    {
        var score = 0;

        // Check if search explicitly mentions subsidiary indicators
        var searchText = originalLower + " " + queryLower;

        // Penalize subsidiary indicators (unless search explicitly included them)
        // Use word boundary matching to avoid "SCOTLAND" matching "land"
        foreach (var indicator in SubsidiaryIndicators)
        {
            if (ContainsWholeWord(itemTitleLower, indicator))
            {
                // Only penalize if the search didn't explicitly include this indicator
                if (!ContainsWholeWord(searchText, indicator))
                {
                    score -= 10; // Significant penalty for subsidiaries
                }
                break; // Only apply one subsidiary penalty
            }
        }

        // Boost main company indicators
        foreach (var indicator in MainCompanyIndicators)
        {
            if (ContainsWholeWord(itemTitleLower, indicator))
            {
                score += 5; // Boost for main trading companies
                break; // Only apply one boost
            }
        }

        // Slight boost for PLC (usually the parent/main company)
        if (itemTitleLower.EndsWith(" plc"))
        {
            score += 3;
        }

        return score;
    }

    private async Task CacheCompanyAsync(CompaniesHouseSearchItem item, CompaniesHouseCompany? details)
    {
        try
        {
            await using var dbContext = await _dbContextFactory.CreateDbContextAsync();

            var existingCache = await dbContext.CompanyCache
                .FirstOrDefaultAsync(c => c.CompanyNumber == item.CompanyNumber);

            var sicCodes = details?.SicCodes ?? item.SicCodes;
            var sicCodesJson = sicCodes != null ? JsonSerializer.Serialize(sicCodes) : null;
            var accountsCategory = details?.Accounts?.LastAccounts?.Type;

            if (existingCache is not null)
            {
                existingCache.CompanyName = item.Title;
                existingCache.Status = item.CompanyStatus ?? "Unknown";
                existingCache.CompanyType = item.CompanyType;
                existingCache.IncorporationDate = DateHelpers.ParseDate(item.DateOfCreation);
                existingCache.DissolutionDate = DateHelpers.ParseDate(item.DateOfCessation);
                existingCache.AccountsCategory = accountsCategory;
                existingCache.SicCodesJson = sicCodesJson;
                existingCache.CachedAt = DateTime.UtcNow;
            }
            else
            {
                var cacheEntry = new CompanyCache
                {
                    CompanyNumber = item.CompanyNumber,
                    CompanyName = item.Title,
                    Status = item.CompanyStatus ?? "Unknown",
                    CompanyType = item.CompanyType,
                    IncorporationDate = DateHelpers.ParseDate(item.DateOfCreation),
                    DissolutionDate = DateHelpers.ParseDate(item.DateOfCessation),
                    AccountsCategory = accountsCategory,
                    SicCodesJson = sicCodesJson,
                    CachedAt = DateTime.UtcNow
                };

                dbContext.CompanyCache.Add(cacheEntry);
            }

            await dbContext.SaveChangesAsync();
        }
        catch (DbUpdateException ex) when (ex.InnerException?.Message.Contains("PK_CompanyCache") == true)
        {
            // Race condition: another task already cached this company - ignore
            _logger.LogDebug("Company {CompanyNumber} already cached by another task", item.CompanyNumber);
        }
    }

    private CompanyVerificationResult CreateResultFromCache(
        CompanyCache cached,
        string claimedCompany,
        DateOnly? startDate,
        DateOnly? endDate,
        string? jobTitle,
        List<CompanyVerificationFlag> flags)
    {
        var matchScore = Fuzz.TokenSetRatio(
            claimedCompany.ToUpperInvariant(),
            cached.CompanyName.ToUpperInvariant());

        List<string>? sicCodes = null;
        if (!string.IsNullOrEmpty(cached.SicCodesJson))
        {
            try
            {
                sicCodes = JsonSerializer.Deserialize<List<string>>(cached.SicCodesJson);
            }
            catch (JsonException)
            {
                // Ignore malformed JSON in cache
            }
        }

        // Run all verification checks
        CheckIncorporationDate(flags, startDate, cached.IncorporationDate, cached.CompanyName);
        CheckDissolutionDate(flags, endDate, cached.DissolutionDate, cached.Status, cached.CompanyName);
        CheckDormantCompany(flags, cached.AccountsCategory, jobTitle, cached.CompanyName);
        CheckCompanySizeVsRole(flags, cached.AccountsCategory, jobTitle, cached.CompanyName);

        var (jobPlausible, jobNotes) = CheckJobTitlePlausibility(jobTitle, cached.CompanyType);
        if (jobPlausible == false)
        {
            flags.Add(new CompanyVerificationFlag
            {
                Type = "ImplausibleJobTitle",
                Severity = "Critical",
                Message = jobNotes ?? "Job title requires verification",
                ScoreImpact = -15
            });
        }

        return new CompanyVerificationResult
        {
            ClaimedCompany = claimedCompany,
            MatchedCompanyName = cached.CompanyName,
            MatchedCompanyNumber = cached.CompanyNumber,
            MatchScore = matchScore,
            IsVerified = true,
            VerificationNotes = null,
            ClaimedStartDate = startDate,
            ClaimedEndDate = endDate,
            CompanyType = cached.CompanyType,
            CompanyStatus = cached.Status,
            IncorporationDate = cached.IncorporationDate,
            DissolutionDate = cached.DissolutionDate,
            AccountsCategory = cached.AccountsCategory,
            SicCodes = sicCodes,
            ClaimedJobTitle = jobTitle,
            JobTitlePlausible = jobPlausible,
            JobTitleNotes = jobNotes,
            Flags = flags
        };
    }

    private static CompanyVerificationResult CreateUnverifiedResult(
        string companyName,
        DateOnly? startDate,
        DateOnly? endDate,
        string? jobTitle,
        string reason)
    {
        return new CompanyVerificationResult
        {
            ClaimedCompany = companyName,
            MatchedCompanyName = null,
            MatchedCompanyNumber = null,
            MatchScore = 0,
            IsVerified = false,
            VerificationNotes = reason,
            ClaimedStartDate = startDate,
            ClaimedEndDate = endDate,
            ClaimedJobTitle = jobTitle
        };
    }

    /// <summary>
    /// Generates alternative search queries to find companies that may be registered
    /// with slightly different names (e.g., "U.K." vs "UK", "Limited" vs "Ltd").
    /// Also handles "Brand (Parent Company)" format by extracting and prioritizing the parent.
    /// </summary>
    private static List<string> GenerateSearchQueries(string companyName)
    {
        var queries = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
        var normalized = companyName.Trim();

        // Step 0a: Check for "Brand (Parent Company)" format and extract parent company
        // Parent company is more likely to be the registered name, so search it first
        var parentMatch = System.Text.RegularExpressions.Regex.Match(normalized, @"\(([^)]+)\)\s*$");
        if (parentMatch.Success)
        {
            var parentCompany = parentMatch.Groups[1].Value.Trim();
            // Generate queries for parent company first (higher priority)
            foreach (var parentQuery in GenerateNameVariations(parentCompany))
            {
                queries.Add(parentQuery);
            }
            // Also try the brand name without parenthetical
            var brandName = normalized[..parentMatch.Index].Trim();
            if (brandName.Length >= 3)
            {
                foreach (var brandQuery in GenerateNameVariations(brandName))
                {
                    queries.Add(brandQuery);
                }
            }
        }

        // Step 0b: Check for "Name1/Name2" format (e.g., "ASDA/WALMART")
        // Try each part separately as they may be different registered names
        if (normalized.Contains('/'))
        {
            var parts = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
            foreach (var part in parts)
            {
                if (part.Length >= 3)
                {
                    foreach (var partQuery in GenerateNameVariations(part))
                    {
                        queries.Add(partQuery);
                    }
                }
            }
        }

        // Step 0c: Try first word as potential parent company (e.g., "UNILEVER BESTFOOD" -> "UNILEVER")
        // Many company names are "ParentCompany Division" or "ParentCompany Brand"
        var words = normalized.Split(' ', StringSplitOptions.RemoveEmptyEntries);
        if (words.Length >= 2)
        {
            var firstWord = words[0];
            // Only try if first word is substantial (not "The", "A", common prefixes)
            var skipWords = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
                { "the", "a", "an", "uk", "british", "national", "international", "global", "new" };

            if (firstWord.Length >= 4 && !skipWords.Contains(firstWord))
            {
                foreach (var firstWordQuery in GenerateNameVariations(firstWord))
                {
                    queries.Add(firstWordQuery);
                }
                // Also try first word + PLC/Limited for major corporations
                queries.Add(firstWord + " PLC");
                queries.Add(firstWord + " Limited");
            }
        }

        // Also add variations of the full original name
        foreach (var query in GenerateNameVariations(normalized))
        {
            queries.Add(query);
        }

        return queries.ToList();
    }

    /// <summary>
    /// Generates name variations for a single company name (UK/U.K., Ltd/Limited, etc.)
    /// </summary>
    private static List<string> GenerateNameVariations(string name)
    {
        var variations = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { name };

        // Step 1: Generate UK/U.K. variations
        var ukVariants = new List<string> { name };

        if (name.Contains(" UK", StringComparison.OrdinalIgnoreCase))
        {
            // Add U.K. variant
            var withDots = name
                .Replace(" UK ", " U.K. ", StringComparison.OrdinalIgnoreCase)
                .Replace(" UK", " U.K.", StringComparison.OrdinalIgnoreCase);
            if (withDots != name)
                ukVariants.Add(withDots);
        }
        if (name.Contains(" U.K.", StringComparison.OrdinalIgnoreCase))
        {
            // Add UK variant (no dots)
            var withoutDots = name
                .Replace(" U.K. ", " UK ", StringComparison.OrdinalIgnoreCase)
                .Replace(" U.K.", " UK", StringComparison.OrdinalIgnoreCase);
            if (withoutDots != name)
                ukVariants.Add(withoutDots);
        }

        // Step 2: For each UK variant, generate suffix variations (Ltd/Limited)
        foreach (var variant in ukVariants)
        {
            variations.Add(variant);

            // Try Ltd -> Limited
            if (variant.EndsWith(" Ltd", StringComparison.OrdinalIgnoreCase))
            {
                variations.Add(variant[..^4] + " Limited");
            }
            // Try Limited -> Ltd
            else if (variant.EndsWith(" Limited", StringComparison.OrdinalIgnoreCase))
            {
                variations.Add(variant[..^8] + " Ltd");
            }
            // Try PLC variations
            else if (variant.EndsWith(" PLC", StringComparison.OrdinalIgnoreCase))
            {
                variations.Add(variant[..^4] + " Public Limited Company");
            }
            else if (variant.EndsWith(" Public Limited Company", StringComparison.OrdinalIgnoreCase))
            {
                variations.Add(variant[..^24] + " PLC");
            }
            // Try Plc (mixed case) variations
            else if (variant.EndsWith(" Plc", StringComparison.Ordinal))
            {
                variations.Add(variant[..^4] + " PLC");
                variations.Add(variant[..^4] + " Public Limited Company");
            }
        }

        // Step 3: Try core name without suffix
        var suffixesToRemove = new[] { " Ltd", " Limited", " PLC", " Plc", " LLP", " Inc", " Corporation", " Corp" };
        var coreName = name;
        foreach (var suffix in suffixesToRemove)
        {
            if (coreName.EndsWith(suffix, StringComparison.OrdinalIgnoreCase))
            {
                coreName = coreName[..^suffix.Length].Trim();
                break;
            }
        }
        if (coreName != name && coreName.Length >= 3)
        {
            variations.Add(coreName);
            variations.Add(coreName + " Limited");
            variations.Add(coreName + " PLC");

            // Also add U.K. variant of core name if applicable
            if (coreName.Contains(" UK", StringComparison.OrdinalIgnoreCase))
            {
                var coreWithDots = coreName
                    .Replace(" UK ", " U.K. ", StringComparison.OrdinalIgnoreCase)
                    .Replace(" UK", " U.K.", StringComparison.OrdinalIgnoreCase);
                variations.Add(coreWithDots);
                variations.Add(coreWithDots + " Limited");
            }
        }

        return variations.ToList();
    }

    /// <summary>
    /// Determines which non-employment entity categories the search query is explicitly looking for.
    /// Returns a set of category names (e.g., "Clubs", "Trusts") that should NOT be filtered out.
    /// </summary>
    private static HashSet<string> GetSearchEntityTypes(string originalLower, string queryLower)
    {
        var allowedCategories = new HashSet<string>();
        var searchTerms = originalLower + " " + queryLower;

        foreach (var (category, patterns) in NonEmploymentEntityPatterns)
        {
            // If any pattern from this category appears in the search, allow matches from that category
            if (patterns.Any(pattern => searchTerms.Contains(pattern)))
            {
                allowedCategories.Add(category);
            }
        }

        return allowedCategories;
    }

    /// <summary>
    /// Checks if a company title represents a valid employment entity.
    /// Filters out non-employment entities (clubs, trusts, etc.) unless the search explicitly targets that type.
    /// </summary>
    private static bool IsValidEmploymentEntity(string itemTitleLower, HashSet<string> allowedCategories)
    {
        foreach (var (category, patterns) in NonEmploymentEntityPatterns)
        {
            // Skip this category if the search explicitly allows it
            if (allowedCategories.Contains(category))
            {
                continue;
            }

            // Check if the item matches any pattern in this non-employment category
            // Use whole-word matching for single words, substring for multi-word patterns
            if (patterns.Any(pattern => pattern.Contains(' ')
                ? itemTitleLower.Contains(pattern)
                : ContainsWholeWord(itemTitleLower, pattern)))
            {
                return false; // This is a non-employment entity type that wasn't explicitly searched for
            }
        }

        return true; // No non-employment patterns matched, this is likely a valid employment entity
    }

    /// <summary>
    /// Checks if a string contains a word as a whole word (not as a substring of another word).
    /// E.g., "scotland" does NOT contain whole word "land", but "land holdings" does.
    /// </summary>
    private static bool ContainsWholeWord(string text, string word)
    {
        if (string.IsNullOrEmpty(text) || string.IsNullOrEmpty(word))
            return false;

        var pattern = $@"\b{System.Text.RegularExpressions.Regex.Escape(word)}\b";
        return System.Text.RegularExpressions.Regex.IsMatch(text, pattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase);
    }

    /// <summary>
    /// Checks if a CV company name matches a well-known brand and returns its info.
    /// </summary>
    private static WellKnownBrand? GetWellKnownBrand(string companyName)
    {
        if (string.IsNullOrWhiteSpace(companyName))
            return null;

        // Try exact match first
        if (WellKnownBrands.TryGetValue(companyName.Trim(), out var brand))
            return brand;

        // Try with common suffixes removed
        var normalized = companyName.Trim();
        var suffixes = new[] { " Ltd", " Limited", " PLC", " UK", " Group" };
        foreach (var suffix in suffixes)
        {
            if (normalized.EndsWith(suffix, StringComparison.OrdinalIgnoreCase))
            {
                var withoutSuffix = normalized[..^suffix.Length].Trim();
                if (WellKnownBrands.TryGetValue(withoutSuffix, out brand))
                    return brand;
            }
        }

        return null;
    }

    /// <summary>
    /// Checks if a candidate company name matches a well-known brand's acceptable patterns.
    /// </summary>
    private static bool MatchesWellKnownBrandPatterns(string candidateName, WellKnownBrand brand)
    {
        var upper = candidateName.ToUpperInvariant();
        return brand.AcceptablePatterns.Any(p => upper.Contains(p.ToUpperInvariant()));
    }

    /// <summary>
    /// Counts "meaningful" extra words in a candidate that aren't in the original.
    /// Excludes safe expansion words (UK, Limited, Company, Group, etc.)
    /// </summary>
    private static int CountMeaningfulExtraWords(List<string> originalCoreWords, List<string> candidateCoreWords)
    {
        var extraWords = candidateCoreWords
            .Where(w => !originalCoreWords.Any(o => o.Equals(w, StringComparison.OrdinalIgnoreCase)))
            .Where(w => !SafeExpansionWords.Contains(w))
            .ToList();

        return extraWords.Count;
    }

    /// <summary>
    /// Determines if the extra words in a candidate are "safe" expansions that don't change company identity.
    /// E.g., "Boots" -> "BOOTS UK LIMITED" has only safe expansions (UK, LIMITED)
    /// E.g., "Boots" -> "BOOTS AND BEARDS" has unsafe expansion (BEARDS)
    /// </summary>
    private static bool HasOnlySafeExpansions(string originalName, string candidateName, ILogger? logger = null)
    {
        var originalCores = ExtractCoreIdentifiers(originalName);
        var candidateCores = ExtractCoreIdentifiers(candidateName);

        // Find words in candidate that aren't in original
        var extraWords = candidateCores
            .Where(w => !originalCores.Any(o => o.Equals(w, StringComparison.OrdinalIgnoreCase)))
            .ToList();

        // Check if all extra words are safe expansions
        var unsafeWords = extraWords
            .Where(w => !SafeExpansionWords.Contains(w))
            .ToList();

        if (unsafeWords.Count > 0 && logger != null)
        {
            logger.LogDebug("Candidate '{Candidate}' has unsafe extra words: [{Unsafe}] (safe extras: [{Safe}])",
                candidateName,
                string.Join(", ", unsafeWords),
                string.Join(", ", extraWords.Where(w => SafeExpansionWords.Contains(w))));
        }

        return unsafeWords.Count == 0;
    }

    // Expanded skip words list for core identifier extraction
    // These words are too common to be meaningful differentiators between companies
    private static readonly HashSet<string> SkipWords = new(StringComparer.OrdinalIgnoreCase)
    {
        // Articles and conjunctions
        "the", "a", "an", "and", "or", "of", "for", "in", "at", "on", "by", "to", "with",

        // Geographic - Countries and regions
        "uk", "u.k.", "gb", "british", "britain", "england", "english", "scotland", "scottish",
        "wales", "welsh", "ireland", "irish", "northern",
        "europe", "european", "america", "american", "usa", "us", "u.s.", "u.s.a.",
        "canada", "canadian", "asia", "asian", "pacific", "atlantic",
        "australia", "australian", "africa", "african", "india", "indian",
        "france", "french", "germany", "german", "spain", "spanish", "italy", "italian",
        "japan", "japanese", "china", "chinese", "korea", "korean",
        "middle", "east", "west", "north", "south", "central", "western", "eastern",

        // Geographic - Cities
        "london", "manchester", "birmingham", "leeds", "glasgow", "edinburgh", "bristol",
        "liverpool", "sheffield", "newcastle", "cardiff", "belfast", "nottingham",
        "southampton", "portsmouth", "brighton", "leicester", "coventry", "hull",

        // Legal suffixes
        "limited", "ltd", "plc", "llp", "llc", "inc", "incorporated", "corporation", "corp",
        "company", "co", "partners", "partnership", "enterprises", "unlimited",
        "registered", "cic", "cio", "se", "ag", "gmbh", "sarl", "bv", "nv",

        // Business descriptors - only truly generic ones that don't identify the business type
        // Note: Removed words that can be meaningful business type identifiers:
        // - "industries", "technology", "solutions", "services", "consulting" - identify business type
        // - e.g., "Paramount Consulting" ≠ "Paramount", "Tech Solutions" ≠ "Tech"
        "group", "holdings", "holding", "parent", "subsidiary", "division", "branch",
        "commercial", "trading", "trade",
        "business", "businesses", "operational",
        "associated",

        // Size/Scope descriptors
        "national", "international", "global", "worldwide", "world", "regional", "local",
        "universal", "standard", "prime", "first", "one",
        // Note: Removed "general" and "premier" as they are meaningful in brand names
        // like "Legal & General", "General Electric", "Premier Inn"

        // Quality/Marketing terms
        "new", "modern", "advanced", "innovative", "premier", "elite", "premium",
        "quality", "superior", "excellent", "best", "top", "leading", "major",

        // Ownership indicators (excluding "royal" as it's a meaningful company identifier)
        "imperial", "crown", "state", "public", "private", "independent",
        "mutual", "cooperative", "coop", "community",

        // Time-related
        "century", "millennium", "annual", "year", "years",

        // Numbers as words
        "one", "two", "three", "four", "five", "first", "second", "third"
    };

    /// <summary>
    /// Extracts ALL core identifying words from a company name.
    /// These are significant words that aren't common prefixes/suffixes.
    /// E.g., "BMW Group Canada" -> ["BMW"], "Lloyds Bowmaker" -> ["LLOYDS", "BOWMAKER"]
    /// "Royal Bank of Scotland" -> ["ROYAL", "BANK"] (Scotland is a geographic skipWord)
    /// </summary>
    private static List<string> ExtractCoreIdentifiers(string companyName)
    {
        if (string.IsNullOrWhiteSpace(companyName)) return new List<string>();

        // Remove parenthetical content first
        var cleanName = System.Text.RegularExpressions.Regex.Replace(companyName, @"\([^)]*\)", "").Trim();

        // Split into words and collect all significant words
        var words = cleanName.Split(new[] { ' ', '-', '/', '&' }, StringSplitOptions.RemoveEmptyEntries);
        var coreWords = new List<string>();

        foreach (var word in words)
        {
            var cleanWord = word.Trim('.', ',', '\'');
            if (cleanWord.Length >= 2 && !SkipWords.Contains(cleanWord))
            {
                coreWords.Add(cleanWord.ToUpperInvariant());
            }
        }

        return coreWords;
    }

    #endregion
}
