feat: three major improvements - stable sources, archival, email alerts

1. Focus on Stable International/Regional Sources
   - Improved TED EU scraper (5 search strategies, 5 pages each)
   - All stable sources now hourly (TED EU, Sell2Wales, PCS Scotland, eTendersNI)
   - De-prioritize unreliable UK gov sites (100% removal rate)

2. Archival Feature
   - New DB columns: archived, archived_at, archived_snapshot, last_validated, validation_failures
   - Cleanup script now preserves full tender snapshots before archiving
   - Gradual failure handling (3 retries before archiving)
   - No data loss - historical record preserved

3. Email Alerts
   - Daily digest (8am) - all new tenders from last 24h
   - High-value alerts (every 4h) - tenders >£100k
   - Professional HTML emails with all tender details
   - Configurable via environment variables

Expected outcomes:
- 50-100 stable tenders (vs 26 currently)
- Zero 404 errors (archived data preserved)
- Proactive notifications (no missed opportunities)
- Historical archive for trend analysis

Files:
- scrapers/ted-eu.js (improved)
- cleanup-with-archival.mjs (new)
- send-tender-alerts.mjs (new)
- migrations/add-archival-fields.sql (new)
- THREE_IMPROVEMENTS_SUMMARY.md (documentation)

All cron jobs updated for hourly scraping + daily cleanup + alerts

test-dashboard.mjs

@@ -0,0 +1,106 @@
+import { chromium } from 'playwright';
+import pg from 'pg';
+
+const pool = new pg.Pool({
+  connectionString: 'postgresql://tenderpilot:jqrmilIBr6imtT0fKS01@localhost:5432/tenderpilot'
+});
+
+const browser = await chromium.launch({ headless: true });
+const page = await browser.newPage();
+
+console.log('Testing TenderRadar Dashboard...\n');
+
+// Get some real tender URLs from database
+console.log('1. Getting tender URLs from database...');
+const result = await pool.query(`
+  SELECT id, title, notice_url, source 
+  FROM tenders 
+  WHERE notice_url IS NOT NULL AND notice_url != ''
+  ORDER BY created_at DESC 
+  LIMIT 10
+`);
+
+console.log('   Found', result.rows.length, 'tenders with URLs\n');
+
+// Test each URL
+console.log('2. Testing Apply Now URLs...\n');
+
+let working = 0;
+let broken = 0;
+
+for (const tender of result.rows) {
+  const shortTitle = tender.title.substring(0, 60);
+  console.log('   [' + tender.source + '] ' + shortTitle);
+  console.log('   URL:', tender.notice_url);
+  
+  try {
+    const response = await fetch(tender.notice_url, { 
+      method: 'HEAD',
+      redirect: 'follow',
+      signal: AbortSignal.timeout(10000)
+    });
+    
+    const status = response.status;
+    
+    if (status === 404) {
+      console.log('   ** 404 NOT FOUND **');
+      broken++;
+    } else if (status >= 400) {
+      console.log('   HTTP', status);
+      broken++;
+    } else {
+      console.log('   OK - HTTP', status);
+      working++;
+    }
+    
+  } catch (error) {
+    console.log('   ERROR:', error.message);
+    broken++;
+  }
+  
+  console.log('');
+}
+
+console.log('\n3. Summary:');
+console.log('   Working:', working, '/' + result.rows.length);
+console.log('   Broken:', broken, '/' + result.rows.length);
+
+// Now test the dashboard page itself
+console.log('\n4. Loading dashboard page...');
+try {
+  await page.goto('https://tenderradar.co.uk/dashboard.html', { waitUntil: 'networkidle', timeout: 30000 });
+  
+  await page.waitForTimeout(3000);
+  
+  // Check if tenders loaded
+  const tenderCount = await page.evaluate(() => {
+    return document.querySelectorAll('.tender-item').length;
+  });
+  
+  console.log('   Dashboard loaded, found', tenderCount, 'tender items');
+  
+  // Take screenshot
+  await page.screenshot({ path: '/tmp/tenderradar-dashboard.png', fullPage: true });
+  console.log('   Screenshot saved: /tmp/tenderradar-dashboard.png');
+  
+  // Check for JavaScript errors
+  const errors = [];
+  page.on('console', msg => {
+    if (msg.type() === 'error') {
+      errors.push(msg.text());
+    }
+  });
+  
+  if (errors.length > 0) {
+    console.log('\n   JavaScript errors:');
+    errors.forEach(err => console.log('     -', err));
+  }
+  
+} catch (error) {
+  console.log('   ERROR loading dashboard:', error.message);
+}
+
+await browser.close();
+await pool.end();
+
+console.log('\nTest complete.');