c6b0169f3e
1. Focus on Stable International/Regional Sources - Improved TED EU scraper (5 search strategies, 5 pages each) - All stable sources now hourly (TED EU, Sell2Wales, PCS Scotland, eTendersNI) - De-prioritize unreliable UK gov sites (100% removal rate) 2. Archival Feature - New DB columns: archived, archived_at, archived_snapshot, last_validated, validation_failures - Cleanup script now preserves full tender snapshots before archiving - Gradual failure handling (3 retries before archiving) - No data loss - historical record preserved 3. Email Alerts - Daily digest (8am) - all new tenders from last 24h - High-value alerts (every 4h) - tenders >£100k - Professional HTML emails with all tender details - Configurable via environment variables Expected outcomes: - 50-100 stable tenders (vs 26 currently) - Zero 404 errors (archived data preserved) - Proactive notifications (no missed opportunities) - Historical archive for trend analysis Files: - scrapers/ted-eu.js (improved) - cleanup-with-archival.mjs (new) - send-tender-alerts.mjs (new) - migrations/add-archival-fields.sql (new) - THREE_IMPROVEMENTS_SUMMARY.md (documentation) All cron jobs updated for hourly scraping + daily cleanup + alerts
107 lines
2.7 KiB
JavaScript
107 lines
2.7 KiB
JavaScript
import { chromium } from 'playwright';
|
|
import pg from 'pg';
|
|
|
|
const pool = new pg.Pool({
|
|
connectionString: 'postgresql://tenderpilot:jqrmilIBr6imtT0fKS01@localhost:5432/tenderpilot'
|
|
});
|
|
|
|
const browser = await chromium.launch({ headless: true });
|
|
const page = await browser.newPage();
|
|
|
|
console.log('Testing TenderRadar Dashboard...\n');
|
|
|
|
// Get some real tender URLs from database
|
|
console.log('1. Getting tender URLs from database...');
|
|
const result = await pool.query(`
|
|
SELECT id, title, notice_url, source
|
|
FROM tenders
|
|
WHERE notice_url IS NOT NULL AND notice_url != ''
|
|
ORDER BY created_at DESC
|
|
LIMIT 10
|
|
`);
|
|
|
|
console.log(' Found', result.rows.length, 'tenders with URLs\n');
|
|
|
|
// Test each URL
|
|
console.log('2. Testing Apply Now URLs...\n');
|
|
|
|
let working = 0;
|
|
let broken = 0;
|
|
|
|
for (const tender of result.rows) {
|
|
const shortTitle = tender.title.substring(0, 60);
|
|
console.log(' [' + tender.source + '] ' + shortTitle);
|
|
console.log(' URL:', tender.notice_url);
|
|
|
|
try {
|
|
const response = await fetch(tender.notice_url, {
|
|
method: 'HEAD',
|
|
redirect: 'follow',
|
|
signal: AbortSignal.timeout(10000)
|
|
});
|
|
|
|
const status = response.status;
|
|
|
|
if (status === 404) {
|
|
console.log(' ** 404 NOT FOUND **');
|
|
broken++;
|
|
} else if (status >= 400) {
|
|
console.log(' HTTP', status);
|
|
broken++;
|
|
} else {
|
|
console.log(' OK - HTTP', status);
|
|
working++;
|
|
}
|
|
|
|
} catch (error) {
|
|
console.log(' ERROR:', error.message);
|
|
broken++;
|
|
}
|
|
|
|
console.log('');
|
|
}
|
|
|
|
console.log('\n3. Summary:');
|
|
console.log(' Working:', working, '/' + result.rows.length);
|
|
console.log(' Broken:', broken, '/' + result.rows.length);
|
|
|
|
// Now test the dashboard page itself
|
|
console.log('\n4. Loading dashboard page...');
|
|
try {
|
|
await page.goto('https://tenderradar.co.uk/dashboard.html', { waitUntil: 'networkidle', timeout: 30000 });
|
|
|
|
await page.waitForTimeout(3000);
|
|
|
|
// Check if tenders loaded
|
|
const tenderCount = await page.evaluate(() => {
|
|
return document.querySelectorAll('.tender-item').length;
|
|
});
|
|
|
|
console.log(' Dashboard loaded, found', tenderCount, 'tender items');
|
|
|
|
// Take screenshot
|
|
await page.screenshot({ path: '/tmp/tenderradar-dashboard.png', fullPage: true });
|
|
console.log(' Screenshot saved: /tmp/tenderradar-dashboard.png');
|
|
|
|
// Check for JavaScript errors
|
|
const errors = [];
|
|
page.on('console', msg => {
|
|
if (msg.type() === 'error') {
|
|
errors.push(msg.text());
|
|
}
|
|
});
|
|
|
|
if (errors.length > 0) {
|
|
console.log('\n JavaScript errors:');
|
|
errors.forEach(err => console.log(' -', err));
|
|
}
|
|
|
|
} catch (error) {
|
|
console.log(' ERROR loading dashboard:', error.message);
|
|
}
|
|
|
|
await browser.close();
|
|
await pool.end();
|
|
|
|
console.log('\nTest complete.');
|