Files
tenderpilot/test-apply-buttons.mjs
Peter Foster c6b0169f3e feat: three major improvements - stable sources, archival, email alerts
1. Focus on Stable International/Regional Sources
   - Improved TED EU scraper (5 search strategies, 5 pages each)
   - All stable sources now hourly (TED EU, Sell2Wales, PCS Scotland, eTendersNI)
   - De-prioritize unreliable UK gov sites (100% removal rate)

2. Archival Feature
   - New DB columns: archived, archived_at, archived_snapshot, last_validated, validation_failures
   - Cleanup script now preserves full tender snapshots before archiving
   - Gradual failure handling (3 retries before archiving)
   - No data loss - historical record preserved

3. Email Alerts
   - Daily digest (8am) - all new tenders from last 24h
   - High-value alerts (every 4h) - tenders >£100k
   - Professional HTML emails with all tender details
   - Configurable via environment variables

Expected outcomes:
- 50-100 stable tenders (vs 26 currently)
- Zero 404 errors (archived data preserved)
- Proactive notifications (no missed opportunities)
- Historical archive for trend analysis

Files:
- scrapers/ted-eu.js (improved)
- cleanup-with-archival.mjs (new)
- send-tender-alerts.mjs (new)
- migrations/add-archival-fields.sql (new)
- THREE_IMPROVEMENTS_SUMMARY.md (documentation)

All cron jobs updated for hourly scraping + daily cleanup + alerts
2026-02-15 14:42:17 +00:00

115 lines
3.5 KiB
JavaScript

import { chromium } from 'playwright';
const browser = await chromium.launch({ headless: true });
const page = await browser.newPage();
console.log('Testing TenderRadar site...\n');
// Test homepage
console.log('1. Loading homepage: https://tenderradar.co.uk');
await page.goto('https://tenderradar.co.uk', { waitUntil: 'networkidle', timeout: 30000 });
await page.waitForTimeout(2000);
const title = await page.title();
console.log(' Page title:', title);
// Check for any JavaScript errors
const errors = [];
page.on('console', msg => {
if (msg.type() === 'error') {
errors.push(msg.text());
}
});
// Try to get some tender data
console.log('\n2. Checking for tender listings...');
await page.waitForSelector('.tender-item, .tender-list, #tenders-list', { timeout: 10000 }).catch(() => {
console.log(' No tender items found on page');
});
// Get all Apply Now buttons
const applyButtons = await page.evaluate(() => {
const buttons = Array.from(document.querySelectorAll('a[href*="notice"], a.btn-primary, button'));
return buttons
.filter(btn => btn.textContent.toLowerCase().includes('apply'))
.map(btn => ({
text: btn.textContent.trim().substring(0, 50),
href: btn.href || btn.onclick?.toString() || 'no-href',
tag: btn.tagName
}))
.slice(0, 10);
});
console.log(' Found', applyButtons.length, '"Apply" buttons/links');
applyButtons.forEach((btn, idx) => {
console.log(' ' + (idx + 1) + '. [' + btn.tag + '] "' + btn.text + '" -> ' + btn.href.substring(0, 80));
});
// Take a screenshot
await page.screenshot({ path: '/tmp/tenderradar-home.png', fullPage: true });
console.log('\n3. Screenshot saved: /tmp/tenderradar-home.png');
// Check if we can get tender data from the page
const tenderData = await page.evaluate(() => {
const tenders = [];
const items = document.querySelectorAll('.tender-item, [data-tender-id], tr[onclick]');
items.forEach((item, idx) => {
if (idx > 5) return; // Limit to first 5
const applyBtn = item.querySelector('a[href*="notice"], a.btn-primary, button.btn-primary');
if (applyBtn) {
tenders.push({
id: item.dataset?.tenderId || idx,
title: item.querySelector('h3, .tender-title, .title')?.textContent?.trim()?.substring(0, 60) || 'Unknown',
applyUrl: applyBtn.href || 'No URL',
buttonText: applyBtn.textContent.trim()
});
}
});
return tenders;
});
if (tenderData.length > 0) {
console.log('\n4. Testing individual Apply Now URLs...\n');
for (const tender of tenderData) {
console.log(' Testing:', tender.title);
console.log(' URL:', tender.applyUrl);
if (tender.applyUrl === 'No URL' || !tender.applyUrl.startsWith('http')) {
console.log(' INVALID URL\n');
continue;
}
try {
const response = await page.goto(tender.applyUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });
const status = response.status();
if (status === 404) {
console.log(' ** 404 NOT FOUND **\n');
} else if (status >= 400) {
console.log(' HTTP', status, '\n');
} else {
console.log(' OK - HTTP', status, '\n');
}
await page.waitForTimeout(1000);
} catch (error) {
console.log(' ERROR:', error.message, '\n');
}
}
} else {
console.log('\n4. No tender data found on page');
}
if (errors.length > 0) {
console.log('\nJavaScript Errors Found:');
errors.forEach(err => console.log(' -', err));
}
await browser.close();
console.log('\nTest complete.');