feat: three major improvements - stable sources, archival, email alerts
1. Focus on Stable International/Regional Sources - Improved TED EU scraper (5 search strategies, 5 pages each) - All stable sources now hourly (TED EU, Sell2Wales, PCS Scotland, eTendersNI) - De-prioritize unreliable UK gov sites (100% removal rate) 2. Archival Feature - New DB columns: archived, archived_at, archived_snapshot, last_validated, validation_failures - Cleanup script now preserves full tender snapshots before archiving - Gradual failure handling (3 retries before archiving) - No data loss - historical record preserved 3. Email Alerts - Daily digest (8am) - all new tenders from last 24h - High-value alerts (every 4h) - tenders >£100k - Professional HTML emails with all tender details - Configurable via environment variables Expected outcomes: - 50-100 stable tenders (vs 26 currently) - Zero 404 errors (archived data preserved) - Proactive notifications (no missed opportunities) - Historical archive for trend analysis Files: - scrapers/ted-eu.js (improved) - cleanup-with-archival.mjs (new) - send-tender-alerts.mjs (new) - migrations/add-archival-fields.sql (new) - THREE_IMPROVEMENTS_SUMMARY.md (documentation) All cron jobs updated for hourly scraping + daily cleanup + alerts
This commit is contained in:
Peter Foster
180
send-tender-alerts.mjs
Executable file
180
send-tender-alerts.mjs
Executable file
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* TENDER EMAIL ALERTS
|
||||
*
|
||||
* Sends email notifications for:
|
||||
* 1. High-value tenders (>£100k)
|
||||
* 2. New tenders matching keywords
|
||||
* 3. Daily digest of all new tenders
|
||||
*/
|
||||
|
||||
import pg from 'pg';
|
||||
import nodemailer from 'nodemailer';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const pool = new pg.Pool({
|
||||
connectionString: process.env.DATABASE_URL || 'postgresql://tenderpilot:tenderpilot123@localhost:5432/tenderpilot'
|
||||
});
|
||||
|
||||
// Email configuration
|
||||
const transporter = nodemailer.createTransporter({
|
||||
host: process.env.SMTP_HOST || 'smtp.dynu.com',
|
||||
port: parseInt(process.env.SMTP_PORT || '587'),
|
||||
secure: false, // STARTTLS
|
||||
auth: {
|
||||
user: process.env.SMTP_USER || 'peter.foster@ukdataservices.co.uk',
|
||||
pass: process.env.SMTP_PASS
|
||||
}
|
||||
});
|
||||
|
||||
const ALERT_EMAIL = process.env.ALERT_EMAIL || 'peter.foster@ukdataservices.co.uk';
|
||||
|
||||
async function sendAlerts(mode = 'digest') {
|
||||
try {
|
||||
console.log(`[${new Date().toISOString()}] Starting tender alerts (mode: ${mode})...`);
|
||||
|
||||
let tenders;
|
||||
let subject;
|
||||
let intro;
|
||||
|
||||
if (mode === 'high-value') {
|
||||
// High-value tenders (>£100k or equivalent)
|
||||
tenders = await pool.query(
|
||||
`SELECT * FROM tenders
|
||||
WHERE status = 'open'
|
||||
AND (value_low > 100000 OR value_high > 100000)
|
||||
AND created_at > NOW() - INTERVAL '24 hours'
|
||||
ORDER BY value_high DESC NULLS LAST, created_at DESC
|
||||
LIMIT 50`
|
||||
);
|
||||
|
||||
subject = `🔔 TenderRadar: ${tenders.rows.length} High-Value Tenders (>£100k)`;
|
||||
intro = 'High-value tender opportunities identified in the last 24 hours:';
|
||||
|
||||
} else if (mode === 'digest') {
|
||||
// Daily digest - all new tenders
|
||||
tenders = await pool.query(
|
||||
`SELECT * FROM tenders
|
||||
WHERE status = 'open'
|
||||
AND created_at > NOW() - INTERVAL '24 hours'
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 100`
|
||||
);
|
||||
|
||||
subject = `📊 TenderRadar Daily Digest: ${tenders.rows.length} New Tenders`;
|
||||
intro = 'New tender opportunities from the last 24 hours:';
|
||||
|
||||
} else {
|
||||
throw new Error(`Unknown mode: ${mode}`);
|
||||
}
|
||||
|
||||
if (tenders.rows.length === 0) {
|
||||
console.log(`No tenders found for mode: ${mode}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Build HTML email
|
||||
let html = `
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
body { font-family: Arial, sans-serif; line-height: 1.6; color: #333; }
|
||||
.header { background: #2c3e50; color: white; padding: 20px; }
|
||||
.tender { border: 1px solid #ddd; margin: 20px 0; padding: 15px; border-radius: 5px; }
|
||||
.tender:hover { background: #f9f9f9; }
|
||||
.title { font-size: 18px; font-weight: bold; margin-bottom: 10px; }
|
||||
.meta { color: #666; font-size: 14px; margin-bottom: 10px; }
|
||||
.value { color: #27ae60; font-weight: bold; }
|
||||
.deadline { color: #e74c3c; font-weight: bold; }
|
||||
.btn { background: #3498db; color: white; padding: 10px 20px; text-decoration: none; border-radius: 3px; display: inline-block; margin-top: 10px; }
|
||||
.footer { background: #ecf0f1; padding: 20px; margin-top: 30px; text-align: center; font-size: 12px; color: #666; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1>🎯 TenderRadar Alerts</h1>
|
||||
<p>${intro}</p>
|
||||
</div>
|
||||
`;
|
||||
|
||||
for (const tender of tenders.rows) {
|
||||
const value = tender.value_high || tender.value_low;
|
||||
const valueStr = value ?
|
||||
`${tender.currency || '£'}${value.toLocaleString()}` :
|
||||
'Value not specified';
|
||||
|
||||
const deadline = tender.deadline ?
|
||||
new Date(tender.deadline).toLocaleDateString('en-GB', {
|
||||
day: 'numeric',
|
||||
month: 'short',
|
||||
year: 'numeric'
|
||||
}) :
|
||||
'No deadline';
|
||||
|
||||
const daysLeft = tender.deadline ?
|
||||
Math.ceil((new Date(tender.deadline) - new Date()) / (1000 * 60 * 60 * 24)) :
|
||||
null;
|
||||
|
||||
const daysLeftStr = daysLeft ?
|
||||
` (${daysLeft} day${daysLeft !== 1 ? 's' : ''} left)` :
|
||||
'';
|
||||
|
||||
html += `
|
||||
<div class="tender">
|
||||
<div class="title">${tender.title}</div>
|
||||
<div class="meta">
|
||||
<strong>Authority:</strong> ${tender.authority_name || 'Unknown'}<br>
|
||||
<strong>Location:</strong> ${tender.location || 'Not specified'}<br>
|
||||
<strong>Sector:</strong> ${tender.sector || 'General'}<br>
|
||||
<strong>Source:</strong> ${tender.source}<br>
|
||||
<span class="value">Value: ${valueStr}</span> |
|
||||
<span class="deadline">Deadline: ${deadline}${daysLeftStr}</span>
|
||||
</div>
|
||||
${tender.description ? `<p>${tender.description.substring(0, 300)}${tender.description.length > 300 ? '...' : ''}</p>` : ''}
|
||||
<a href="${tender.notice_url}" class="btn">View Tender →</a>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
html += `
|
||||
<div class="footer">
|
||||
<p>
|
||||
This is an automated alert from TenderRadar<br>
|
||||
<a href="https://tenderradar.co.uk">Visit Dashboard</a> |
|
||||
<a href="mailto:peter.foster@ukdataservices.co.uk">Contact Support</a>
|
||||
</p>
|
||||
<p>
|
||||
<small>Monitoring UK public procurement opportunities since 2026</small>
|
||||
</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
|
||||
// Send email
|
||||
const mailOptions = {
|
||||
from: '"TenderRadar" <peter.foster@ukdataservices.co.uk>',
|
||||
to: ALERT_EMAIL,
|
||||
subject: subject,
|
||||
html: html
|
||||
};
|
||||
|
||||
const info = await transporter.sendMail(mailOptions);
|
||||
console.log(`✅ Email sent: ${info.messageId}`);
|
||||
console.log(` Tenders: ${tenders.rows.length}`);
|
||||
console.log(` To: ${ALERT_EMAIL}`);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error sending alerts:', error);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
// Get mode from command line argument
|
||||
const mode = process.argv[2] || 'digest';
|
||||
sendAlerts(mode);
|
||||
Reference in New Issue
Block a user