feat: visual polish, nav login link, pricing badge fix, cursor fix, button contrast
- Hero mockup: enhanced 3D perspective and shadow - Testimonials: illustrated SVG avatars - Growth pricing card: visual prominence (scale, gradient, badge) - Most Popular badge: repositioned to avoid overlapping heading - Nav: added Log In link next to Start Free Trial - Fixed btn-primary text colour on anchor tags (white on blue) - Fixed cursor: default on all non-interactive elements - Disabled user-select on non-form content to prevent text caret
This commit is contained in:
155
scrapers/sell2wales.js
Normal file
155
scrapers/sell2wales.js
Normal file
@@ -0,0 +1,155 @@
|
||||
import axios from 'axios';
|
||||
import * as cheerio from 'cheerio';
|
||||
import pg from 'pg';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const pool = new pg.Pool({
|
||||
connectionString: process.env.DATABASE_URL || 'postgresql://tenderpilot:tenderpilot123@localhost:5432/tenderpilot'
|
||||
});
|
||||
|
||||
function parseDate(dateStr) {
|
||||
if (!dateStr || dateStr.trim() === '') return null;
|
||||
|
||||
try {
|
||||
// Handle format like "13/02/2026"
|
||||
if (dateStr.match(/^\d{2}\/\d{2}\/\d{4}$/)) {
|
||||
const [day, month, year] = dateStr.split('/');
|
||||
const date = new Date(`${year}-${month}-${day}`);
|
||||
if (isNaN(date.getTime())) return null;
|
||||
return date.toISOString();
|
||||
}
|
||||
|
||||
// Try general parsing
|
||||
const date = new Date(dateStr);
|
||||
if (isNaN(date.getTime())) return null;
|
||||
return date.toISOString();
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrapeTenders() {
|
||||
try {
|
||||
console.log(`[${new Date().toISOString()}] Starting Sell2Wales scrape...`);
|
||||
|
||||
let insertedCount = 0;
|
||||
|
||||
const url = 'https://www.sell2wales.gov.wales/search/Search_MainPage.aspx';
|
||||
|
||||
const response = await axios.get(url, {
|
||||
timeout: 30000,
|
||||
headers: {
|
||||
'User-Agent': 'TenderRadar/1.0 (UK Public Procurement Aggregator; contact@tenderradar.co.uk)'
|
||||
}
|
||||
});
|
||||
|
||||
const $ = cheerio.load(response.data);
|
||||
|
||||
// Find all links to tender detail pages
|
||||
const tenderLinks = $('a[href*="search_view.aspx?ID="]');
|
||||
|
||||
console.log(`Found ${tenderLinks.length} potential tenders`);
|
||||
|
||||
// Group by parent containers to avoid duplicates
|
||||
const processed = new Set();
|
||||
|
||||
for (let i = 0; i < tenderLinks.length; i++) {
|
||||
try {
|
||||
const link = tenderLinks.eq(i);
|
||||
const href = link.attr('href');
|
||||
|
||||
if (!href || processed.has(href)) continue;
|
||||
processed.add(href);
|
||||
|
||||
const title = link.text().trim();
|
||||
if (!title || title.length === 0) continue;
|
||||
|
||||
const noticeUrl = href.startsWith('http') ? href : 'https://www.sell2wales.gov.wales' + href;
|
||||
|
||||
// Get the parent container for this tender
|
||||
const container = link.closest('div, li, tr');
|
||||
const containerText = container.text();
|
||||
|
||||
// Extract reference number from URL
|
||||
const idMatch = href.match(/ID=([A-Z0-9]+)/);
|
||||
const sourceId = idMatch ? idMatch[1] : ('s2w_' + Date.now() + '_' + i);
|
||||
|
||||
// Extract metadata
|
||||
const refMatch = containerText.match(/Reference no:\s*([A-Z0-9]+)/i);
|
||||
const finalRef = refMatch ? refMatch[1] : sourceId;
|
||||
|
||||
const authorityMatch = containerText.match(/Published by:\s*([^\n]+)/i);
|
||||
const authority = authorityMatch ? authorityMatch[1].trim() : 'Unknown';
|
||||
|
||||
const pubDateMatch = containerText.match(/Publication date:\s*(\d{2}\/\d{2}\/\d{4})/i);
|
||||
const publishedDate = pubDateMatch ? parseDate(pubDateMatch[1]) : null;
|
||||
|
||||
const deadlineMatch = containerText.match(/Deadline date:\s*(\d{2}\/\d{2}\/\d{4})/i);
|
||||
const deadline = deadlineMatch ? parseDate(deadlineMatch[1]) : null;
|
||||
|
||||
const noticeTypeMatch = containerText.match(/Notice Type:\s*([^\n]+)/i);
|
||||
const noticeType = noticeTypeMatch ? noticeTypeMatch[1].trim() : '';
|
||||
|
||||
const locationMatch = containerText.match(/Location:\s*([^\n#]+)/i);
|
||||
const location = locationMatch ? locationMatch[1].trim() : 'Wales';
|
||||
|
||||
const valueMatch = containerText.match(/Value:\s*(\d+)/i);
|
||||
let valueLow = null;
|
||||
if (valueMatch) {
|
||||
valueLow = parseInt(valueMatch[1]);
|
||||
}
|
||||
|
||||
// Look for description in nearby paragraphs or divs
|
||||
let description = '';
|
||||
const nearbyP = container.find('p').first();
|
||||
if (nearbyP.length > 0) {
|
||||
description = nearbyP.text().trim();
|
||||
}
|
||||
if (!description || description.length < 10) {
|
||||
description = noticeType || title;
|
||||
}
|
||||
|
||||
await pool.query(
|
||||
`INSERT INTO tenders (
|
||||
source, source_id, title, description, summary, cpv_codes,
|
||||
value_low, value_high, currency, published_date, deadline,
|
||||
authority_name, authority_type, location, documents_url, notice_url, status
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
|
||||
ON CONFLICT (source_id) DO NOTHING`,
|
||||
[
|
||||
'sell2wales',
|
||||
finalRef,
|
||||
title.substring(0, 500),
|
||||
description.substring(0, 1000),
|
||||
description.substring(0, 500),
|
||||
[],
|
||||
valueLow,
|
||||
valueLow,
|
||||
'GBP',
|
||||
publishedDate,
|
||||
deadline,
|
||||
authority.substring(0, 255),
|
||||
'government',
|
||||
location.substring(0, 255),
|
||||
'',
|
||||
noticeUrl,
|
||||
deadline && new Date(deadline) > new Date() ? 'open' : 'closed'
|
||||
]
|
||||
);
|
||||
insertedCount++;
|
||||
} catch (e) {
|
||||
console.error('Error inserting tender:', e.message);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[${new Date().toISOString()}] Sell2Wales scrape complete. Inserted/updated ${insertedCount} tenders`);
|
||||
} catch (error) {
|
||||
console.error('Error scraping Sell2Wales:', error.message);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
scrapeTenders();
|
||||
Reference in New Issue
Block a user