Add sector classification module, integrate into all 7 scrapers, fix CF pagination

This commit is contained in:
Peter Foster
2026-02-14 17:12:51 +00:00
parent d1aa21c59f
commit 771fcf9d76
23 changed files with 2044 additions and 83 deletions

36
etendersni-simple.mjs Normal file
View File

@@ -0,0 +1,36 @@
import axios from 'axios';
const client = axios.create({
timeout: 5000,
maxRedirects: 5,
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
});
(async () => {
try {
// Try the home page with a search query parameter
const url = 'https://etendersni.gov.uk/epps/home.do?status=open';
console.log('Fetching:', url);
const resp = await client.get(url);
console.log('Status:', resp.status);
console.log('URL:', resp.config.url);
// Check if it has entry IDs
if (resp.data.includes('entryId')) {
console.log('✓ Found entryId in response');
const matches = resp.data.match(/entryId=(\d+)/g);
if (matches) {
console.log(`Found ${matches.length} tenders:`);
matches.slice(0, 5).forEach(m => console.log(' ', m));
}
} else {
console.log('No entryId found');
console.log('Sample:', resp.data.substring(1000, 2000));
}
} catch (e) {
console.log('Error:', e.message);
}
})();