Add sector classification module, integrate into all 7 scrapers, fix CF pagination
This commit is contained in:
36
etendersni-simple.mjs
Normal file
36
etendersni-simple.mjs
Normal file
@@ -0,0 +1,36 @@
|
||||
import axios from 'axios';
|
||||
|
||||
const client = axios.create({
|
||||
timeout: 5000,
|
||||
maxRedirects: 5,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||||
}
|
||||
});
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
// Try the home page with a search query parameter
|
||||
const url = 'https://etendersni.gov.uk/epps/home.do?status=open';
|
||||
console.log('Fetching:', url);
|
||||
const resp = await client.get(url);
|
||||
console.log('Status:', resp.status);
|
||||
console.log('URL:', resp.config.url);
|
||||
|
||||
// Check if it has entry IDs
|
||||
if (resp.data.includes('entryId')) {
|
||||
console.log('✓ Found entryId in response');
|
||||
const matches = resp.data.match(/entryId=(\d+)/g);
|
||||
if (matches) {
|
||||
console.log(`Found ${matches.length} tenders:`);
|
||||
matches.slice(0, 5).forEach(m => console.log(' ', m));
|
||||
}
|
||||
} else {
|
||||
console.log('No entryId found');
|
||||
console.log('Sample:', resp.data.substring(1000, 2000));
|
||||
}
|
||||
|
||||
} catch (e) {
|
||||
console.log('Error:', e.message);
|
||||
}
|
||||
})();
|
||||
Reference in New Issue
Block a user