peter/tenderpilot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add sector classification module, integrate into all 7 scrapers, fix CF pagination

Browse Source
This commit is contained in:
Peter Foster
2026-02-14 17:12:51 +00:00
parent d1aa21c59f
commit 771fcf9d76
23 changed files with 2044 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
scrapers/find-tender.js
View File

@@ -1,5 +1,6 @@
import axios from 'axios';
import * as cheerio from 'cheerio';
import { classifySector } from './classify-sector.js';
import pg from 'pg';
import dotenv from 'dotenv';
@@ -32,7 +33,7 @@ async function scrapeTenders() {
});
const $ = cheerio.load(response.data);
const tenderElements = $('.search-result');
const tenderElements = $('div.search-result');
if (tenderElements.length === 0) {
console.log('No more tenders found, stopping pagination');
@@ -82,8 +83,8 @@ async function scrapeTenders() {
`INSERT INTO tenders (
source, source_id, title, description, summary, cpv_codes,
value_low, value_high, currency, published_date, deadline,
authority_name, authority_type, location, documents_url, notice_url, status
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
authority_name, authority_type, location, documents_url, notice_url, status, sector
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)
ON CONFLICT (source_id) DO NOTHING`,
[
'find_tender',
@@ -102,7 +103,8 @@ async function scrapeTenders() {
'UK',
'',
noticeUrl,
deadline && new Date(deadline) > new Date() ? 'open' : 'closed'
deadline && new Date(deadline) > new Date() ? 'open' : 'closed',
classifySector(title, description, authority)
]
);
insertedCount++;