105 lines
3.5 KiB
JavaScript
105 lines
3.5 KiB
JavaScript
|
import axios from 'axios';
|
||
|
|
import pg from 'pg';
|
||
|
|
import dotenv from 'dotenv';
|
||
|
|
|
||
|
|
dotenv.config();
|
||
|
|
|
||
|
|
const pool = new pg.Pool({
|
||
|
|
connectionString: process.env.DATABASE_URL || 'postgresql://tenderpilot:tenderpilot123@localhost:5432/tenderpilot'
|
||
|
|
});
|
||
|
|
|
||
|
|
async function scrapeTenders() {
|
||
|
|
try {
|
||
|
|
console.log(`[${new Date().toISOString()}] Starting tender scrape...`);
|
||
|
|
|
||
|
|
// Get date from 30 days ago
|
||
|
|
const fromDate = new Date();
|
||
|
|
fromDate.setDate(fromDate.getDate() - 30);
|
||
|
|
const dateStr = fromDate.toISOString().split('T')[0];
|
||
|
|
|
||
|
|
const url = `https://www.contractsfinder.service.gov.uk/Published/Notices/OCDS/Search?stage=tender&output=json&publishedFrom=${dateStr}`;
|
||
|
|
|
||
|
|
console.log(`Fetching from: ${url}`);
|
||
|
|
const response = await axios.get(url, { timeout: 30000 });
|
||
|
|
|
||
|
|
const data = response.data;
|
||
|
|
const releases = data.releases || [];
|
||
|
|
|
||
|
|
console.log(`Found ${releases.length} tenders`);
|
||
|
|
|
||
|
|
let insertedCount = 0;
|
||
|
|
|
||
|
|
for (const release of releases) {
|
||
|
|
try {
|
||
|
|
const tender = release.tender || {};
|
||
|
|
const planning = release.planning || {};
|
||
|
|
const parties = release.parties || [];
|
||
|
|
|
||
|
|
// Find procuring entity
|
||
|
|
const procurer = parties.find(p => p.roles && p.roles.includes('procurer'));
|
||
|
|
|
||
|
|
const sourceId = release.ocid || release.id;
|
||
|
|
const title = tender.title || 'Untitled';
|
||
|
|
const description = tender.description || '';
|
||
|
|
const publishedDate = release.date;
|
||
|
|
const deadline = tender.tenderPeriod?.endDate;
|
||
|
|
const authority = procurer?.name || 'Unknown';
|
||
|
|
const location = planning?.budget?.description || tender.procurementMethod || '';
|
||
|
|
const noticeUrl = release.url || '';
|
||
|
|
const documentsUrl = tender.documents?.length > 0 ? tender.documents[0].url : '';
|
||
|
|
|
||
|
|
// Extract value
|
||
|
|
let valueLow = null, valueHigh = null;
|
||
|
|
if (planning?.budget?.amount?.amount) {
|
||
|
|
valueLow = planning.budget.amount.amount;
|
||
|
|
valueHigh = planning.budget.amount.amount;
|
||
|
|
} else if (tender.value?.amount) {
|
||
|
|
valueLow = tender.value.amount;
|
||
|
|
valueHigh = tender.value.amount;
|
||
|
|
}
|
||
|
|
|
||
|
|
const cpvCodes = tender.classification ? [tender.classification.scheme] : [];
|
||
|
|
|
||
|
|
await pool.query(
|
||
|
|
`INSERT INTO tenders (
|
||
|
|
source, source_id, title, description, summary, cpv_codes,
|
||
|
|
value_low, value_high, currency, published_date, deadline,
|
||
|
|
authority_name, authority_type, location, documents_url, notice_url, status
|
||
|
|
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
|
||
|
|
ON CONFLICT (source_id) DO NOTHING`,
|
||
|
|
[
|
||
|
|
'contracts_finder',
|
||
|
|
sourceId,
|
||
|
|
title.substring(0, 500),
|
||
|
|
description,
|
||
|
|
description.substring(0, 500),
|
||
|
|
cpvCodes,
|
||
|
|
valueLow,
|
||
|
|
valueHigh,
|
||
|
|
'GBP',
|
||
|
|
publishedDate,
|
||
|
|
deadline,
|
||
|
|
authority,
|
||
|
|
'government',
|
||
|
|
location.substring(0, 255),
|
||
|
|
documentsUrl,
|
||
|
|
noticeUrl,
|
||
|
|
'open'
|
||
|
|
]
|
||
|
|
);
|
||
|
|
insertedCount++;
|
||
|
|
} catch (e) {
|
||
|
|
console.error('Error inserting tender:', e.message);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
console.log(`[${new Date().toISOString()}] Scrape complete. Inserted/updated ${insertedCount} tenders`);
|
||
|
|
} catch (error) {
|
||
|
|
console.error('Error scraping tenders:', error.message);
|
||
|
|
} finally {
|
||
|
|
await pool.end();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
scrapeTenders();
|