import axios from 'axios';
import * as cheerio from 'cheerio';

const url = 'https://ted.europa.eu/en/search/result?query=GB&pageNum=0';
const response = await axios.get(url, {
  timeout: 30000,
  headers: {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
  }
});

const $ = cheerio.load(response.data);

console.log('=== Searching for data-testid attributes ===');
let count = 0;
$('[data-testid]').each((i, el) => {
  if (count < 10) {
    console.log(`${count}: data-testid="${$(el).attr('data-testid')}" tag=${el.name}`);
    count++;
  }
});

console.log('\n=== Looking for href containing Notice ===');
count = 0;
$('a[href*="Notice"]').each((i, el) => {
  if (count < 5) {
    const text = $(el).text().substring(0, 80);
    const href = $(el).attr('href').substring(0, 100);
    console.log(`${count}: ${href} - text: ${text}`);
    count++;
  }
});

console.log('\n=== Looking for main content div ===');
$('main').each((i, el) => {
  if (i < 3) {
    const html = $(el).html().substring(0, 1000);
    console.log(`main[${i}]: ${html}`);
  }
});

console.log('\n=== All a tags (first 20 with long href) ===');
count = 0;
$('a').each((i, el) => {
  if (count < 20) {
    const href = $(el).attr('href') || '';
    if (href.includes('notice') || href.includes('Notice') || href.includes('OP-') || href.length > 50) {
      console.log(`${count}: ${href.substring(0, 150)}`);
      count++;
    }
  }
});

console.log('\n=== Looking for articles/items ===');
count = 0;
$('article, [role="article"], .item, .result, .row').each((i, el) => {
  if (count < 10) {
    const text = $(el).text().substring(0, 100);
    console.log(`${count}: ${text}`);
    count++;
  }
});