2025-06-08 11:21:30 +01:00
|
|
|
<?php
|
|
|
|
|
// Enhanced security headers
|
|
|
|
|
header('Strict-Transport-Security: max-age=31536000; includeSubDomains');
|
|
|
|
|
|
|
|
|
|
// Article-specific SEO variables
|
2026-02-10 10:25:07 +00:00
|
|
|
$article_title = "How to Handle CAPTCHAs in Web Scraping: 7 Methods That Work (2026)";
|
|
|
|
|
$article_description = "Solve reCAPTCHA, hCaptcha & Turnstile ethically. 7 tested methods with code snippets & success rate benchmarks.";
|
2025-06-08 11:21:30 +01:00
|
|
|
$article_keywords = "CAPTCHA handling, web scraping CAPTCHAs, CAPTCHA bypass, automated CAPTCHA solving, web scraping ethics, CAPTCHA services";
|
2026-02-22 09:54:47 +00:00
|
|
|
$article_author = "Michael Thompson";
|
2025-06-08 11:21:30 +01:00
|
|
|
$canonical_url = "https://ukdataservices.co.uk/blog/articles/handling-captchas-scraping";
|
|
|
|
|
$article_published = "2025-05-05T09:00:00+00:00";
|
|
|
|
|
$article_modified = "2025-05-05T09:00:00+00:00";
|
|
|
|
|
$og_image = "https://ukdataservices.co.uk/assets/images/icon-security.svg";
|
|
|
|
|
$read_time = 8;
|
|
|
|
|
?>
|
|
|
|
|
<!DOCTYPE html>
|
|
|
|
|
<html lang="en">
|
|
|
|
|
<head>
|
|
|
|
|
<meta charset="UTF-8">
|
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
|
|
|
<title><?php echo htmlspecialchars($article_title); ?> | UK Data Services Blog</title>
|
|
|
|
|
<meta name="description" content="<?php echo htmlspecialchars($article_description); ?>">
|
|
|
|
|
<meta name="keywords" content="<?php echo htmlspecialchars($article_keywords); ?>">
|
|
|
|
|
<meta name="author" content="<?php echo htmlspecialchars($article_author); ?>">
|
|
|
|
|
<meta name="robots" content="index, follow">
|
|
|
|
|
<link rel="canonical" href="<?php echo htmlspecialchars($canonical_url); ?>">
|
|
|
|
|
|
|
|
|
|
<!-- Article-specific meta tags -->
|
|
|
|
|
<meta name="article:published_time" content="<?php echo $article_published; ?>">
|
|
|
|
|
<meta name="article:modified_time" content="<?php echo $article_modified; ?>">
|
|
|
|
|
<meta name="article:author" content="<?php echo htmlspecialchars($article_author); ?>">
|
|
|
|
|
<meta name="article:section" content="Web Scraping">
|
|
|
|
|
<meta name="article:tag" content="CAPTCHA, Web Scraping, Security, Automation">
|
|
|
|
|
|
|
|
|
|
<!-- Preload critical resources -->
|
|
|
|
|
<link rel="preload" href="../../assets/css/main.css" as="style">
|
|
|
|
|
<link rel="preload" href="../../assets/images/ukds-main-logo.png" as="image">
|
|
|
|
|
|
|
|
|
|
<!-- Open Graph / Social Media -->
|
|
|
|
|
<meta property="og:type" content="article">
|
|
|
|
|
<meta property="og:url" content="<?php echo htmlspecialchars($canonical_url); ?>">
|
|
|
|
|
<meta property="og:title" content="<?php echo htmlspecialchars($article_title); ?>">
|
|
|
|
|
<meta property="og:description" content="<?php echo htmlspecialchars($article_description); ?>">
|
|
|
|
|
<meta property="og:image" content="<?php echo htmlspecialchars($og_image); ?>">
|
|
|
|
|
|
|
|
|
|
<!-- Twitter Card -->
|
|
|
|
|
<meta name="twitter:card" content="summary_large_image">
|
|
|
|
|
<meta name="twitter:title" content="<?php echo htmlspecialchars($article_title); ?>">
|
|
|
|
|
<meta name="twitter:description" content="<?php echo htmlspecialchars($article_description); ?>">
|
|
|
|
|
<meta name="twitter:image" content="<?php echo htmlspecialchars($og_image); ?>">
|
|
|
|
|
|
|
|
|
|
<!-- Favicon and App Icons -->
|
|
|
|
|
<link rel="icon" type="image/svg+xml" href="../../assets/images/favicon.svg">
|
|
|
|
|
<link rel="apple-touch-icon" sizes="180x180" href="../../assets/images/apple-touch-icon.svg">
|
|
|
|
|
|
|
|
|
|
<!-- Fonts -->
|
|
|
|
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
|
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
|
|
|
<link href="https://fonts.googleapis.com/css2?family=Roboto+Slab:wght@300;400;500;600;700&family=Lato:wght@300;400;500;600;700&display=swap" rel="stylesheet">
|
|
|
|
|
|
|
|
|
|
<!-- Styles -->
|
|
|
|
|
<link rel="stylesheet" href="../../assets/css/main.css">
|
2026-02-05 04:11:15 +00:00
|
|
|
<link rel="stylesheet" href="../../assets/css/cro-enhancements.css">
|
2025-06-08 11:21:30 +01:00
|
|
|
|
|
|
|
|
<!-- Article Schema -->
|
|
|
|
|
<script type="application/ld+json">
|
|
|
|
|
{
|
|
|
|
|
"@context": "https://schema.org",
|
|
|
|
|
"@type": "Article",
|
|
|
|
|
"mainEntityOfPage": {
|
|
|
|
|
"@type": "WebPage",
|
|
|
|
|
"@id": "<?php echo htmlspecialchars($canonical_url); ?>"
|
|
|
|
|
},
|
|
|
|
|
"headline": "<?php echo htmlspecialchars($article_title); ?>",
|
|
|
|
|
"description": "<?php echo htmlspecialchars($article_description); ?>",
|
|
|
|
|
"image": "<?php echo htmlspecialchars($og_image); ?>",
|
|
|
|
|
"author": {
|
|
|
|
|
"@type": "Organization",
|
|
|
|
|
"name": "UK Data Services",
|
|
|
|
|
"url": "https://ukdataservices.co.uk"
|
|
|
|
|
},
|
|
|
|
|
"publisher": {
|
|
|
|
|
"@type": "Organization",
|
|
|
|
|
"name": "UK Data Services",
|
|
|
|
|
"logo": {
|
|
|
|
|
"@type": "ImageObject",
|
|
|
|
|
"url": "https://ukdataservices.co.uk/assets/images/ukds-main-logo.png"
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"datePublished": "<?php echo $article_published; ?>",
|
|
|
|
|
"dateModified": "<?php echo $article_modified; ?>"
|
|
|
|
|
}
|
|
|
|
|
</script>
|
|
|
|
|
</head>
|
|
|
|
|
<body>
|
|
|
|
|
<!-- Skip to content link for accessibility -->
|
|
|
|
|
<a href="#main-content" class="skip-to-content">Skip to main content</a>
|
|
|
|
|
|
2026-02-10 22:21:16 +00:00
|
|
|
<?php include($_SERVER["DOCUMENT_ROOT"] . "/includes/nav.php"); ?><!-- Article Content -->
|
2025-06-08 11:21:30 +01:00
|
|
|
<main id="main-content">
|
|
|
|
|
<article class="article-page">
|
|
|
|
|
<div class="container">
|
2025-06-09 05:47:40 +00:00
|
|
|
<div class="article-meta">
|
|
|
|
|
<span class="category"><a href="/blog/categories/web-scraping.php">Web Scraping</a></span>
|
|
|
|
|
<time datetime="2025-05-05">5 May 2025</time>
|
|
|
|
|
<span class="read-time">8 min read</span>
|
|
|
|
|
</div>
|
|
|
|
|
<header class="article-header">
|
2025-06-08 11:21:30 +01:00
|
|
|
<h1><?php echo htmlspecialchars($article_title); ?></h1>
|
|
|
|
|
<p class="article-lead"><?php echo htmlspecialchars($article_description); ?></p>
|
|
|
|
|
|
|
|
|
|
<div class="article-author">
|
|
|
|
|
<div class="author-info">
|
|
|
|
|
<span>By <?php echo htmlspecialchars($article_author); ?></span>
|
|
|
|
|
</div>
|
|
|
|
|
<div class="share-buttons">
|
|
|
|
|
<a href="https://www.linkedin.com/sharing/share-offsite/?url=<?php echo urlencode($canonical_url); ?>" class="share-button linkedin" aria-label="Share on LinkedIn" rel="noopener" target="_blank">
|
2026-02-05 04:11:15 +00:00
|
|
|
<img loading="lazy" src="../../assets/images/icon-linkedin.svg" alt="LinkedIn">
|
2025-06-08 11:21:30 +01:00
|
|
|
</a>
|
|
|
|
|
<a href="https://twitter.com/intent/tweet?url=<?php echo urlencode($canonical_url); ?>&text=<?php echo urlencode($article_title); ?>" class="share-button twitter" aria-label="Share on Twitter" rel="noopener" target="_blank">
|
2026-02-05 04:11:15 +00:00
|
|
|
<img loading="lazy" src="../../assets/images/icon-twitter.svg" alt="Twitter">
|
2025-06-08 11:21:30 +01:00
|
|
|
</a>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</header>
|
|
|
|
|
|
|
|
|
|
<div class="article-content">
|
|
|
|
|
<div class="content-wrapper">
|
|
|
|
|
<h2>Understanding CAPTCHAs and Their Purpose</h2>
|
|
|
|
|
<p>CAPTCHAs (Completely Automated Public Turing Test to Tell Computers and Humans Apart) are security measures designed to prevent automated access to websites. While they serve important security purposes, they can pose challenges for legitimate web scraping operations.</p>
|
|
|
|
|
|
|
|
|
|
<h3>Types of CAPTCHAs</h3>
|
|
|
|
|
<ul>
|
|
|
|
|
<li><strong>Text-based CAPTCHAs:</strong> Distorted text that users must read and type</li>
|
|
|
|
|
<li><strong>Image CAPTCHAs:</strong> Select images matching specific criteria</li>
|
|
|
|
|
<li><strong>Audio CAPTCHAs:</strong> Audio challenges for accessibility</li>
|
|
|
|
|
<li><strong>reCAPTCHA:</strong> Google's advanced CAPTCHA system</li>
|
|
|
|
|
<li><strong>hCaptcha:</strong> Privacy-focused alternative to reCAPTCHA</li>
|
|
|
|
|
<li><strong>Invisible CAPTCHAs:</strong> Background behavior analysis</li>
|
|
|
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
<h2>Ethical Considerations</h2>
|
|
|
|
|
|
|
|
|
|
<h3>Legal and Ethical Framework</h3>
|
|
|
|
|
<p>Before implementing CAPTCHA handling techniques, consider:</p>
|
|
|
|
|
<ul>
|
|
|
|
|
<li><strong>Terms of Service:</strong> Review website terms regarding automated access</li>
|
|
|
|
|
<li><strong>robots.txt:</strong> Respect site crawling guidelines</li>
|
|
|
|
|
<li><strong>Rate Limiting:</strong> Avoid overwhelming servers</li>
|
|
|
|
|
<li><strong>Data Usage:</strong> Ensure compliance with data protection laws</li>
|
|
|
|
|
<li><strong>Business Purpose:</strong> Have legitimate reasons for data collection</li>
|
|
|
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
<h3>Best Practices for Ethical Scraping</h3>
|
|
|
|
|
<ul>
|
|
|
|
|
<li>Contact website owners for API access when possible</li>
|
|
|
|
|
<li>Implement respectful delays between requests</li>
|
|
|
|
|
<li>Use proper user agents and headers</li>
|
|
|
|
|
<li>Avoid scraping personal or sensitive data</li>
|
|
|
|
|
<li>Consider the impact on website performance</li>
|
|
|
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
<h2>Prevention Strategies</h2>
|
|
|
|
|
|
|
|
|
|
<h3>Avoiding CAPTCHAs Through Good Practices</h3>
|
|
|
|
|
<p>The best approach to CAPTCHA handling is prevention:</p>
|
|
|
|
|
|
|
|
|
|
<h4>1. Behavioral Mimicking</h4>
|
|
|
|
|
<pre><code>
|
|
|
|
|
import random
|
|
|
|
|
import time
|
|
|
|
|
from selenium import webdriver
|
|
|
|
|
|
|
|
|
|
def human_like_browsing():
|
|
|
|
|
driver = webdriver.Chrome()
|
|
|
|
|
|
|
|
|
|
# Random delays between actions
|
|
|
|
|
def random_delay():
|
|
|
|
|
time.sleep(random.uniform(1, 3))
|
|
|
|
|
|
|
|
|
|
# Simulate human scrolling
|
|
|
|
|
def scroll_slowly():
|
|
|
|
|
total_height = driver.execute_script("return document.body.scrollHeight")
|
|
|
|
|
for i in range(1, int(total_height/100)):
|
|
|
|
|
driver.execute_script(f"window.scrollTo(0, {i*100});")
|
|
|
|
|
time.sleep(random.uniform(0.1, 0.3))
|
|
|
|
|
|
|
|
|
|
# Mouse movement patterns
|
|
|
|
|
def random_mouse_movement():
|
|
|
|
|
from selenium.webdriver.common.action_chains import ActionChains
|
|
|
|
|
actions = ActionChains(driver)
|
|
|
|
|
|
|
|
|
|
# Random cursor movements
|
|
|
|
|
for _ in range(random.randint(2, 5)):
|
|
|
|
|
x_offset = random.randint(-50, 50)
|
|
|
|
|
y_offset = random.randint(-50, 50)
|
|
|
|
|
actions.move_by_offset(x_offset, y_offset)
|
|
|
|
|
actions.perform()
|
|
|
|
|
time.sleep(random.uniform(0.1, 0.5))
|
|
|
|
|
|
|
|
|
|
# Usage example
|
|
|
|
|
def scrape_with_human_behavior(url):
|
|
|
|
|
driver = webdriver.Chrome()
|
|
|
|
|
driver.get(url)
|
|
|
|
|
|
|
|
|
|
# Simulate reading time
|
|
|
|
|
time.sleep(random.uniform(3, 7))
|
|
|
|
|
|
|
|
|
|
# Random scrolling
|
|
|
|
|
scroll_slowly()
|
|
|
|
|
|
|
|
|
|
# Random mouse movements
|
|
|
|
|
random_mouse_movement()
|
|
|
|
|
|
|
|
|
|
# Extract data after human-like interaction
|
|
|
|
|
data = driver.find_element("tag", "content").text
|
|
|
|
|
|
|
|
|
|
driver.quit()
|
|
|
|
|
return data
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
|
|
|
|
<h4>2. Session Management</h4>
|
|
|
|
|
<pre><code>
|
|
|
|
|
import requests
|
|
|
|
|
from requests.adapters import HTTPAdapter
|
|
|
|
|
from urllib3.util.retry import Retry
|
|
|
|
|
|
|
|
|
|
class SessionManager:
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self.session = requests.Session()
|
|
|
|
|
self.setup_session()
|
|
|
|
|
|
|
|
|
|
def setup_session(self):
|
|
|
|
|
# Retry strategy
|
|
|
|
|
retry_strategy = Retry(
|
|
|
|
|
total=3,
|
|
|
|
|
backoff_factor=1,
|
|
|
|
|
status_forcelist=[429, 500, 502, 503, 504],
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
adapter = HTTPAdapter(max_retries=retry_strategy)
|
|
|
|
|
self.session.mount("http://", adapter)
|
|
|
|
|
self.session.mount("https://", adapter)
|
|
|
|
|
|
|
|
|
|
# Human-like headers
|
|
|
|
|
self.session.headers.update({
|
|
|
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
|
|
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
|
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
|
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
|
|
|
'Connection': 'keep-alive',
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
def get_with_delay(self, url, delay_range=(1, 3)):
|
|
|
|
|
time.sleep(random.uniform(*delay_range))
|
|
|
|
|
return self.session.get(url)
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
|
|
|
|
<h4>3. Proxy Rotation</h4>
|
|
|
|
|
<pre><code>
|
|
|
|
|
import itertools
|
|
|
|
|
import random
|
|
|
|
|
|
|
|
|
|
class ProxyRotator:
|
|
|
|
|
def __init__(self, proxy_list):
|
|
|
|
|
self.proxies = itertools.cycle(proxy_list)
|
|
|
|
|
self.current_proxy = None
|
|
|
|
|
self.failed_proxies = set()
|
|
|
|
|
|
|
|
|
|
def get_proxy(self):
|
|
|
|
|
"""Get next working proxy"""
|
|
|
|
|
for _ in range(len(self.proxy_list)):
|
|
|
|
|
proxy = next(self.proxies)
|
|
|
|
|
if proxy not in self.failed_proxies:
|
|
|
|
|
self.current_proxy = proxy
|
|
|
|
|
return {
|
|
|
|
|
'http': f'http://{proxy}',
|
|
|
|
|
'https': f'https://{proxy}'
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# If all proxies failed, reset and try again
|
|
|
|
|
self.failed_proxies.clear()
|
|
|
|
|
return self.get_proxy()
|
|
|
|
|
|
|
|
|
|
def mark_proxy_failed(self):
|
|
|
|
|
"""Mark current proxy as failed"""
|
|
|
|
|
if self.current_proxy:
|
|
|
|
|
self.failed_proxies.add(self.current_proxy)
|
|
|
|
|
|
|
|
|
|
def test_proxy(self, proxy_dict):
|
|
|
|
|
"""Test if proxy is working"""
|
|
|
|
|
try:
|
|
|
|
|
response = requests.get(
|
|
|
|
|
'http://httpbin.org/ip',
|
|
|
|
|
proxies=proxy_dict,
|
|
|
|
|
timeout=10
|
|
|
|
|
)
|
|
|
|
|
return response.status_code == 200
|
|
|
|
|
except:
|
|
|
|
|
return False
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
|
|
|
|
<h2>CAPTCHA Detection</h2>
|
|
|
|
|
|
|
|
|
|
<h3>Identifying CAPTCHA Presence</h3>
|
|
|
|
|
<pre><code>
|
|
|
|
|
from selenium.webdriver.common.by import By
|
|
|
|
|
from selenium.common.exceptions import NoSuchElementException
|
|
|
|
|
|
|
|
|
|
def detect_captcha(driver):
|
|
|
|
|
"""Detect various types of CAPTCHAs"""
|
|
|
|
|
captcha_indicators = [
|
|
|
|
|
# reCAPTCHA
|
|
|
|
|
(By.CLASS_NAME, "g-recaptcha"),
|
|
|
|
|
(By.ID, "g-recaptcha"),
|
|
|
|
|
(By.XPATH, "//iframe[contains(@src, 'recaptcha')]"),
|
|
|
|
|
|
|
|
|
|
# hCaptcha
|
|
|
|
|
(By.CLASS_NAME, "h-captcha"),
|
|
|
|
|
(By.XPATH, "//iframe[contains(@src, 'hcaptcha')]"),
|
|
|
|
|
|
|
|
|
|
# Generic CAPTCHA indicators
|
|
|
|
|
(By.XPATH, "//*[contains(text(), 'captcha')]"),
|
|
|
|
|
(By.XPATH, "//*[contains(text(), 'CAPTCHA')]"),
|
|
|
|
|
(By.XPATH, "//img[contains(@alt, 'captcha')]"),
|
|
|
|
|
|
|
|
|
|
# Common form names
|
|
|
|
|
(By.NAME, "captcha"),
|
|
|
|
|
(By.ID, "captcha"),
|
|
|
|
|
(By.CLASS_NAME, "captcha"),
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
for locator_type, locator_value in captcha_indicators:
|
|
|
|
|
try:
|
|
|
|
|
element = driver.find_element(locator_type, locator_value)
|
|
|
|
|
if element.is_displayed():
|
|
|
|
|
return True, locator_type, locator_value
|
|
|
|
|
except NoSuchElementException:
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
return False, None, None
|
|
|
|
|
|
|
|
|
|
# Usage
|
|
|
|
|
def check_for_captcha_and_handle(driver):
|
|
|
|
|
has_captcha, locator_type, locator_value = detect_captcha(driver)
|
|
|
|
|
|
|
|
|
|
if has_captcha:
|
|
|
|
|
print(f"CAPTCHA detected: {locator_type} = {locator_value}")
|
|
|
|
|
# Implement handling strategy here
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
|
|
|
|
<h2>Automated CAPTCHA Solving</h2>
|
|
|
|
|
|
|
|
|
|
<h3>Third-Party CAPTCHA Solving Services</h3>
|
|
|
|
|
<p>When legitimate automation requires CAPTCHA solving:</p>
|
|
|
|
|
|
|
|
|
|
<h4>Popular Services</h4>
|
|
|
|
|
<ul>
|
|
|
|
|
<li><strong>2captcha:</strong> Supports most CAPTCHA types</li>
|
|
|
|
|
<li><strong>Anti-Captcha:</strong> High success rates</li>
|
|
|
|
|
<li><strong>DeathByCaptcha:</strong> Established service</li>
|
|
|
|
|
<li><strong>CapMonster:</strong> Software-based solution</li>
|
|
|
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
<h4>Implementation Example</h4>
|
|
|
|
|
<pre><code>
|
|
|
|
|
import base64
|
|
|
|
|
import time
|
|
|
|
|
import requests
|
|
|
|
|
|
|
|
|
|
class CaptchaSolver:
|
|
|
|
|
def __init__(self, api_key, service_url):
|
|
|
|
|
self.api_key = api_key
|
|
|
|
|
self.service_url = service_url
|
|
|
|
|
|
|
|
|
|
def solve_image_captcha(self, image_path):
|
|
|
|
|
"""Solve image-based CAPTCHA"""
|
|
|
|
|
|
|
|
|
|
# Encode image
|
|
|
|
|
with open(image_path, 'rb') as f:
|
|
|
|
|
image_data = base64.b64encode(f.read()).decode()
|
|
|
|
|
|
|
|
|
|
# Submit CAPTCHA
|
|
|
|
|
submit_url = f"{self.service_url}/in.php"
|
|
|
|
|
data = {
|
|
|
|
|
'key': self.api_key,
|
|
|
|
|
'method': 'base64',
|
|
|
|
|
'body': image_data
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
response = requests.post(submit_url, data=data)
|
|
|
|
|
|
|
|
|
|
if response.text.startswith('OK|'):
|
|
|
|
|
captcha_id = response.text.split('|')[1]
|
|
|
|
|
return self.get_captcha_result(captcha_id)
|
|
|
|
|
else:
|
|
|
|
|
raise Exception(f"CAPTCHA submission failed: {response.text}")
|
|
|
|
|
|
|
|
|
|
def get_captcha_result(self, captcha_id):
|
|
|
|
|
"""Poll for CAPTCHA solution"""
|
|
|
|
|
result_url = f"{self.service_url}/res.php"
|
|
|
|
|
|
|
|
|
|
for _ in range(30): # Wait up to 5 minutes
|
|
|
|
|
time.sleep(10)
|
|
|
|
|
|
|
|
|
|
response = requests.get(result_url, params={
|
|
|
|
|
'key': self.api_key,
|
|
|
|
|
'action': 'get',
|
|
|
|
|
'id': captcha_id
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
if response.text == 'CAPCHA_NOT_READY':
|
|
|
|
|
continue
|
|
|
|
|
elif response.text.startswith('OK|'):
|
|
|
|
|
return response.text.split('|')[1]
|
|
|
|
|
else:
|
|
|
|
|
raise Exception(f"CAPTCHA solving failed: {response.text}")
|
|
|
|
|
|
|
|
|
|
raise Exception("CAPTCHA solving timeout")
|
|
|
|
|
|
|
|
|
|
# Usage
|
|
|
|
|
def solve_captcha_if_present(driver):
|
|
|
|
|
has_captcha, _, _ = detect_captcha(driver)
|
|
|
|
|
|
|
|
|
|
if has_captcha:
|
|
|
|
|
# Take screenshot of CAPTCHA
|
|
|
|
|
captcha_element = driver.find_element(By.CLASS_NAME, "captcha-image")
|
|
|
|
|
captcha_element.screenshot("captcha.png")
|
|
|
|
|
|
|
|
|
|
# Solve CAPTCHA
|
|
|
|
|
solver = CaptchaSolver("your_api_key", "https://2captcha.com")
|
|
|
|
|
solution = solver.solve_image_captcha("captcha.png")
|
|
|
|
|
|
|
|
|
|
# Input solution
|
|
|
|
|
captcha_input = driver.find_element(By.NAME, "captcha")
|
|
|
|
|
captcha_input.send_keys(solution)
|
|
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
|
|
|
|
<h2>Advanced Techniques</h2>
|
|
|
|
|
|
|
|
|
|
<h3>reCAPTCHA v2 Handling</h3>
|
|
|
|
|
<pre><code>
|
|
|
|
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
|
|
|
from selenium.webdriver.support import expected_conditions as EC
|
|
|
|
|
|
|
|
|
|
def handle_recaptcha_v2(driver):
|
|
|
|
|
"""Handle reCAPTCHA v2 checkbox"""
|
|
|
|
|
try:
|
|
|
|
|
# Wait for reCAPTCHA iframe to load
|
|
|
|
|
wait = WebDriverWait(driver, 10)
|
|
|
|
|
|
|
|
|
|
# Switch to reCAPTCHA iframe
|
|
|
|
|
recaptcha_iframe = wait.until(
|
|
|
|
|
EC.presence_of_element_located((By.XPATH, "//iframe[contains(@src, 'recaptcha')]"))
|
|
|
|
|
)
|
|
|
|
|
driver.switch_to.frame(recaptcha_iframe)
|
|
|
|
|
|
|
|
|
|
# Click the checkbox
|
|
|
|
|
checkbox = wait.until(
|
|
|
|
|
EC.element_to_be_clickable((By.ID, "recaptcha-anchor"))
|
|
|
|
|
)
|
|
|
|
|
checkbox.click()
|
|
|
|
|
|
|
|
|
|
# Switch back to main content
|
|
|
|
|
driver.switch_to.default_content()
|
|
|
|
|
|
|
|
|
|
# Wait for challenge to complete or appear
|
|
|
|
|
time.sleep(2)
|
|
|
|
|
|
|
|
|
|
# Check if challenge appeared
|
|
|
|
|
try:
|
|
|
|
|
challenge_iframe = driver.find_element(By.XPATH, "//iframe[contains(@src, 'bframe')]")
|
|
|
|
|
if challenge_iframe.is_displayed():
|
|
|
|
|
print("reCAPTCHA challenge appeared - manual intervention needed")
|
|
|
|
|
return False
|
|
|
|
|
except NoSuchElementException:
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
print(f"reCAPTCHA handling failed: {e}")
|
|
|
|
|
return False
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
|
|
|
|
<h3>Invisible reCAPTCHA</h3>
|
|
|
|
|
<p>Invisible reCAPTCHAs analyze user behavior. Key strategies:</p>
|
|
|
|
|
<ul>
|
|
|
|
|
<li><strong>Mouse Movement:</strong> Simulate natural cursor patterns</li>
|
|
|
|
|
<li><strong>Keyboard Timing:</strong> Vary typing speeds and patterns</li>
|
|
|
|
|
<li><strong>Scroll Behavior:</strong> Implement human-like scrolling</li>
|
|
|
|
|
<li><strong>Page Interaction:</strong> Click on non-essential elements</li>
|
|
|
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
<h2>Monitoring and Debugging</h2>
|
|
|
|
|
|
|
|
|
|
<h3>CAPTCHA Detection Logging</h3>
|
|
|
|
|
<pre><code>
|
|
|
|
|
import logging
|
|
|
|
|
from datetime import datetime
|
|
|
|
|
|
|
|
|
|
class CaptchaLogger:
|
|
|
|
|
def __init__(self):
|
|
|
|
|
logging.basicConfig(
|
|
|
|
|
level=logging.INFO,
|
|
|
|
|
format='%(asctime)s - %(levelname)s - %(message)s',
|
|
|
|
|
handlers=[
|
|
|
|
|
logging.FileHandler('captcha_log.txt'),
|
|
|
|
|
logging.StreamHandler()
|
|
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
self.logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
def log_captcha_encounter(self, url, captcha_type):
|
|
|
|
|
self.logger.info(f"CAPTCHA encountered: {captcha_type} at {url}")
|
|
|
|
|
|
|
|
|
|
def log_captcha_solved(self, url, solve_time):
|
|
|
|
|
self.logger.info(f"CAPTCHA solved in {solve_time}s at {url}")
|
|
|
|
|
|
|
|
|
|
def log_captcha_failed(self, url, error):
|
|
|
|
|
self.logger.error(f"CAPTCHA solving failed at {url}: {error}")
|
|
|
|
|
|
|
|
|
|
# Usage in scraping script
|
|
|
|
|
logger = CaptchaLogger()
|
|
|
|
|
|
|
|
|
|
def scrape_with_captcha_logging(url):
|
|
|
|
|
driver = webdriver.Chrome()
|
|
|
|
|
driver.get(url)
|
|
|
|
|
|
|
|
|
|
if check_for_captcha_and_handle(driver):
|
|
|
|
|
logger.log_captcha_encounter(url, "reCAPTCHA")
|
|
|
|
|
|
|
|
|
|
start_time = time.time()
|
|
|
|
|
success = solve_captcha_if_present(driver)
|
|
|
|
|
solve_time = time.time() - start_time
|
|
|
|
|
|
|
|
|
|
if success:
|
|
|
|
|
logger.log_captcha_solved(url, solve_time)
|
|
|
|
|
else:
|
|
|
|
|
logger.log_captcha_failed(url, "Solution timeout")
|
|
|
|
|
</code></pre>
|
|
|
|
|
|
|
|
|
|
<h2>Legal and Compliance Considerations</h2>
|
|
|
|
|
|
|
|
|
|
<h3>UK Legal Framework</h3>
|
|
|
|
|
<ul>
|
|
|
|
|
<li><strong>Computer Misuse Act 1990:</strong> Avoid unauthorized access</li>
|
|
|
|
|
<li><strong>GDPR:</strong> Handle personal data appropriately</li>
|
|
|
|
|
<li><strong>Copyright Laws:</strong> Respect intellectual property</li>
|
|
|
|
|
<li><strong>Contract Law:</strong> Adhere to terms of service</li>
|
|
|
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
<h3>Best Practice Checklist</h3>
|
|
|
|
|
<ul>
|
|
|
|
|
<li>✅ Review website terms of service</li>
|
|
|
|
|
<li>✅ Check robots.txt compliance</li>
|
|
|
|
|
<li>✅ Implement rate limiting</li>
|
|
|
|
|
<li>✅ Use proper attribution</li>
|
|
|
|
|
<li>✅ Respect CAPTCHA purposes</li>
|
|
|
|
|
<li>✅ Consider alternative data sources</li>
|
|
|
|
|
<li>✅ Document legitimate business purposes</li>
|
|
|
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
<h2>Alternative Approaches</h2>
|
|
|
|
|
|
|
|
|
|
<h3>API-First Strategy</h3>
|
|
|
|
|
<p>Before implementing CAPTCHA handling:</p>
|
|
|
|
|
<ul>
|
|
|
|
|
<li>Contact website owners for API access</li>
|
|
|
|
|
<li>Check for existing public APIs</li>
|
|
|
|
|
<li>Explore data partnerships</li>
|
|
|
|
|
<li>Consider paid data services</li>
|
|
|
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
<h3>Headless Browser Alternatives</h3>
|
|
|
|
|
<ul>
|
|
|
|
|
<li><strong>HTTP Libraries:</strong> Faster for simple data extraction</li>
|
|
|
|
|
<li><strong>API Reverse Engineering:</strong> Direct endpoint access</li>
|
|
|
|
|
<li><strong>RSS/XML Feeds:</strong> Structured data sources</li>
|
|
|
|
|
<li><strong>Open Data Initiatives:</strong> Government and public datasets</li>
|
|
|
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
<div class="article-cta">
|
|
|
|
|
<h3>Professional CAPTCHA Handling Solutions</h3>
|
|
|
|
|
<p>UK Data Services provides compliant web scraping solutions that handle CAPTCHAs professionally while respecting website terms and legal requirements.</p>
|
2025-06-08 20:51:14 +00:00
|
|
|
<a href="/quote" class="btn btn-primary">Get Expert Consultation</a>
|
2025-06-08 11:21:30 +01:00
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<!-- Related Articles -->
|
|
|
|
|
<aside class="related-articles">
|
|
|
|
|
<h3>Related Articles</h3>
|
|
|
|
|
<div class="related-grid">
|
|
|
|
|
<article class="related-card">
|
|
|
|
|
<span class="category">Web Scraping</span>
|
|
|
|
|
<h4><a href="web-scraping-compliance-uk-guide.php">Complete Guide to Web Scraping Compliance in the UK</a></h4>
|
2026-02-22 09:58:16 +00:00
|
|
|
<span class="read-time">12 min read</span> <article class="related-card">
|
2025-06-08 11:21:30 +01:00
|
|
|
<span class="category">Technology</span>
|
2026-02-10 10:25:07 +00:00
|
|
|
<h4><a href="selenium-vs-playwright-comparison.php">Selenium vs Playwright 2026: Speed Tests & Honest Comparison</a></h4>
|
2026-02-22 09:58:16 +00:00
|
|
|
<span class="read-time">9 min read</span> <article class="related-card">
|
2025-06-08 11:21:30 +01:00
|
|
|
<span class="category">Web Scraping</span>
|
|
|
|
|
<h4><a href="python-scrapy-enterprise-guide.php">Python Scrapy Enterprise Guide: Scaling Web Scraping Operations</a></h4>
|
2026-02-22 09:58:16 +00:00
|
|
|
<span class="read-time">12 min read</span> </div>
|
2025-06-08 11:21:30 +01:00
|
|
|
</aside>
|
|
|
|
|
</div>
|
2025-12-07 11:49:39 +00:00
|
|
|
<?php include($_SERVER['DOCUMENT_ROOT'] . '/includes/author-bio.php'); ?>
|
|
|
|
|
|
|
|
|
|
<?php include($_SERVER['DOCUMENT_ROOT'] . '/includes/article-footer.php'); ?>
|
|
|
|
|
</div>
|
|
|
|
|
</article>
|
2025-06-08 11:21:30 +01:00
|
|
|
</main>
|
|
|
|
|
|
|
|
|
|
<!-- Footer -->
|
|
|
|
|
<footer class="footer">
|
|
|
|
|
<div class="container">
|
|
|
|
|
<div class="footer-content">
|
|
|
|
|
<div class="footer-section">
|
|
|
|
|
<div class="footer-logo">
|
2026-02-05 04:11:15 +00:00
|
|
|
<img loading="lazy" src="../../assets/images/logo-white.svg" alt="UK Data Services" loading="lazy">
|
2025-06-08 11:21:30 +01:00
|
|
|
</div>
|
|
|
|
|
<p>Enterprise data intelligence solutions for modern British business.</p>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div class="footer-section">
|
|
|
|
|
<h3>Quick Links</h3>
|
|
|
|
|
<ul>
|
2025-06-08 15:51:38 +00:00
|
|
|
<li><a href="/#services">Services</a></li>
|
|
|
|
|
<li><a href="/blog/">Blog</a></li>
|
|
|
|
|
<li><a href="/case-studies/">Case Studies</a></li>
|
2025-06-08 20:51:14 +00:00
|
|
|
<li><a href="/about">About</a></li>
|
2025-06-08 15:51:38 +00:00
|
|
|
<li><a href="/#contact">Contact</a></li>
|
2025-06-08 11:21:30 +01:00
|
|
|
</ul>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div class="footer-section">
|
|
|
|
|
<h3>Legal</h3>
|
|
|
|
|
<ul>
|
2025-06-08 20:51:14 +00:00
|
|
|
<li><a href="/privacy-policy">Privacy Policy</a></li>
|
|
|
|
|
<li><a href="/terms-of-service">Terms of Service</a></li>
|
|
|
|
|
<li><a href="/cookie-policy">Cookie Policy</a></li>
|
|
|
|
|
<li><a href="/gdpr-compliance">GDPR Compliance</a></li>
|
2025-06-08 11:21:30 +01:00
|
|
|
</ul>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div class="footer-bottom">
|
|
|
|
|
<p>© <?php echo date('Y'); ?> UK Data Services. All rights reserved.</p>
|
|
|
|
|
<div class="social-links">
|
2025-12-21 08:08:45 +00:00
|
|
|
<a href="https://linkedin.com/company/uk-data-services" aria-label="LinkedIn" rel="noopener" target="_blank">
|
2026-02-05 04:11:15 +00:00
|
|
|
<img loading="lazy" src="../../assets/images/icon-linkedin.svg" alt="LinkedIn" loading="lazy">
|
2025-06-08 11:21:30 +01:00
|
|
|
</a>
|
|
|
|
|
<a href="https://twitter.com/ukdataservices" aria-label="Twitter" rel="noopener" target="_blank">
|
2026-02-05 04:11:15 +00:00
|
|
|
<img loading="lazy" src="../../assets/images/icon-twitter.svg" alt="Twitter" loading="lazy">
|
2025-06-08 11:21:30 +01:00
|
|
|
</a>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</div>
|
|
|
|
|
</footer>
|
|
|
|
|
|
|
|
|
|
<!-- Scripts -->
|
|
|
|
|
<script src="../../assets/js/main.js"></script>
|
2026-02-05 04:11:15 +00:00
|
|
|
<script src="../../assets/js/cro-enhancements.js"></script>
|
2025-06-08 11:21:30 +01:00
|
|
|
</body>
|
|
|
|
|
</html>
|
