# UK Data Services - robots.txt
# https://ukdataservices.co.uk

User-agent: *
Allow: /

# Block sensitive directories and files
Disallow: /includes/
Disallow: /assets/
Disallow: /admin/
Disallow: /logs/
Disallow: /vendor/
Disallow: /config/
Disallow: /database/
Disallow: /docker/
Disallow: /redis/

# Block configuration and handler files
Disallow: /*-handler.php
Disallow: /*.log$
Disallow: /*.inc$
Disallow: /*.sql$
Disallow: /*.sh$
Disallow: /*.bak$
Disallow: /db-config.php
Disallow: /.email-config.php
Disallow: /.recaptcha-config.php

# Block query string URLs to prevent duplicate content
Disallow: /*?*

# Allow important static assets for rendering
Allow: /assets/css/*.css
Allow: /assets/js/*.js
Allow: /assets/images/*.webp
Allow: /assets/images/*.png
Allow: /assets/images/*.jpg
Allow: /assets/images/*.svg

# Sitemaps
Sitemap: https://ukdataservices.co.uk/sitemap.xml
Sitemap: https://ukdataservices.co.uk/sitemap-index.xml
Sitemap: https://ukdataservices.co.uk/sitemap-blog.xml
Sitemap: https://ukdataservices.co.uk/sitemap-services.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Specific instructions for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 2