Initial commit: UK Data Services website

robots.txt

@@ -0,0 +1,24 @@
+User-agent: *
+Allow: /
+
+# Prioritize important pages
+Allow: /index.php
+Allow: /quote.php
+Allow: /assets/css/
+Allow: /assets/js/
+Allow: /assets/images/
+
+# Block sensitive areas
+Disallow: /logs/
+Disallow: /vendor/
+Disallow: /config/
+Disallow: /*.log$
+Disallow: /*.inc$
+Disallow: /contact-handler.php
+Disallow: /quote-handler.php
+
+# Sitemap location
+Sitemap: https://ukdataservices.co.uk/sitemap.xml
+
+# Crawl delay to be respectful
+Crawl-delay: 1