# www.leavers.com.au Robots.txt Configuration
# Generated: September 15, 2026
# Purpose: AI Bot and Search Engine Crawler Control

# ======================
# General Web Crawlers
# ======================
User-agent: *
Allow: /
Disallow: /adminlogin/
Disallow: /User/payment/
Disallow: /User/login/
Disallow: /login/
Disallow: /pay/
Disallow: /check-out/
Disallow: /Booking/
Disallow: /bookeasy/
Disallow: /quote/
Disallow: /registrations/
Disallow: /obj/
Disallow: /bin/
Disallow: /App_GlobalResources/
Disallow: /Configs/
Crawl-delay: 1

# ======================
# AI Training Bots
# ======================

# OpenAI GPTBot
User-agent: GPTBot
Allow: /destinations/
Allow: /BuildingList.aspx
Allow: /BuildingDetails.aspx
Allow: /faq/
Allow: /Content.aspx
Allow: /Destinations.aspx
Disallow: /User/
Disallow: /login/
Disallow: /pay/
Disallow: /adminlogin/
Crawl-delay: 2

# Google Extended (Bard/Gemini Training)
User-agent: Google-Extended
Allow: /

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /

# Anthropic AI (Claude)
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# ======================
# Social Media Crawlers
# ======================
# Allows link previews on TikTok, Instagram, and Facebook

# ByteDance (TikTok)
User-agent: Bytespider
Allow: /
Disallow: /User/
Disallow: /login/
Disallow: /pay/
Disallow: /adminlogin/

# Facebook/Meta (Instagram & Facebook)
User-agent: FacebookBot
Allow: /
Disallow: /User/
Disallow: /login/
Disallow: /pay/
Disallow: /adminlogin/

# ======================
# Sitemaps
# ======================
Sitemap: https://www.leavers.com.au/sitemap.xml
Sitemap: https://www.leavers.com.au/sitemap-destinations.xml
Sitemap: https://www.leavers.com.au/sitemap-buildings.xml
Sitemap: https://www.leavers.com.au/sitemap-content.xml

# ======================
# Crawl Rate Settings
# ======================
# Host: www.leavers.com.au
# Crawl-delay: 1 second (reduce server load)