# OzLevel — robots.txt
# ozlevel.com.au
# Updated: May 2026

# ── Major search engines ──────────────────────────────────────────────
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Slurp
User-agent: Applebot
User-agent: YandexBot
Allow: /

# ── AI answer & retrieval crawlers ────────────────────────────────────
# Allow: these power live query answers and citations in AI search tools
User-agent: GPTBot
User-agent: AhrefsBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: meta-externalagent
User-agent: YouBot
Allow: /

# ── Bad bots — SEO scrapers and content harvesters ────────────────────
User-agent: SemrushBot
User-agent: DotBot
User-agent: MJ12bot
User-agent: BLEXBot
User-agent: PetalBot
User-agent: CCBot
User-agent: Diffbot
User-agent: omgilibot
Disallow: /

# ── Default: allow all other crawlers ─────────────────────────────────
User-agent: *
Allow: /

Sitemap: https://ozlevel.com.au/sitemap.xml