# robots.txt for https://ranksenseai.com/
# Last updated: 2026-06-03
# Strategy: maximize visibility in Google + AI search, protect WP internals,
# block only abusive scrapers. (This is an AI-SEO brand — being seen is the goal.)

# ============================================================
# SEARCH ENGINES — allow full access (this is how you rank)
# ============================================================
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# ============================================================
# AI SEARCH / ANSWER ENGINES — allow (so you get CITED in
# ChatGPT, Claude, Perplexity, and Google AI Overviews)
# ============================================================
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

# ============================================================
# AI TRAINING CRAWLERS — allowed ON PURPOSE for a marketing
# site (you WANT to be in model knowledge).
# To opt OUT of training later, change "Allow: /" to "Disallow: /"
# ============================================================
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

# ============================================================
# ABUSIVE / LOW-VALUE SCRAPERS — block (waste bandwidth,
# give nothing back in visibility)
# ============================================================
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Diffbot
Disallow: /

# ============================================================
# DEFAULT RULES — apply to every other bot.
# Protect WordPress internals; keep all public content open.
# ============================================================
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /?s=
Disallow: /search/
Allow: /

# ============================================================
# SITEMAP — verify this path resolves (Yoast uses sitemap_index.xml,
# Rank Math uses sitemap_index.xml too; some setups use sitemap.xml)
# ============================================================
Sitemap: https://ranksenseai.com/sitemap_index.xml