# https://failegal.site/robots.txt
# FAI Legal Research Assistant — Pakistan statute & precedent search
# Machine-readable product summary for LLMs: https://failegal.site/llms.txt

# One group, many named agents. Search crawlers and AI answer engines get the same
# rules deliberately: this product WANTS to be quotable by assistants, since a lawyer
# asking an assistant for Pakistani case-law tooling is exactly the audience.
# They share a group rather than getting one each because a named agent obeys ONLY
# its own group — separate groups would silently exempt each bot from the API
# Disallow list below.
User-agent: *
User-agent: Googlebot
User-agent: Google-Extended
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: cohere-ai
User-agent: YouBot
User-agent: CCBot
User-agent: MistralAI-User
User-agent: FacebookBot
User-agent: Bytespider
# xAI publishes no official crawler token. "xAI-Bot" is the name the ecosystem
# has settled on, so it is listed here as a good-faith allow — but Grok's
# retrieval fetches often arrive as ordinary browser traffic with no bot
# signal at all, so no robots.txt line can be relied on to govern them.
# What actually gets this site into Grok is llms.txt + clean public HTML.
User-agent: xAI-Bot
User-agent: Grok
Allow: /

# API surface — no crawl value, and several of these are rate-limited or metered.
Disallow: /api/
Disallow: /ask
Disallow: /upload
Disallow: /makepdf
Disallow: /config
Disallow: /health
Disallow: /machine
Disallow: /delete-downloads
Disallow: /admin/

# Transactional auth pages — never useful as a search result.
Disallow: /reset
Disallow: /confirm
Disallow: /confirmed

# NOTE: /app is deliberately left crawlable. It carries <meta name="robots"
# content="noindex,follow">, and a blocked page cannot be read, so Google would
# never see that tag and could still index the bare URL. Crawlable + noindex is
# what actually keeps it out of the index.

Sitemap: https://failegal.site/sitemap.xml
Sitemap: https://failegal.site/media/sitemap.xml