# robots.txt for https://sankon.in/ (Sankon Technologies India Pvt. Ltd.)
# Last updated: 2026-11-03
# Purpose: allow all search engines, AI answer engines (AEO) and generative
# engine crawlers (GEO) to access public product, solution and contact content,
# while keeping private/transactional/duplicate URLs out of the index.

# ---------------------------------------------------------------
# Default rule: all crawlers
# ---------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /search?
Disallow: /*?s=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?utm_
Disallow: /*?sessionid=
Disallow: /tmp/
Disallow: /cgi-bin/
Allow: /wp-admin/admin-ajax.php

# ---------------------------------------------------------------
# Traditional search engines
# ---------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

# ---------------------------------------------------------------
# AI search / answer-engine crawlers (AEO) - retrieval and citation
# ---------------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: MistralAI-User
Allow: /

# ---------------------------------------------------------------
# AI training / generative engine crawlers (GEO) - brand visibility in LLMs
# ---------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Diffbot
Allow: /

User-agent: CCBot
Allow: /

# ---------------------------------------------------------------
# Aggressive SEO scrapers - rate-limited / blocked to save server resources
# ---------------------------------------------------------------
User-agent: AhrefsBot
Crawl-delay: 10
Allow: /

User-agent: SemrushBot
Crawl-delay: 10
Allow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# ---------------------------------------------------------------
# Machine-readable resources for crawlers and LLMs
# ---------------------------------------------------------------
# LLM guidance files:
#   https://sankon.in/llms.txt
#   https://sankon.in/llms-full.txt

Host: sankon.in
Sitemap: https://sankon.in/sitemap.xml