User-Agent: GPTBot
User-Agent: ChatGPT-User
User-Agent: OAI-SearchBot
User-Agent: ClaudeBot
User-Agent: Claude-User
User-Agent: Claude-SearchBot
User-Agent: anthropic-ai
User-Agent: PerplexityBot
User-Agent: Perplexity-User
User-Agent: Google-Extended
User-Agent: Applebot-Extended
User-Agent: DeepSeekBot
User-Agent: MistralAI-User
User-Agent: DuckAssistBot
User-Agent: meta-externalagent
User-Agent: Amazonbot
User-Agent: cohere-ai
User-Agent: YouBot
User-Agent: ora-agent
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /docs/llms.txt
Allow: /developers/llms.txt
Allow: /api/llms.txt
Allow: /pricing.md
Allow: /index.md
Allow: /agents.md
Allow: /auth.md
Allow: /about
Allow: /info-for-ai
Allow: /developers
Allow: /mcp
Allow: /skills/
Allow: /schemamap.json
Allow: /openapi.json
Allow: /.well-known/
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /onboarding/
Disallow: /reset-password
Disallow: /auth/
Disallow: /studio

User-Agent: CCBot
User-Agent: Bytespider
User-Agent: ByteSpider
User-Agent: Diffbot
User-Agent: Omgili
User-Agent: omgilibot
User-Agent: Webzio-Extended
User-Agent: ImagesiftBot
User-Agent: Timpibot
User-Agent: Scrapy
User-Agent: PetalBot
Disallow: /

User-Agent: *
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /onboarding/
Disallow: /reset-password
Disallow: /auth/
Disallow: /studio

Host: https://shiken.ai
Sitemap: https://shiken.ai/sitemap.xml
Schemamap: https://shiken.ai/schemamap.json
LLM-Index: https://shiken.ai/llms.txt
Agent-Skills: https://shiken.ai/.well-known/agent-skills/index.json