# SecBot robots policy. AI-readable site guide: https://secbot.trade/llms.txt (full: https://secbot.trade/llms-full.txt)
# Search engines and AI search/assistant crawlers are welcome on public pages.
# Data APIs and protocol endpoints are not for crawling.

User-agent: Googlebot
User-agent: Google-Extended
User-agent: Bingbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: DuckDuckBot
User-agent: DuckAssistBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: PetalBot
User-agent: Yeti
User-agent: SeznamBot
User-agent: MojeekBot
User-agent: Qwantbot
User-agent: Kagibot
User-agent: Pinterestbot
User-agent: Google-CloudVertexBot
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: OAI-AdsBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: meta-externalfetcher
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
User-agent: YouBot
User-agent: MistralAI-User
User-agent: MistralAI-Index
User-agent: ShapBot
User-agent: Reflectionbot
User-agent: LinkupBot
Allow: /
Disallow: /api/
Allow: /mcp/server-card
Disallow: /mcp
Disallow: /oauth/
Disallow: /internal/
Disallow: /ws
Disallow: /_pt
Disallow: /*__data.json
Disallow: /dashboard/admin/
Disallow: /search/?
Disallow: /search?
Disallow: /r/
Disallow: /login
Disallow: /register
Disallow: /logout
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /force-reset
Disallow: /verify-email
Disallow: /unsubscribe
Disallow: /account
Disallow: /affiliate/login
Disallow: /affiliate/signup
Disallow: /affiliate/logout
Disallow: /affiliate/forgot-password
Disallow: /affiliate/reset-password
Disallow: /affiliate/resend-verification
Disallow: /affiliate/verify-email
Disallow: /affiliate/dashboard
Disallow: /export/all-reports

User-agent: CCBot
Crawl-delay: 2
Allow: /
Disallow: /api/
Allow: /mcp/server-card
Disallow: /mcp
Disallow: /oauth/
Disallow: /internal/
Disallow: /ws
Disallow: /_pt
Disallow: /*__data.json
Disallow: /dashboard/admin/
Disallow: /search/?
Disallow: /search?
Disallow: /r/
Disallow: /login
Disallow: /register
Disallow: /logout
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /force-reset
Disallow: /verify-email
Disallow: /unsubscribe
Disallow: /account
Disallow: /affiliate/login
Disallow: /affiliate/signup
Disallow: /affiliate/logout
Disallow: /affiliate/forgot-password
Disallow: /affiliate/reset-password
Disallow: /affiliate/resend-verification
Disallow: /affiliate/verify-email
Disallow: /affiliate/dashboard
Disallow: /export/all-reports

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
Crawl-delay: 10
Allow: /
Disallow: /api/
Allow: /mcp/server-card
Disallow: /mcp
Disallow: /oauth/
Disallow: /internal/
Disallow: /ws
Disallow: /_pt
Disallow: /*__data.json
Disallow: /dashboard/admin/
Disallow: /search/?
Disallow: /search?
Disallow: /r/
Disallow: /login
Disallow: /register
Disallow: /logout
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /force-reset
Disallow: /verify-email
Disallow: /unsubscribe
Disallow: /account
Disallow: /affiliate/login
Disallow: /affiliate/signup
Disallow: /affiliate/logout
Disallow: /affiliate/forgot-password
Disallow: /affiliate/reset-password
Disallow: /affiliate/resend-verification
Disallow: /affiliate/verify-email
Disallow: /affiliate/dashboard
Disallow: /export/all-reports

User-agent: *
Allow: /
Disallow: /api/
Allow: /mcp/server-card
Disallow: /mcp
Disallow: /oauth/
Disallow: /internal/
Disallow: /ws
Disallow: /_pt
Disallow: /*__data.json
Disallow: /dashboard/admin/
Disallow: /search/?
Disallow: /search?
Disallow: /r/
Disallow: /login
Disallow: /register
Disallow: /logout
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /force-reset
Disallow: /verify-email
Disallow: /unsubscribe
Disallow: /account
Disallow: /affiliate/login
Disallow: /affiliate/signup
Disallow: /affiliate/logout
Disallow: /affiliate/forgot-password
Disallow: /affiliate/reset-password
Disallow: /affiliate/resend-verification
Disallow: /affiliate/verify-email
Disallow: /affiliate/dashboard
Disallow: /export/all-reports

Sitemap: https://secbot.trade/sitemap.xml
Sitemap: https://blog.secbot.trade/sitemap.xml
Sitemap: https://docs.secbot.trade/sitemap.xml