File Path
/mnt/user-data/outputs/klozer-llms-final/robots.txt
Copy
# =========================================================================
# robots.txt for klozer.io
# AI-Powered Cloud Contact Centre Software
# Last Updated: April 2026
# =========================================================================

# -------------------------------------------------------------------------
# DEFAULT RULES — All Crawlers
# -------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-json/
Disallow: /feed/
Disallow: /comments/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /tag/*/feed/
Disallow: /category/*/feed/
Allow: /wp-content/uploads/

# -------------------------------------------------------------------------
# GOOGLE — Standard Search Crawlers
# -------------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

# -------------------------------------------------------------------------
# BING — Standard Search Crawler
# -------------------------------------------------------------------------
User-agent: Bingbot
Allow: /

# -------------------------------------------------------------------------
# AI CRAWLERS — Explicitly Allowed for AI Visibility & Citation
# -------------------------------------------------------------------------

# OpenAI — ChatGPT, SearchGPT
User-agent: GPTBot
Allow: /
Allow: /llms.txt

User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /
Allow: /llms.txt

User-agent: anthropic-ai
Allow: /
Allow: /llms.txt

# Google — AI Training & Gemini
User-agent: Google-Extended
Allow: /
Allow: /llms.txt

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt

# Common Crawl — Used for AI model training datasets
User-agent: CCBot
Allow: /
Allow: /llms.txt

# Meta AI
User-agent: FacebookBot
Allow: /
Allow: /llms.txt

User-agent: meta-externalagent
Allow: /
Allow: /llms.txt

# Apple — Siri, Apple Intelligence
User-agent: Applebot
Allow: /
Allow: /llms.txt

User-agent: Applebot-Extended
Allow: /
Allow: /llms.txt

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /llms.txt

# Amazon / Alexa
User-agent: Amazonbot
Allow: /
Allow: /llms.txt

# Microsoft Copilot
User-agent: CopilotBot
Allow: /
Allow: /llms.txt

# ByteDance / TikTok AI
User-agent: Bytespider
Allow: /
Allow: /llms.txt

# -------------------------------------------------------------------------
# SCRAPERS & UNWANTED BOTS — Blocked
# -------------------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

# -------------------------------------------------------------------------
# SITEMAPS & AI INDEX
# -------------------------------------------------------------------------
Sitemap: https://klozer.io/sitemap.xml

# LLMs.txt — Structured AI content index for Large Language Models
# See: https://klozer.io/llms.txt