# robots.txt for http://ar.cleaningsteamer.com
# Last updated: 2026-07-21
# Language: ar (Arabic) — الموقع العربي لأجهزة التنظيف بالبخار

User-agent: *
Allow: /

# Crawl-delay: polite crawling requested for all bots
Crawl-delay: 5

# --------------------------------------------------
# AI / Generative Engine crawlers -- explicitly allowed
# --------------------------------------------------

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google
User-agent: Google-Extended
Allow: /

User-agent: Gemini
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

# Apple
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Amazon / Alexa
User-agent: Amazonbot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Microsoft / Bing
User-agent: bingbot
Allow: /

User-agent: BingPreview
Allow: /

# Yandex
User-agent: YandexBot
Allow: /

User-agent: YandexRenderResourcesBot
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Brave
User-agent: Bravebot
Allow: /

# --------------------------------------------------
# Arabic-region & Chinese AI crawlers & search engines
# --------------------------------------------------

# Baidu (ERNIE Bot / Baidu Search)
User-agent: Baiduspider
Allow: /

User-agent: Baiduspider-render
Allow: /

# ByteDance (Doubao / Toutiao Search)
User-agent: Bytespider
Allow: /

# Sogou (Sogou Search)
User-agent: Sogou Spider
Allow: /

User-agent: Sogou web spider
Allow: /

# 360 Search
User-agent: 360Spider
Allow: /

# Shenma (UC/Alibaba)
User-agent: iaskspider
Allow: /

User-agent: msnbot
Allow: /

# Alibaba / Qwen (Tongyi Qianwen)
User-agent: AlibabaBot
Allow: /

# Tencent (Hunyuan)
User-agent: TencentBot
Allow: /

# DeepSeek
User-agent: DeepSeekBot
Allow: /

# Petal Search (Huawei)
User-agent: PetalBot
Allow: /

# --------------------------------------------------
# Disallow admin and private areas
# --------------------------------------------------

Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /private/
Disallow: /tmp/
Disallow: /schema-demo/

# --------------------------------------------------
# Sitemaps (all formats for maximum discovery)
# --------------------------------------------------

Sitemap: http://ar.cleaningsteamer.com/sitemap-index.xml
Sitemap: http://ar.cleaningsteamer.com/sitemap.xml
Sitemap: http://ar.cleaningsteamer.com/sitemap_baidu.xml
Sitemap: http://ar.cleaningsteamer.com/sitemap-images.xml
Sitemap: http://ar.cleaningsteamer.com/sitemap-news.xml
Sitemap: http://ar.cleaningsteamer.com/sitemap.txt
Sitemap: http://ar.cleaningsteamer.com/sitemap.html

# --------------------------------------------------
# AI & Machine-Readable Reference Files
# --------------------------------------------------

# AI crawler declaration
# http://ar.cleaningsteamer.com/.well-known/ai.txt

# LLM-optimized content files
# http://ar.cleaningsteamer.com/llms.txt
# http://ar.cleaningsteamer.com/llms-full.txt

# GEO & compliance data
# http://ar.cleaningsteamer.com/data/geo/geo-data.json
# http://ar.cleaningsteamer.com/data/geo/compliance.json
# http://ar.cleaningsteamer.com/data/geo/regions.txt

# Security contact
# http://ar.cleaningsteamer.com/.well-known/security.txt

# RSS Feed
# http://ar.cleaningsteamer.com/rss.xml

# Human-readable sitemap
# http://ar.cleaningsteamer.com/sitemap.html

# Plain-text URL lists (for Baidu & other engines)
# http://ar.cleaningsteamer.com/urllist.txt
# http://ar.cleaningsteamer.com/sitemap.txt
# http://ar.cleaningsteamer.com/urls.txt