# OpenAI
User-agent: GPTBot
Allow: /

# OpenAI (ChatGPT Plugins & new crawlers)
User-agent: ChatGPT-User
Allow: /
User-agent: ChatGPT-Editor
Allow: /

# Google AI training and Gemini
User-agent: Google-Extended
Allow: /
User-agent: GoogleOther
Allow: /
User-agent: GoogleOther-Image
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /
User-agent: PerplexityBot/1.0
Allow: /

# Anthropic Claude
User-agent: anthropic-ai
Allow: /

# Meta AI (Facebook / Instagram AI training)
User-agent: FacebookBot
Allow: /
User-agent: FacebookExternalHit
Allow: /
User-agent: Meta-ExternalAgent
Allow: /

# Common Crawl (massive AI dataset source)
User-agent: CCBot
Allow: /

# Amazon (AWS AI / Alexa)
User-agent: Amazonbot
Allow: /

# AppleBot (Siri / Apple AI)
User-agent: Applebot
Allow: /

# Bing / Microsoft (Copilot indexing)
User-agent: bingbot
Allow: /
User-agent: BingPreview
Allow: /
User-agent: AdsBot-bing
Allow: /
User-agent: msnbot
Allow: /

# HuggingFace (HF Dataset crawler)
User-agent: HuggingFace
Allow: /

# You.com AI search
User-agent: youBot
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Brave Search AI
User-agent: BraveBot
Allow: /

# NeevaAI (acquired but crawler still seen)
User-agent: NeevaBot
Allow: /

# Yandex AI search
User-agent: YandexBot
Allow: /

# ByteDance (TikTok / CapCut AI training crawlers)
User-agent: Bytespider
Allow: /

# Baidu AI search
User-agent: Baiduspider
Allow: /

# Sogou AI
User-agent: Sogou Spider
Allow: /

# Internet Archive
User-agent: ia_archiver
Allow: /

# LinkedIn (SEO + AI embeddings)
User-agent: LinkedInBot
Allow: /

# Pinterest scraping
User-agent: Pinterestbot
Allow: /

# General
User-agent: *
Allow: /

Sitemap: https://www.irshadpc.com/sitemap.xml