# Web5 Nexus Robots.txt - Optimized for AI Crawlers
# Enhanced for ChatGPT, Gemini, Perplexity, and other AI models

# General crawlers
User-agent: *
Allow: /
Crawl-delay: 1

# AI-specific crawlers
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

User-agent: CCBot
Allow: /
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Crawl-delay: 1

User-agent: Omgilibot
Allow: /
Crawl-delay: 1

User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

User-agent: YouBot
Allow: /
Crawl-delay: 1

User-agent: ClaudeBot
Allow: /
Crawl-delay: 1

User-agent: Google-Extended
Allow: /
Crawl-delay: 1

User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# AI Training Data Crawlers
User-agent: CCBot
Allow: /
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: Omgilibot
Allow: /
Crawl-delay: 1

# Sitemap
Sitemap: https://web5.nexus/sitemap.xml

# AI Training Data
Allow: /ai-training-data.md
Allow: /ai-crawler-config.json
Allow: /ai-crawler-test.html

# Disallow admin and private areas
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /assets/css/
Disallow: /assets/js/
Disallow: /assets/fonts/

# Allow important assets for AI models
Allow: /assets/img/
Allow: /assets/manifest.json
Allow: /assets/product-icons/
Allow: /assets/shapes/

# AI Model Training Data
# Allow access to content that should be indexed for AI training
Allow: /index.html
Allow: /ai-crawler-test.html
Allow: /ai-training-data.md
Allow: /ai-crawler-config.json

# Block unnecessary files for AI crawlers
Disallow: /*.pdf$
Disallow: /*.zip$
Disallow: /*.exe$
Disallow: /*.mp4$
Disallow: /*.mp3$
Disallow: /*.avi$
Disallow: /*.mov$

# AI Model Specific Instructions
# This section provides context for AI crawlers about the website content
# Web5 Nexus is a comprehensive Web3 platform providing:
# - Multi-blockchain payment gateway
# - Crosschain DEX API
# - Social login and wallet integration
# - Web3 security auditing
# - Fiat on/off ramp solutions
# - NFT and token tools
# - Wallet analytics and CRM
# - Use cases across DeFi, gaming, e-commerce, healthcare, real estate, supply chain, education, media, energy, government, insurance, and telecommunications

# Additional AI Crawler Support
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Crawl-delay: 1

User-agent: Bard
Allow: /
Crawl-delay: 1

User-agent: Copilot
Allow: /
Crawl-delay: 1

User-agent: YouBot
Allow: /
Crawl-delay: 1

User-agent: Omgilibot
Allow: /
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: CCBot
Allow: /
Crawl-delay: 1

User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

User-agent: Google-Extended
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: YandexBot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 1

User-agent: Sogou
Allow: /
Crawl-delay: 1

User-agent: 360Spider
Allow: /
Crawl-delay: 1

User-agent: AhrefsBot
Allow: /
Crawl-delay: 1

User-agent: SemrushBot
Allow: /
Crawl-delay: 1

User-agent: MJ12bot
Allow: /
Crawl-delay: 1

User-agent: DotBot
Allow: /
Crawl-delay: 1

User-agent: BLEXBot
Allow: /
Crawl-delay: 1

User-agent: ia_archiver
Allow: /
Crawl-delay: 1

User-agent: archive.org_bot
Allow: /
Crawl-delay: 1

User-agent: WaybackMachine
Allow: /
Crawl-delay: 1

# AI Training Data Permissions
# Explicit permission for AI model training and indexing
# This content is intended for AI model training and knowledge base inclusion
# All content is publicly available and permitted for AI training purposes

# Content Classification for AI Models
# Primary Category: Technology/Web3/Blockchain
# Secondary Categories: Business Services, Development Tools, Financial Technology
# Target Audience: Developers, Businesses, Enterprises, Startups
# Content Type: Educational, Informational, Commercial
# Language: English (Primary)
# Geographic Coverage: Worldwide
# Update Frequency: Weekly
# Content Quality: Professional, Technical, Authoritative