# NOTICE: The collection of content and other data on this
# site through automated means, including any device, tool,
# or process designed to data mine or scrape content, is
# prohibited except (1) for the purpose of search engine indexing or
# artificial intelligence retrieval augmented generation or (2) with express
# written permission from this site’s operator.

# To request permission to license our intellectual
# property and/or other materials, please contact this
# site’s operator directly.

# BEGIN Cloudflare Managed content

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content
# Robots.txt for AI Free Forever - https://aifreeforever.com
# Allow all search engines and AI crawlers to access the site

# Default rule for all crawlers
User-agent: *
Allow: /


# Google Search Engine
User-agent: Googlebot
Allow: /

# Google Images
User-agent: Googlebot-Image
Allow: /

# Google News
User-agent: Googlebot-News
Allow: /

# Google Mobile
User-agent: Googlebot-Mobile
Allow: /

# Bing Search Engine
User-agent: Bingbot
Allow: /


# Yahoo Search Engine
User-agent: Slurp
Allow: /


# Yandex Search Engine
User-agent: YandexBot
Allow: /


# Baidu Search Engine
User-agent: Baiduspider
Allow: /


# DuckDuckGo Search Engine
User-agent: DuckDuckBot
Allow: /


# AI Search Engines and Crawlers

# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Crawl-delay: 1

# ChatGPT User Agent
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

# OpenAI WebCrawler
User-agent: OpenAI-WebCrawler
Allow: /
Crawl-delay: 1

# Anthropic Claude
User-agent: Claude-Web
Allow: /
Crawl-delay: 1

# Anthropic ClaudeBot
User-agent: ClaudeBot
Allow: /
Crawl-delay: 1

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

# You.com AI Search
User-agent: YouBot
Allow: /
Crawl-delay: 1

# Microsoft Copilot
User-agent: MSEdge
Allow: /
Crawl-delay: 1

# Bing AI Chat
User-agent: BingPreview
Allow: /
Crawl-delay: 1

# Facebook/Meta AI
User-agent: FacebookBot
Allow: /
Crawl-delay: 1

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 1

# X (Twitter) AI
User-agent: Twitterbot
Allow: /
Crawl-delay: 1

# LinkedIn AI
User-agent: LinkedInBot
Allow: /
Crawl-delay: 1

# Apple Intelligence
User-agent: AppleBot
Allow: /
Crawl-delay: 1

# Cohere AI
User-agent: CohereBot
Allow: /
Crawl-delay: 1

# Mistral AI
User-agent: MistralBot
Allow: /
Crawl-delay: 1

# Hugging Face
User-agent: HuggingFaceBot
Allow: /
Crawl-delay: 1

# Stability AI
User-agent: StabilityBot
Allow: /
Crawl-delay: 1

# Replicate AI
User-agent: ReplicateBot
Allow: /
Crawl-delay: 1

# Character.AI
User-agent: CharacterAI
Allow: /
Crawl-delay: 1

# Jasper AI
User-agent: JasperBot
Allow: /
Crawl-delay: 1

# Copy.ai
User-agent: CopyAI
Allow: /
Crawl-delay: 1

# Writesonic
User-agent: WritesonicBot
Allow: /
Crawl-delay: 1

# Grammarly AI
User-agent: GrammarlyBot
Allow: /
Crawl-delay: 1

# Notion AI
User-agent: NotionBot
Allow: /
Crawl-delay: 1

# Canva AI
User-agent: CanvaBot
Allow: /
Crawl-delay: 1

# Adobe AI
User-agent: AdobeBot
Allow: /
Crawl-delay: 1

# Midjourney (if they have a crawler)
User-agent: MidjourneyBot
Allow: /
Crawl-delay: 1

# DALL-E related crawlers
User-agent: DALLEBot
Allow: /
Crawl-delay: 1

# Stable Diffusion crawlers
User-agent: StableDiffusionBot
Allow: /
Crawl-delay: 1

# Academic and Research AI Crawlers
User-agent: ScholarBot
Allow: /
Crawl-delay: 1

User-agent: ResearchBot
Allow: /
Crawl-delay: 1

User-agent: AcademicBot
Allow: /
Crawl-delay: 1

# Social Media Platform Crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: DiscordBot
Allow: /

User-agent: SlackBot
Allow: /

User-agent: RedditBot
Allow: /

# SEO and Analytics Tools
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Crawl-delay: 2

User-agent: MozBot
Allow: /
Crawl-delay: 2

User-agent: MajesticSEO
Allow: /
Crawl-delay: 2

User-agent: UptimeRobot
Allow: /

User-agent: pingdom
Allow: /

# Archive Crawlers
User-agent: archive.org_bot
Allow: /

User-agent: ia_archiver
Allow: /

User-agent: Wayback
Allow: /

# RSS and Feed Readers
User-agent: FeedlyBot
Allow: /

User-agent: FlipboardProxy
Allow: /

# Disallow sensitive areas (if any exist)
# Uncomment and modify as needed
# Disallow: /admin/
# Disallow: /private/
# Disallow: /api/internal/
# Disallow: /.env
# Disallow: /config/

# Allow important files
Allow: /sitemap-0.xml
Allow: /robots.txt
Allow: /favicon.ico
Allow: /manifest.json

# Sitemap location
Sitemap: https://aifreeforever.com/sitemap-0.xml

mended)
Host: https://aifreeforever.com

# Clean URLs - Allow trailing slashes
Allow: /*/

# Allow common file types
Allow: /*.css
Allow: /*.js
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.webp
Allow: /*.svg
Allow: /*.ico
Allow: /*.woff
Allow: /*.woff2
Allow: /*.ttf
Allow: /*.eot
Allow: /*.pdf
Allow: /*.txt
Allow: /*.xml
Allow: /*.json

# Last updated: 2025-01-11
# This robots.txt file is designed to be AI-friendly and supports all major search engines
# and AI crawlers to help with content discovery and indexing