# ForkLog robots.txt User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # ----------------------- # Search Engines # ----------------------- User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: YandexBot Allow: / User-agent: Googlebot-News Allow: / # ----------------------- # AI Search / Citation (answer engines) - full access # ----------------------- User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: PerplexityBot Allow: / # ----------------------- # AI Training Crawlers - news only, plus the AI-systems policy page; everything else blocked # ----------------------- User-agent: GPTBot Allow: /news/ Allow: /for-ai-systems Allow: /.well-known/ai-access Disallow: / User-agent: ClaudeBot Allow: /news/ Allow: /for-ai-systems Allow: /.well-known/ai-access Disallow: / User-agent: anthropic-ai Allow: /news/ Allow: /for-ai-systems Allow: /.well-known/ai-access Disallow: / User-agent: CCBot Allow: /news/ Allow: /for-ai-systems Allow: /.well-known/ai-access Disallow: / User-agent: Applebot-Extended Allow: /news/ Allow: /for-ai-systems Allow: /.well-known/ai-access Disallow: / User-agent: Meta-ExternalAgent Allow: /news/ Allow: /for-ai-systems Allow: /.well-known/ai-access Disallow: / User-agent: Amazonbot Allow: /news/ Allow: /for-ai-systems Allow: /.well-known/ai-access Disallow: / # Bytespider (ByteDance) - aggressive scraper: content fully blocked, only the AI policy page is readable User-agent: Bytespider Allow: /for-ai-systems Allow: /.well-known/ai-access Disallow: / # ----------------------- # Heavy Commercial Crawlers # ----------------------- User-agent: AhrefsBot Crawl-delay: 10 User-agent: SemrushBot Crawl-delay: 10 # ----------------------- # Sitemaps # ----------------------- Sitemap: https://forklog.com/sitemap.xml Sitemap: https://forklog.com/news-sitemap.xml Sitemap: https://forklog.com/hub-archive/sitemap-hub.xml Sitemap: https://forklog.com/en/sitemap.xml Sitemap: https://forklog.com/en/news-sitemap.xml # AI systems access and licensing - https://forklog.com/for-ai-systems AI-Access: https://forklog.com/.well-known/ai-access AI-Access-JSON: https://forklog.com/.well-known/ai-access.json