# robots.txt - Nur große Suchmaschinen erlauben, AI-Bots blockieren

# === ERLAUBTE SUCHMASCHINEN ===
# Google
User-agent: Googlebot
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /fileadmin/_temp_/
Disallow: /fileadmin/_processed_/
Allow: /typo3conf/ext/*/Resources/Public/

# Bing
User-agent: Bingbot
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /fileadmin/_temp_/
Disallow: /fileadmin/_processed_/
Allow: /typo3conf/ext/*/Resources/Public/

# Yahoo
User-agent: Slurp
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /fileadmin/_temp_/
Disallow: /fileadmin/_processed_/
Allow: /typo3conf/ext/*/Resources/Public/

# Yandex
User-agent: YandexBot
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /fileadmin/_temp_/
Disallow: /fileadmin/_processed_/
Allow: /typo3conf/ext/*/Resources/Public/

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /fileadmin/_temp_/
Disallow: /fileadmin/_processed_/
Allow: /typo3conf/ext/*/Resources/Public/

# Baidu
User-agent: Baiduspider
Allow: /
Disallow: /typo3/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /fileadmin/_temp_/
Disallow: /fileadmin/_processed_/
Allow: /typo3conf/ext/*/Resources/Public/

# === AI-BOTS UND SCRAPER BLOCKIEREN ===
# OpenAI (ChatGPT)
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

# Google AI (Bard/Gemini)
User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

# Common Crawl
User-agent: CCBot
Disallow: /

# Facebook/Meta AI
User-agent: FacebookBot
Disallow: /

User-agent: facebookexternalhit
Disallow: /

# AI2Bot (Allen Institute)
User-agent: AI2Bot
Disallow: /

# Perplexity AI
User-agent: PerplexityBot
Disallow: /

# Weitere AI/ML Bots
User-agent: PetalBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Scraping Bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SeznamBot/
Disallow: /

# === ALLE ANDEREN BOTS BLOCKIEREN ===
User-agent: *
Disallow: /

# === SITEMAPS ===
# Sitemap: https://intosai.org/sitemap.xml

# === CRAWL-DELAY FÜR ERLAUBTE BOTS ===
# Verhindert zu aggressive Crawling-Frequenz
Crawl-delay: 1