# My Net Worth - Robots.txt
# https://mynetworth.today

# Allow all crawlers
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://mynetworth.today/sitemap.xml

# Crawl delay (optional, be nice to servers)
Crawl-delay: 1

# Specific bot rules
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# AI search and user-requested fetches (Oct 2026). "*" already allows them; the
# groups say so on purpose. A site map for agents: https://mynetworth.today/llms.txt
# Training crawlers (GPTBot, ClaudeBot, Google-Extended, CCBot) fall under "*"
# and are allowed too; give them their own Disallow group to opt out.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# Allow SEO tools for auditing
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /