# Allow all crawlers
User-agent: *
Allow: /

# Disallow admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /auth/
Disallow: /*?_rsc=
Disallow: /monitoring
Disallow: /keywords

# Disallow lang dir
Disallow: /en
Disallow: /en/
# Disallow: /ja
# Disallow: /ja/
# Disallow: /ko
# Disallow: /ko/
# Disallow: /th
# Disallow: /th/
# Disallow: /zh-TW
# Disallow: /zh-TW/

# Sitemap location
Sitemap: https://www.novelago.com/sitemap.xml

# Crawl-delay for all bots
# Crawl-delay: 1

# Block specific bots that might cause issues
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Allow: /

# Block image crawlers from specific directories
User-agent: Googlebot-Image
Disallow: /images/private/
Allow: /public/