42 lines
970 B
Plaintext
42 lines
970 B
Plaintext
# robots.txt for Nail Salon Booking Website
|
|
|
|
# Allow all crawlers to access public pages
|
|
User-agent: *
|
|
Allow: /
|
|
Allow: /prices
|
|
Allow: /contact
|
|
|
|
# Block API endpoints (prevent indexing of API routes)
|
|
Disallow: /api/
|
|
|
|
# Block admin/authentication pages (not meant for search engines)
|
|
Disallow: /login
|
|
Disallow: /admin
|
|
Disallow: /dashboard
|
|
|
|
# Block booking page if it requires authentication
|
|
# (Remove this line if you want the booking page indexed)
|
|
Disallow: /book
|
|
|
|
# Block SvelteKit internal files
|
|
Disallow: /_app/
|
|
Disallow: /.svelte-kit/
|
|
|
|
# Crawl delay for low-capacity server (1-2 users/day expected)
|
|
# Prevents aggressive crawling from overwhelming the server
|
|
Crawl-delay: 10
|
|
|
|
# More restrictive for aggressive crawlers
|
|
User-agent: GPTBot
|
|
User-agent: ChatGPT-User
|
|
User-agent: CCBot
|
|
User-agent: anthropic-ai
|
|
User-agent: Claude-Web
|
|
Disallow: /
|
|
|
|
# Aggressive SEO crawlers
|
|
User-agent: AhrefsBot
|
|
User-agent: SemrushBot
|
|
User-agent: MJ12bot
|
|
User-agent: DotBot
|
|
Crawl-delay: 30 |