19 lines
318 B
Text
19 lines
318 B
Text
# robots.txt for scott.scoble.me
|
|
# https://scott.scoble.me
|
|
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
# Sitemap
|
|
Sitemap: https://scott.scoble.me/sitemap.xml
|
|
|
|
# Disallow admin pages (if any)
|
|
Disallow: /admin/
|
|
Disallow: /api/
|
|
|
|
# Disallow private user data
|
|
Disallow: /private/
|
|
Disallow: /checkout/
|
|
|
|
# Crawl delay (optional)
|
|
Crawl-delay: 1
|