User-agent: *
Disallow: /geneweb
Disallow: /images
Disallow: /astro/report
Disallow: /astro_en/report
Disallow: /astro/photo
Disallow: /astro_en/photo
Disallow: /astro/spmanual
Disallow: /link
Disallow: /bot
Disallow: /Scripts
User-agent: Slurp
Disallow: /astro/download
Disallow: /astro/download/
Disallow: /geneweb
Disallow: /images
Disallow: /astro/report
Disallow: /astro_en/report
Disallow: /astro/photo
Disallow: /astro_en/photo
Disallow: /astro/spmanual
Disallow: /link
Disallow: /bot
Disallow: /Scripts
User-agent: htdig_local
Disallow: /geneweb
Disallow: /images
Disallow: /link
Disallow: /bot
Disallow: /gallery
User-agent: Iconsurf
Disallow: /
User-agent: Jetbot
Disallow: /
User-agent: turnitinbot
Disallow: /
User-Agent: MJ12bot
Crawl-Delay:   10
Disallow: /