User-agent: *           # directed to all spiders,

Disallow: /cgi/
Disallow: /dev/
Disallow: /web/