# /robots.txt file for http://webcrawler.com/
# mod 15Nov03


User-agent: *
Disallow: /counters
Disallow: /cgi-bin
Disallow: /errorfiles
Disallow: /Templates
Disallow: /Library
Disallow: /_notes