# Robot Policy file as per Robot Exclusion standard 17-jun-94
# Details at http://info.webcrawler.com/mak/projects/robots/robots.html

#User-agent: Lycos       # Match Lycos
# Match local htdig search engine
User-agent: htdig
Disallow: /cgi-bin/
Disallow: /usage/
Disallow: /mirror/
Disallow: /pub/linux/
Disallow: /pub/src/
Disallow: /pub/tex/tetex/
Disallow: /pub/tcl/
Disallow: /pub/mirrors/
Disallow: /pub/fedora/
Disallow: /pub/redhat/
Disallow: /pub/epel/
Disallow: /liamtog
Disallow: /weasels

# Match any robot
User-agent: *
Crawl-Delay: 10
Disallow: /cgi-bin/
Disallow: /usage/
Disallow: /mirror/
Disallow: /pub/
Disallow: /liamtog
Disallow: /weasels