# robots.txt file for httpwebcrawler.com
# mail webmaster@webcrawler.com for constructive criticism

User-agent: Slurp
Crawl-delay: 5

User-agent: *
Disallow: /openads/