# Robots.txt file 
# All robots will spider the domain 
User-agent: * 
# Disallow directory /cgi-bin/ 
Disallow: /cgi-bin/
Disallow: /images/ 
# Disallow other things
Disallow: /recommend/User-Agent: *
Crawl-Delay: 10