# Deny all other spiders
User-agent: *
Crawl-delay: 30
Allow: /*.html
Allow: /*.asp
Allow: /*.php
Allow: /*.cgi
Disallow: /