# robots.txt for http://www.texascrane.com/ User-agent: * Disallow: /beta/ # This is a development directory Disallow: /test/ # This is a development directory Disallow: /stats/ # webmasters space Disallow: /cgi-bin/ # spiders have no business here