# robots.txt for http://www.bps.org.uk/ # block all spiders from the following sections User-agent: * Disallow: /app_cache Disallow: /app_config Disallow: /CFIDE Disallow: /report Disallow: /lib Disallow: /healthmonitor Disallow: /variablesDump User-agent: gsa-crawler Allow: *