#START: User-Agent: *
User-Agent: *
#start: disallow directory indexing
Disallow: /bpre/
Disallow: /common/
Disallow: /fckeditor/
Disallow: /mail/
Disallow: /rss/
Disallow: /site/
Disallow: /update/
#end: disallow directory indexing
Sitemap: http://pub.wbweb.co.uk/sitemap.xml
#start: disallow file indexing
Disallow: /favicon.ico
Disallow: /pub.rss
#end: disallow file indexing
#END: User-Agent: *
#START: User-Agent: Sphider2
User-Agent: Sphider2
#start: disallow directory indexing
Disallow: /bpre/
Disallow: /common/
Disallow: /fckeditor/
Disallow: /mail/
Disallow: /rss/
Disallow: /site/
Disallow: /update/
#end: disallow directory indexing
Sitemap: http://pub.wbweb.co.uk/sitemap.xml
#start: disallow file indexing
Disallow: /favicon.ico
Disallow: /pub.rss
#end: disallow file indexing
#END: User-Agent: Sphider2