# robots.txt for ccidpa.org ( http://www.ccidpa.org and http://m.ccidpa.org )
# Copyright (C) 1998-2009 Collin County IDPA. Re-publication is forbidden.
#
# Disallow Botseer from PSU, Firefox prefetching add-on, etc. Baidu, Seexie,
# SeznamBot, spbot, and Yandex have been banned for violating robots.txt and
# will get a 403-Forbidden response. Also exclude specialized Google and MSN
# 'bots; We have no relevant content for them.
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: msnbot-academic
User-agent: msnbot-media
User-agent: msnbot-newsblogs
User-agent: msnbot-products
User-agent: Baidu
User-agent: Botseer
User-agent: CCBot
User-agent: dotbot
User-agent: Fasterfox
User-agent: SearchMonkey
User-agent: Slurp China
Disallow: /

Crawl-delay: 10

# Googlebot uses meta robots, crawl-delay not supported
# Others use meta robots & crawl-delay.
# Notes: Voyager appears unable to parse multi-agent policy records.
# Twiceler seems to have started correctly parsing multi-agent policy
# records as of 2009-11-18, but periodically reverts to failing.
# WiseNut's Zyborg defunct Oct 2007. Searchme-Charlotte defunct Nov 2009.
User-agent: Googlebot-Mobile
User-agent: Googlebot/
User-agent: botmobi
User-agent: checkbot
User-agent: DuckDuckBot
User-agent: Gigabot
User-agent: mahalobot
User-agent: MJ12bot
User-agent: msnbot
User-agent: ParchBot
User-agent: Robozilla
User-agent: ScSpider
User-agent: Surfsafely
User-agent: Teoma
User-agent: voyager
User-agent: XmarksFetch
Disallow: /*.flv$
Disallow: /*.js$
Disallow: /*.pdf$
Disallow: /*.swf$
Disallow: /*.xls$
Disallow: /accsauth
Disallow: /cgi-
Disallow: /common/em
Disallow: /images/
Disallow: /mat_rslt/archive/
Disallow: /mat_rslt/spreadsheets/
Disallow: /members/
Disallow: /photo_g/
Disallow: /scoring/files/
Disallow: /sp_event/images/
Disallow: /videos/ccidpa-demo.flv
Disallow: /videos/ccidpa-demo.jpg
Disallow: /videos/player.swf
Disallow: /videos/swfobject.js

# Yahoo! Slurp & YahooSeeker mobile; Can't afford to use meta-robots (too many fetches)
User-agent: Slurp
User-agent: YahooSeeker/M1A1-R2D2
Disallow: /*.flv$
Disallow: /*.js$
Disallow: /*.pdf$
Disallow: /*.swf$
Disallow: /*.xls$
Disallow: /cgi-
Disallow: /common/em
Disallow: /images/
Disallow: /mat_rslt/
Disallow: /members/
Disallow: /photo_g
Disallow: /scoring/files/
Disallow: /sp_event/images/
Disallow: /accsauth
Disallow: /ccidpama.html
Disallow: /lwaiver.html
Disallow: /md_sched.html
Disallow: /pri_pol.html
Disallow: /so_orign.html
Disallow: /tg_rules.html
Disallow: /threegun.html
Disallow: /tos.html
Disallow: /tshirts.html
Disallow: /videos/ccidpa-demo.flv
Disallow: /videos/ccidpa-demo.jpg
Disallow: /videos/player.swf
Disallow: /videos/swfobject.js
# Mobile pages (R2D2 does not respect on-page <meta robots> tags)
Disallow: /aboutms.html
Disallow: /drive_
Disallow: /m_cancel.html
Disallow: /mobifaq.html
Disallow: /officers.html
Disallow: /results.html
Disallow: /volunteers.html
Disallow: /webset

# Girafa HTML thumbnailer used by MSN & Internet
# Archive Wayback Machine; allow /images/
User-agent: girafa
User-agent: ia_archiver
User-agent: searchpreview
Disallow: /cgi-
Disallow: /common/em
Disallow: /mat_rslt/
Disallow: /members/
Disallow: /photo_g
Disallow: /scoring/files/
Disallow: /sp_event/images/
Disallow: /accsauth
Disallow: /ccidpama.html
Disallow: /copyright.html
Disallow: /lwaiver.html
Disallow: /md_sched.html
Disallow: /memorial.html
Disallow: /pri_pol.html
Disallow: /so_orign.html
Disallow: /tg_rules.html
Disallow: /threegun.html
Disallow: /tos.html
Disallow: /tshirts.html
Disallow: /basic-equipment.html
Disallow: /calendar/
Disallow: /clastips.html
Disallow: /friends.html
Disallow: /idpa-
Disallow: /newsv1i1.html
Disallow: /ranges.html
Disallow: /sk-guide.html
Disallow: /so-guide.html
Disallow: /so-notes.html
Disallow: /videos/ccidpa-demo.flv
Disallow: /videos/ccidpa-demo.jpg
Disallow: /videos/player.swf
Disallow: /videos/swfobject.js

# all others
User-agent: *
Disallow: /