#This robots.txt file is being used to keep robots from indexing 
#It is placed at the document root of the server.
#Created by:  Royann Janus       Date:  07/26/98
#Updated by:  Glenn J. Schworak  Date:  03/13/09

# Let the world know about our site maps
Sitemap: http://licenseinfo.oregon.gov/siteinfo.xml

# Allow linkchecker at http://validator.w3.org/docs/checklink.html walk our entire site
User-Agent: W3C-checklink
Disallow:

# Allow our internal GOOGLE to walk all folders
User-agent: dcbs-google
allow: /

# Allow Google into some select folders
User-agent: Googlebot
Disallow: /exs/imd/survey/
Disallow: /ex/dfcs/dfcslic/mortgage_lender/
allow: /ex/imd/reports/
allow: /ex/osha/film/
allow: /ex/osha/training/training/
allow: /exs/bcd/minor_labels/


# Keep MSN from searching too often and specifically out of the survey
User-agent: msnbot 
Crawl-delay: 120 
Disallow: /exs/imd/survey/
Disallow: /ex/dfcs/dfcslic/mortgage_lender/
allow: /ex/imd/reports/
allow: /ex/osha/film/
allow: /ex/osha/training/training/
allow: /exs/bcd/minor_labels/

# Keep Yahoo! from searching too often and specifically out of the survey
User-agent: Slurp 
Crawl-delay: 120 
Disallow: /exs/imd/survey/
Disallow: /ex/dfcs/dfcslic/mortgage_lender/
allow: /ex/imd/reports/
allow: /ex/osha/film/
allow: /ex/osha/training/training/
allow: /exs/bcd/minor_labels/


# Keep all bots not specificlally mentioned above out of any folders
# that are not specifically allowed in the list above.
User-agent: *
Disallow: /
Disallow: /exs/imd/survey/
Disallow: /ex/dfcs/dfcslic/mortgage_lender/