# robots.txt
# exclude robots from crawling some pages
# Colorado State University Libraries 
# last modified by gvogl 2009-05-14

User-agent: WebTrends
Disallow: /

User-agent: LinkScan
Disallow: /

User-agent: *

# miscellaneous support files and media
Disallow: /exlibris/dtl/u3_1
Disallow: /exlibris/dtl/d3_1/dtl
Disallow: /exlibris/dtl/d3_1/dtle
Disallow: /exlibris/dtl/d3_1/pds
Disallow: /exlibris/dtl/d3_1/tmp
# this contains all thumbnails and PDF delivery objects; disallow this to prevent any possibility of viewing/crawling of CSU-only or embargoed items
#Disallow: /exlibris/dtl/d3_1/apache_media

# staff modules and pages requiring login
Disallow: /approver
Disallow: /collection
Disallow: /main
Disallow: /mng
Disallow: /pds
Disallow: /view
Disallow: /webingest

# web statistics
Disallow: /webstats

# metadata
Disallow: /webclient/MetadataManager
Disallow: /webclient/DeliveryManager?custom_att_2=simple_viewer

# OAI records
Disallow: /OAI-PUB?verb=GetRecord

# Resource Discovery URLs with session IDs
Disallow: /R?RN=
Disallow: /R/0
Disallow: /R/1
Disallow: /R/2
Disallow: /R/3
Disallow: /R/4
Disallow: /R/5
Disallow: /R/6
Disallow: /R/7
Disallow: /R/8
Disallow: /R/9
Disallow: /R/A
Disallow: /R/B
Disallow: /R/C
Disallow: /R/D
Disallow: /R/E
Disallow: /R/F
Disallow: /R/G
Disallow: /R/H
Disallow: /R/I
Disallow: /R/J
Disallow: /R/K
Disallow: /R/L
Disallow: /R/M
Disallow: /R/N
Disallow: /R/O
Disallow: /R/P
Disallow: /R/Q
Disallow: /R/R
Disallow: /R/S
Disallow: /R/T
Disallow: /R/U
Disallow: /R/V
Disallow: /R/W
Disallow: /R/X
Disallow: /R/Y
Disallow: /R/Z