# robots.txt file for CDS web servers
# Ref: 2005070100

User-agent: *
Disallow: /js/
Disallow: /css/
Disallow: /images/
Disallow: /multimedia/ # Delete entry if we want to allow PDF crawling
Disallow: /dir/ # Disallow crawling of our directory
Disallow: /edc/ # Disallow our admin tools
Disallow: /tools/ # Disallow our admin tools
Disallow: /registration/ # Disallow Shadow
Disallow: /html/ # We don't have static pages
Disallow: /mld/mercurynews/3626332.htm # Disallow a specific story for cache flushing
Disallow: /reg/ # Disallow Registration
Disallow: /mld/krcontent/ # This is a dummy publication and should not be spidered


# Rover is a bad dog <http://www.roverbot.com>
User-agent: Roverbot
Disallow: /