Sitemap: http://www.dut.de/sitemap.167.0.html

# unerwueschte Epider
User-agent: EmailCollector
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: EmailWolf
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: Webwhacker
Disallow: /
User-agent: Webzip
Disallow: /
User-agent: Net Attache
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: WebCapture
Disallow: /
User-agent: WebSauger
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Roverbot
Disallow: /

# besondere Bilder-Spider
User-agent: Googlebot-Image
Disallow: /
User-agent: vscooter
Disallow: /
User-agent: Yahoo-MMCrawler
Disallow: /
User-agent: psbot
Disallow: /

# google versteht nun auch wildcards
User-agent: Googlebot
Disallow: /*.gif$
Disallow: /*.png$
Disallow: /*.jpg$
Disallow: /fileadmin/_pics
Disallow: /fileadmin/layout
Disallow: /uploads/pics
Disallow: /page.php

# allgemein
User-agent: *
Disallow: /fileadmin/_pics
Disallow: /fileadmin/layout
Disallow: /uploads/pics
Disallow: /page.php

# IC-Jobs Crawler
User-agent: ICCrawler - ICjobs
Crawl-delay: 20