User-agent: *
Disallow: /admin/
Disallow: /cgi-bin/
Disallow: /css/
Disallow: /downloads/
Disallow: /include/
Disallow: /images/
Disallow: /js/
Disallow: /mint/
Disallow: /photo/
Disallow: /phpmyadmin/
Disallow: /source/
Disallow: /statistics/

User-agent: *
Crawl-delay: 5 # задает таймаут в 5 секунды 

User-agent: Twiceler
Disallow: /

User-agent: Aport
Disallow: /

User-agent: Slurp
Disallow: /

User-agent: msnbot-media
Disallow: /

User-agent: msnbot-news
Disallow: /

User-agent: msnbot-products
Disallow: /

User-agent: msnbot
Disallow: /

User-agent: ia_archiver
Disallow: /