User-agent: *
Disallow: /adserver/
Disallow: /pt/images/
Disallow: /pt/theme/
Disallow: /pt/language/
Disallow: /pt/resources/
Disallow: /pt/trackback/
Disallow: /pt/comment/
Disallow: /pt/main.do
Disallow: /pt/roller-ui/admin/
Disallow: /pt/roller-ui/yourWebsites.do
Disallow: /pt/roller-ui/login.do
Disallow: /pt/jsps/
Disallow: /pt/custom.html
Disallow: /pt/results.html
Disallow: /pt/css/
Disallow: /pt/api/ptapi.js
Disallow: /pt/js/
Disallow: /pt/ticker/ticker.js
Disallow: /pt/ticker/crawl.js
Disallow: /pt/ticker/video-ticker.js
Disallow: /pt/ticker/video-crawl.js
Disallow: /pt/ticker/ticker.css
Disallow: /developer/
Disallow: /tags/js/
Disallow: /tags/app/
Disallow: /js/
Disallow: /css/

User-agent: attributor
Disallow: /

User-agent: dotbot 
Disallow: / 


# limit EntireWeb to downloading one page every five seconds
User-agent: Speedy
Crawl-Delay: 5