# robots.txt # User-agent: * Crawl-delay: 10 Disallow:*?owa_specialAction* Disallow:/wp-includes Disallow:/wp-content/plugins/podpress Disallow:/wp-admin Disallow:/wp-login.php* Disallow:/cgi-bin Disallow:/css # ignore index.php?wpgb_ Disallow:*wpgb_public_action=query* Disallow:*?wpgb_* # ignore lifestream pages Disallow:*?ls_p=* Disallow:*taxonomy=* Disallow:*trackback/ # I'm not a shoppping website. Don't Index me. User-agent: BecomeBot Disallow:/ User-agent: ia_archiver Crawl-delay: 10 Disallow: User-Agent: MJ12bot Disallow: Sitemap: http://blog.px.ns1.net/sitemap.xml.gz # BEGIN XML-SITEMAP-PLUGIN Sitemap: http://blog.px.ns1.net/sitemap.xml.gz # END XML-SITEMAP-PLUGIN User-agent: * Crawl-delay: 10