############################################################################### # # This is a file retrieved by webwalkers a.k.a. spiders that conform to a # defacto standard. # # See # # Format is: # User-agent: # Disallow: | #------------------------------------------------------------------------------ # # Date: 2001/Feb/28 12:06pm EST: Christopher Rivera # creating initial robots.txt for all groups # robots are excluded from all until the each website is built # #------------------------------------------------------------------------------ User-agent: * Disallow: /cgi-bin Disallow: /wp-admin Disallow: /wp-includes Disallow: /wp-content/plugins Disallow: /wp-content/cache Disallow: /wp-content/themes Disallow: /category/*/* Disallow: */trackback Disallow: */feed Disallow: */comments Disallow: /*?* Disallow: /*? Allow: /wp-content/uploads # Google Image User-agent: Googlebot-Image Disallow: Allow: /* # Google AdSense: allow adsense bot on entire site User-agent: Mediapartners-Google* Disallow: Allow: /* # Internet Archiver Wayback Machine User-agent: ia_archiver Disallow: / # BEGIN XML-SITEMAP-PLUGIN Sitemap: http://www.chrisrivera.com/sitemap.xml.gz # END XML-SITEMAP-PLUGIN