# This rule means it applies to all user-agents User-agent: * # Disallow all directories and files within Disallow: /tommy/wp-admin/ Disallow: /tommy/wp-includes/ Disallow: */feed/ Disallow: */trackback/ # The Googlebot is the main search bot for google User-agent: Googlebot # Disallow all files ending with these extensions Disallow: /*.php$ Disallow: /*.js$ Disallow: /*.inc$ Disallow: /*.css$ Disallow: /*.gz$ Disallow: /*.wmv$ Disallow: /*.tar$ Disallow: /*.tgz$ Disallow: /*.cgi$ Disallow: /*.xhtml$ Disallow: /*.jpg$ Disallow: /*.gif$ Disallow: /*.png$ Disallow: /*.jpeg$ # Disallow Google from parsing indididual post feeds and trackbacks.. Disallow: */feed/ Disallow: */trackback/ # Disallow all files with ? in url Disallow: /*?* Disallow: /*? # The Googlebot-Image is the image bot for google User-agent: Googlebot-Image # Allow Everything Disallow: /* # This is the ad bot for google User-agent: Mediapartners-Google* # Allow Everything Allow: /* User-agent: humanlinks Disallow: / User-agent: LinkextractorPro Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Mata Hari Disallow: / User-agent: LexiBot Disallow: / User-agent: Web Image Collector Disallow: / User-agent: The Intraformant Disallow: / User-agent: True_Robot1.0 Disallow: / User-agent: True_Robot Disallow: / User-agent: BlowFish1.0 Disallow: / User-agent: JennyBot Disallow: / User-agent: MIIxpc4.2 Disallow: / User-agent: BuiltBotTough Disallow: / User-agent: ProPowerBot2.14 Disallow: / User-agent: BackDoorBot1.0 Disallow: / User-agent: toCrawlUrlDispatcher Disallow: / User-agent: WebEnhancer Disallow: / User-agent: suzuran Disallow: / User-agent: VCI WebViewer Disallow: / User-agent: VCI WebViewer Win32 Disallow: / User-agent: VCI Disallow: / User-agent: Szukacz1.4 Disallow: / User-agent: QueryN Metasearch Disallow: / User-agent: Openfind data gathere Disallow: / User-agent: ia_archiver Disallow: / User-agent: Openfind Disallow: / User-agent: Xenu's Link Sleuth 1.1c Disallow: / User-agent: Xenu's Disallow: / User-agent: Zeus Disallow: / User-agent: RepoMonkey Bait & Tacklev1.01 Disallow: / User-agent: RepoMonkey Disallow: / User-agent: Microsoft URL Control Disallow: / User-agent: Openbot Disallow: / User-agent: URL Control Disallow: / User-agent: Zeus Link Scout Disallow: / User-agent: Zeus 32297 Webster Pro V2.9 Win32 Disallow: / User-agent: Webster Pro Disallow: / User-agent: EroCrawler Disallow: / User-agent: LinkScan8.1a Unix Disallow: / User-agent: Keyword Density0.9 Disallow: / User-agent: Kenjin Spider Disallow: / User-agent: Iron331.0.2 Disallow: / User-agent: Bookmark search tool Disallow: / User-agent: GetRight4.2 Disallow: / User-agent: FairAd Client Disallow: / User-agent: Gaisbot Disallow: / User-agent: Aqua_Products Disallow: / User-agent: Radiation Retriever 1.1 Disallow: / User-agent: WebmasterWorld Extractor Disallow: / User-agent: Flaming AttackBot Disallow: / User-agent: Oracle Ultra Search Disallow: / User-agent: MSIECrawler Disallow: / User-agent: PerMan Disallow: / User-agent: searchpreview Disallow: / User-agent: sootle Disallow: / User-agent: es Disallow: / User-agent: Enterprise_Search1.0 Disallow: / User-agent: Enterprise_Search Disallow: /