User-agent: * Disallow: /wp-admin/ Disallow: /psdrepo.html Disallow: /psdrepo_21_11.html Disallow: /psdrepo_27_11.html Disallow: /psdrepo_27_12.html Disallow: /psdrepo_home_27_12.html Allow: /wp-admin/admin-ajax.php # bing.com User-agent: msnbot Crawl-delay: 200 # ahrefs.com # http://ahrefs.com/robot/ # Mozilla/5.0+(compatible;+AhrefsBot/5.0;++http://ahrefs.com/robot/) User-agent: AhrefsBot Disallow: / # exabot.com # Mozilla/5.0+(compatible;+Exabot/3.0;++http://www.exabot.com/go/robot) # http://www.exabot.com/go/robot User-agent: exabot Disallow: / # Riddler # Riddler (http://riddler.io/about) # http://riddler.io/about User-agent: Riddler Disallow: / # SMTBot # http://www.similartech.com/smtbot # Mozilla/5.0+(compatible;+SMTBot/1.0;++http://www.similartech.com/smtbot) User-agent: SMTBot Disallow: / # ScoutJet # http://www.scoutjet.com # Mozilla/5.0+(compatible;+ScoutJet;++http://www.scoutjet.com/) # Mozilla/5.0+(compatible;+Blekkobot;+ScoutJet;++http://blekko.com/about/blekkobot) User-agent: ScoutJet Disallow: / # Majestic Bot (MJ12bot) # http://www.majestic12.co.uk/bot.php # Mozilla/5.0+(compatible;+MJ12bot/v1.4.5;+http://www.majestic12.co.uk/bot.php?+) User-agent: MJ12bot Disallow: / # SeeBot # http://www.seegnify.com/bot # seebot/2.0 (+http://www.seegnify.com/bot) User-agent: SeeBot Disallow: / # AdBeat Bot # https://www.adbeat.com/operation_policy # adbeat_bot User-agent: adbeat_bot Disallow: / # MeanPathBot # http://www.meanpath.com/meanpathbot.html # Mozilla/5.0+(compatible;+meanpathbot/1.0;++http://www.meanpath.com/meanpathbot.html) User-agent: meanpathbot Disallow: / # Mojeek Bot # https://www.mojeek.com/bot.html # Mozilla/5.0+(compatible;+MojeekBot/0.6;++https://www.mojeek.com/bot.html) User-agent: MojeekBot Disallow: / # Speedy Spider / EntireWeb Search # http://www.entireweb.com # Speedy+Spider+(http://www.entireweb.com) User-agent: Speedy Spider Disallow: / # GenieO # http://www.genieo.com/webfilter.html # Mozilla/5.0+(compatible;+Genieo/1.0+http://www.genieo.com/webfilter.html) User-agent: genieo Disallow: / # scrapy.org # by default ROBOTSTXT_OBEY is set to false # http://doc.scrapy.org/en/latest/topics/settings.html User-agent: Scrapy Disallow: / # Gigablast Open Source Search Engine/Crawler/Spider # You can ignore robots.txt # http://gigablast.com/admin.html # GigablastOpenSource/1.0 User-agent: GigablastOpenSource Disallow: / # Semrush Bot # http://www.semrush.com/bot.html # Mozilla/5.0+(compatible;+SemrushBot-SA/0.97;++http://www.semrush.com/bot.html) User-agent: SemrushBot Disallow: / # Proximic # http://www.proximic.com/info/spider.php # Mozilla/5.0+(compatible;+proximic;++http://www.proximic.com/info/spider.php) User-agent: proximic Disallow: / # CCBot # http://commoncrawl.org/faq/ # CCBot/2.0+(http://commoncrawl.org/faq/) User-agent: CCBot Disallow: / # Plukkie # http://www.botje.com/plukkie.htm # Mozilla/5.0+(compatible;+Plukkie/1.5;+http://www.botje.com/plukkie.htm) User-agent: Plukkie Disallow: / # Lipperhey Kaus Australis # https://www.lipperhey.com/en/about/ # Mozilla/5.0+(compatible;+Lipperhey-Kaus-Australis/5.0;++https://www.lipperhey.com/en/about/) User-agent: lipperhey Disallow: / # oBot # http://filterdb.iss.net/crawler/ # Mozilla/5.0+(compatible;+oBot/2.3.1;+http://filterdb.iss.net/crawler/) # Mozilla/5.0+(compatible;+oBot/2.3.1;++http://filterdb.iss.net/crawler/) User-agent: oBot Disallow: / # Screaming Frog SEO Spider # Downloaded software, robots.txt can be ignored # http://www.screamingfrog.co.uk/seo-spider/user-guide/general/ # Screaming+Frog+SEO+Spider/2.55 # Screaming+Frog+SEO+Spider/3.3 User-agent: Screaming Frog SEO Spider Disallow: / # VegeBot # http://www.exactbot.com/vegebot/index.html User-agent: VegeBot Disallow: / # ScreenerBot # ScreenerBot Crawler Beta 2.0 (+http://www.ScreenerBot.com) User-agent: ScreenerBot Disallow: / # Garlik from Experian # GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com) User-agent: Garlik Disallow: / # http-kit/2.0 # BusinessBot: Nathan@lead-caddy.com User-agent: BusinessBot Disallow: / # FAST Enterprise Crawler/5.3.4 (crawler@fast.no) User-agent: FAST Enterprise Disallow: / # Nutch # http://nutch.apache.org/bot.html # Nutch+Spider/Nutch-1.5 # mycrowl/Nutch-1.9 # nutch+crawler/Nutch-1.9 # sky+nutch+crawler/Nutch-1.9 # Kraken/Nutch-2.2.1+(Nutch+crawler+launched+by+Integral+Ad+Science,+Inc.;+TBD;+TBD) User-agent: Nutch Disallow: / # Open Link Profiler # Mozilla/5.0+(compatible;+spbot/4.4.2;++http://OpenLinkProfiler.org/bot+) # http://OpenLinkProfiler.org/bot User-agent: spbot Disallow: / # James Bot # Mozilla/5.0+(Windows;+U;+Windows+NT+5.1;+en-US;+rv:1.8.1.6)+Gecko/20070725+Firefox/2.0.0.6+-+James+BOT+-+WebCrawler+http://cognitiveseo.com/bot.html # http://cognitiveseo.com/bot.html User-agent: JamesBOT Disallow: / # Qwantify # Mozilla/5.0+(compatible;+Qwantify/2.1n;++https://www.qwant.com/)/* # Mozilla/5.0+(compatible;+Qwantify/2.0n;++https://www.qwant.com/)/* # https://www.qwant.com/ User-agent: Qwantify Disallow: / # Kraken # Mozilla/5.0+(compatible;+Kraken/0.1;+http://linkfluence.net/;+bot@linkfluence.net) # http://linkfluence.net/ User-agent: Kraken Disallow: / # Mozilla/5.0+(compatible;+linkdexbot/2.0;++http://www.linkdex.com/bots/) # http://www.linkdex.com/bots/ User-agent: linkdex Disallow: / # Mozilla/5.0+(compatible;+MSIE+10.0;+Windows+NT+6.1;+Trident/6.0)+SiteCheck-sitecrawl+by+Siteimprove.com # Siteimprove.com User-agent: SiteCheck-sitecrawl by Siteimprove.com Disallow: / # Mozilla/5.0+(compatible;+MegaIndex.ru/2.0;++http://megaindex.com/crawler) # http://megaindex.com/crawler User-agent: MegaIndex Disallow: / # Prlog # Mozilla/5.0+(compatible;+Prlog/1.0;++http://prlog.ru/) # http://prlog.ru/ User-agent: Prlog Disallow: / # Mozilla/5.0+(compatible;+aiHitBot/2.9;++https://www.aihitdata.com/about) # https://www.aihitdata.com/about User-agent: AiHitBot Disallow: / # KinglandSystemsCorp/KinglandSystemsCorp-crawler-2.0.1+(A+prototype+nutch+crawler+configuration+from+Kingland+Systems;+http://www.kingland.com;+kyle+at+kingland+dot+com) # http://www.kingland.com User-agent: KinglandSystemsCorp Disallow: / # Mozilla/5.0+(compatible;+SitemapBot/0.4;++http://www.SitemapBot.com;+SitemapBot@stmp.com) # http://www.sitemapbot.com User-agent: SitemapBot Disallow: / # Mozilla/5.0+(compatible;+Googlebot/2.1;+https://www.deepcrawl.com/bot) # https://www.deepcrawl.com/bot User-agent: deepcrawl Disallow: / # Mozilla/5.0+(compatible;+heritrix/3.3.0-SNAPSHOT-20150302-2206++http://127.0.0.1) # https://webarchive.jira.com/wiki/display/Heritrix/Heritrix User-agent: heritrix Disallow: / # Mozilla/5.0+(compatible;+Yeti/1.1;++http://help.naver.com/robots/) # http://help.naver.com/robots/ User-agent: Yeti Disallow: / # Mozilla/5.0+(Windows+NT+6.3;+WOW64;+rv:36.0)+Gecko/20100101+Firefox/36.0+(NetShelter+ContentScan,+contact+abuse@inpwrd.com+for+information) User-agent: NetShelter Disallow: / # VSE/1.0+(rabraham@multiview.com) # LSSRocketCrawler/1.0 LightspeedSystems # http://lightspeedsystems.com User-agent: LSSRocketCrawler Disallow: / # Mozilla/5.0+(compatible;+idmarch+Automatic.beta/1.3;++http://www.idmarch.org/bot.html) # http://www.idmarch.org/bot.html User-agent: idmarch Disallow: / # Mozilla/5.0+(compatible;+MSIE+or+Firefox+mutant;+not+on+Windows+server;)+Daumoa/4.0 # Mozilla/5.0+(compatible;+MSIE+or+Firefox+mutant;+not+on+Windows+server;)+Daumoa/4.0+(Following+Mediapartners-Google) User-agent: Daumoa Disallow: / # python-requests/2.4.3+CPython/3.4.2+Linux/3.14.42-31.38.amzn1.x86 # python-requests/2.0.1+CPython/2.7.6+Linux/3.13.0-36-generic # python-requests/2.7.0 CPython/2.7.3 Linux/3.12-0.bpo.1-amd64 User-agent: CPython Disallow: / # Mozilla/5.0+(compatible;+CukBot;+Not+a+spammer;+++https://www.companiesintheuk.co.uk/bot.html) # https://www.companiesintheuk.co.uk/bot.html User-agent: CukBot Disallow: / # Mozilla/5.0+(compatible;+ExpertSearchSpider++http://www.expertsearch.nl/spider) # http://www.expertsearch.nl/spider User-agent: ExpertSearchSpider Disallow: / # Mozilla/5.0+(compatible;+Goodzer/2.0;+crawler@goodzer.com) User-agent: Goodzer Disallow: / # WeSEE # http://www.wesee.com/ User-agent: WeSEE Disallow: / # http://ltx71.com/ # ltx71 - (http://ltx71.com/) User-agent: ltx71 Disallow: / # Symfony2+BrowserKit User-agent: Symfony2 Disallow: / # rogerbot/1.0+(http://moz.com/help/pro/what-is-rogerbot-,+rogerbot-crawler+shiny@moz.com) # http://moz.com/help/pro/what-is-rogerbot- User-agent: rogerbot Disallow: / # Mozilla/5.0+(Windows;+U;+Windows+NT+6.0;+en-GB;+rv:1.0;+trendictionbot0.5.0;+trendiction+search;+http://www.trendiction.de/bot;+please+let+us+know+of+any+problems;+web+at+trendiction.com)+Gecko/20071127+Firefox/3.0.0.11 # http://www.trendiction.de/bot User-agent: trendictionbot Disallow: / # SearchMassive+internal+links+crawler/Nutch-1.10+(Crawl+and+collect+internal+links+from+websites) User-agent: SearchMassive Disallow: / # spiderbot User-agent: spiderbot Disallow: / # yacybot+(/global;+amd64+Linux+3.16.0-4-amd64;+java+1.7.0_79;+Europe/de)+http://yacy.net/bot.html # yacybot+(/global;+amd64+Linux+3.16.0-4-amd64;+java+1.7.0_79;+America/en)+http://yacy.net/bot.html User-agent: yacybot Disallow: / # AboutUsBot/Harpy+(Website+Analysis;+http://www.aboutus.org/Aboutus:Bot;+help@aboutus.org) # http://www.aboutus.org/Aboutus:Bot User-agent: AboutUsBot Disallow: / # http://www.nominet.org.uk/privacypolicy # Mozilla/5.0+(compatible;+DotBot/1.1;+http://www.opensiteexplorer.org/dotbot,+help@moz.com) # http://www.opensiteexplorer.org/dotbot User-agent: DotBot Disallow: / # Mozilla/5.0+(compatible;+BLEXBot/1.0;++http://webmeup-crawler.com/) # http://webmeup-crawler.com/ User-agent: BLEXBot Disallow: / # Perl-Win32::Internet/0.087 # Twitterbot/1.0 # Xenu Link Sleuth 1.3.4 # Xenu Link Sleuth/1.3.8 User-agent: Xenu Link Sleuth Disallow: / # Go language # Go+1.1+package+http User-agent: Go 1.1 Disallow: / # wotbox.com # http://www.wotbox.com/bot/ # Wotbox/2.01+(+http://www.wotbox.com/bot/) User-agent: Wotbox Disallow: / # yandex # http://yandex.com/bots # Mozilla/5.0+(compatible;+YandexBot/3.0;++http://yandex.com/bots) User-agent: YandexBot Disallow: / # Mozilla/5.0+(compatible;+Yahoo!+Slurp;+http://help.yahoo.com/help/us/ysearch/slurp) # http://help.yahoo.com/help/us/ysearch/slurp # User-agent: Yahoo! Slurp # Disallow: / # domainreanimator.com User-agent: Domain Re-Animator Bot Disallow: / # crazywebcrawler.com User-agent: CrazyWebCrawler-Spider Disallow: / # crazywebcrawler.com User-agent: CrazyWebCrawler Disallow: / # OpenVAS.org #User-agent: OpenVAS #Disallow: / # WeCrawlForThePeace - We are not Evil User-agent: WeCrawForThePeace Disallow: / # WeLikeLinks - WeAreNotEvil User-agent: WeLikeLinks Disallow: / # icevikatam/1.0 #User-agent: icevikatam #Disallow: / # Toweya search engine (FR) # Toweyabot: toweya.com User-agent: Toweyabot Disallow: / # TScoutBot 1.0 User-agent: TScoutBot Disallow: / # Wappalyzer # Chrome/49.0.2623.87 (compatible; Wappalyzer; +https://github.com/AliasIO/Wappalyzer) User-agent: Wappalyzer Disallow: / # http://cliqz.com/company/cliqzbot User-agent: Cliqzbot Disallow: / # http://napoveda.seznam.cz/en/seznambot-intro/ User-agent: SeznamBot Disallow: / # Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php) User-agent: grapeshot Disallow: / User-agent: seoscanners Disallow:/ User-agent: crawl.sogou.com Disallow:/ User-agent: sogouspider Disallow:/ User-agent: Sogou web spider Disallow:/ User-agent: serpstatbot Disallow: / User-agent: SemanticScholarBot Disallow: / User-agent: Mail.RU_Bot/2.0 Disallow: / User-agent: ias_crawler Disallow: / User-agent: coccocbot-image/1.0 Disallow: / User-agent: Barkrowler/0.9 Disallow: / User-agent: IstellaBot Disallow: / SiteMap: https://psdrepo.com/sitemap.xml