Good day, we had some issues over the weekend at LiquidWeb! The problem was a large volume of crawling on some specific websites. Here is a good practice to prevent this from happening.<\/p>\n
—————————————————————- —————————————————————- This input will block access to your website to all bots apart of Google.<\/p>\n In Theory.<\/strong><\/em> Many bots don’t respect it so it is good idea to block them through .htaccess file.<\/p>\n —————————————————————- RewriteCond %{HTTP_USER_AGENT} .*Twice.* [OR] Order Allow,Deny # make your own list \ud83d\ude09 PlaySafe!<\/strong><\/em><\/p>\n Note:<\/strong> RewriteCond ^(.*)$ ,…. will forward all crawler’s to http:\/\/example.com [L,R=301]<\/p>\n Enjoy!<\/strong><\/p>\n","protected":false},"excerpt":{"rendered":" Good day, we had some issues over the weekend at LiquidWeb! The problem was a large volume of crawling on some specific websites. Here is a good practice to prevent this from happening. —————————————————————- robot.txt (Block only those bots) —————————————————————- user-agent: AhrefsBot user-agent: MJ12bot user-agent: Semrushbot disallow: \/ —————————————————————- robot.txt (Block all except google) —————————————————————- … Continue reading Prevent Bad Crawler Bots to overload your server!<\/span><\/a><\/p>\n","protected":false},"author":2,"featured_media":2905,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-2903","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-general"],"_links":{"self":[{"href":"https:\/\/easy-admin.ca\/index.php\/wp-json\/wp\/v2\/posts\/2903","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/easy-admin.ca\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/easy-admin.ca\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/easy-admin.ca\/index.php\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/easy-admin.ca\/index.php\/wp-json\/wp\/v2\/comments?post=2903"}],"version-history":[{"count":0,"href":"https:\/\/easy-admin.ca\/index.php\/wp-json\/wp\/v2\/posts\/2903\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/easy-admin.ca\/index.php\/wp-json\/wp\/v2\/media\/2905"}],"wp:attachment":[{"href":"https:\/\/easy-admin.ca\/index.php\/wp-json\/wp\/v2\/media?parent=2903"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/easy-admin.ca\/index.php\/wp-json\/wp\/v2\/categories?post=2903"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/easy-admin.ca\/index.php\/wp-json\/wp\/v2\/tags?post=2903"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}
\nrobot.txt (Block only those bots)<\/strong>
\n—————————————————————-
\nuser-agent: AhrefsBot
\nuser-agent: MJ12bot
\nuser-agent: Semrushbot<\/strong>
\ndisallow: \/<\/p>\n
\nrobot.txt (Block all except google)<\/strong>
\n—————————————————————-
\nUser-Agent: Googlebot
\nAllow: \/
\nUser-Agent: *
\nDisallow: \/<\/p>\n
\n.htaccess<\/strong>
\n—————————————————————-
\nRewriteEngine On
\nRewriteBase \/<\/p>\n
\nRewriteCond %{HTTP_USER_AGENT} .*Yand.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*Yahoo.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*Voil.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*libw.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*Java.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*Sogou.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*psbot.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*Exabot.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*boitho.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*ajSitemap.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*Rankivabot.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*DBLBot.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*MJ1.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*Rankivabot.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*ask.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*AhrefsBot.* [OR]
\nRewriteCond %{HTTP_USER_AGENT} .*Semrush.*<\/strong>
\nRewriteRule ^(.*)$ http:\/\/example.com\/ [L,R=301]<\/p>\n
\nAllow from all
\nDeny from 104.16.0.0\/12<\/strong>
\nDeny from 110.0.0.0\/8
\nDeny from 111.0.0.0\/8
\nDeny from 112.0.0.0\/5
\nDeny from 120.0.0.0\/6
\nDeny from 124.0.0.0\/8
\nDeny from 125.0.0.0\/8
\nDeny from 147.0.0.0\/8
\nDeny from 169.208.0.0
\nDeny from 175.0.0.0\/8
\nDeny from 180.0.0.0\/8
\nDeny from 182.0.0.0\/8
\nDeny from 183.0.0.0\/8
\nDeny from 202.0.0.0\/8
\nDeny from 203.0.0.0\/8
\nDeny from 210.0.0.0\/8
\nDeny from 211.0.0.0\/8
\nDeny from 218.0.0.0\/8
\nDeny from 219.0.0.0\/8
\nDeny from 220.0.0.0\/8
\nDeny from 221.0.0.0\/8
\nDeny from 222.0.0.0\/8<\/p>\n