[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] block some bots for good

[Holger Levsen]

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
c2549849 by Holger Levsen at 2018-10-30T18:59:25Z
block some bots for good

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


1 changed file:

- hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf


Changes:

=====================================
hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf
=====================================
@@ -20,36 +20,61 @@
 	ServerAdmin holger at layer-acht.org
 
 	#block bad bots with a 403
-	SetEnvIfNoCase User-Agent "facebookexternalhit" bad_bot
-	SetEnvIfNoCase User-Agent "Twitterbot" bad_bot
+	SetEnvIfNoCase User-Agent "AhrefsBot" bad_bot
 	SetEnvIfNoCase User-Agent "Baiduspider" bad_bot
-	SetEnvIfNoCase User-Agent "MetaURI" bad_bot
-	SetEnvIfNoCase User-Agent "mediawords" bad_bot
+	SetEnvIfNoCase User-Agent "Exabot" bad_bot
+	SetEnvIfNoCase User-Agent "Experibot" bad_bot
+	SetEnvIfNoCase User-Agent "facebookexternalhit" bad_bot
 	SetEnvIfNoCase User-Agent "FlipboardProxy" bad_bot
+	SetEnvIfNoCase User-Agent "ltx71" bad_bot
+	SetEnvIfNoCase User-Agent "mediawords" bad_bot
+	SetEnvIfNoCase User-Agent "MetaURI" bad_bot
 	SetEnvIfNoCase User-Agent "SemrushBot" bad_bot
 	SetEnvIfNoCase User-Agent "Sogou web spider" bad_bot
+	SetEnvIfNoCase User-Agent "Twitterbot" bad_bot
+	SetEnvIfNoCase User-Agent "yacybot" bad_bot
 	#end block bad bots
 
 	<Directory />
 		Options FollowSymLinks
 		AllowOverride None
+		<Limit GET POST HEAD>
+		  Order Allow,Deny
+		  Allow from all
+		  Deny from env=bad_bot
+		</Limit>
 	</Directory>
 	<Directory /var/www/>
 		Options Indexes FollowSymLinks MultiViews
 		AllowOverride None
 		Require all granted
 		AddType text/plain .log
+		<Limit GET POST HEAD>
+		  Order Allow,Deny
+		  Allow from all
+		  Deny from env=bad_bot
+		</Limit>
 	</Directory>
 	<Directory /var/lib/jenkins/userContent>
 		Options Indexes FollowSymLinks MultiViews
 		AllowOverride None
 		Require all granted
 		AddType text/plain .log
+		<Limit GET POST HEAD>
+		  Order Allow,Deny
+		  Allow from all
+		  Deny from env=bad_bot
+		</Limit>
 	</Directory>
 	<Directory /srv/reproducible-builds.org/lfs>
 		Options Indexes FollowSymlinks MultiViews
 		AllowOverride None
 		Require all granted
+		<Limit GET POST HEAD>
+		  Order Allow,Deny
+		  Allow from all
+		  Deny from env=bad_bot
+		</Limit>
 	</Directory>
 
 	<FilesMatch "\.gz$">
@@ -80,11 +105,6 @@ Use https-redirect www.diffoscope.org
 	Use common-directives tests.reproducible-builds.org
 	RedirectMatch permanent "^/((?!debian/repository/debian/).*)" https://tests.reproducible-builds.org/$1
 	DocumentRoot /var/lib/jenkins/userContent/reproducible
-	<Limit GET POST HEAD>
-	  Order Allow,Deny
-	  Allow from all
-	  Deny from env=bad_bot
-	</Limit>
 </VirtualHost>
 
 <VirtualHost *:443>
@@ -107,11 +127,6 @@ Use https-redirect www.diffoscope.org
 	DocumentRoot /var/www
 	AddDefaultCharset utf-8
 
-	<Limit GET POST HEAD>
-	  Order Allow,Deny
-	  Allow from all
-	  Deny from env=bad_bot
-	</Limit>
 
 	Alias /userContent /var/lib/jenkins/userContent
 	Alias /robots.txt /var/lib/jenkins/userContent/robots.txt
@@ -172,16 +187,15 @@ Use https-redirect www.diffoscope.org
 	DocumentRoot /var/lib/jenkins/userContent/reproducible
 	AddDefaultCharset utf-8
 
-	<Limit GET POST HEAD>
-	  Order Allow,Deny
-	  Allow from all
-	  Deny from env=bad_bot
-	</Limit>
-
 	Include reproduciblemap.conf
 
 	<Directory /var/lib/jenkins/userContent/reproducible/debian/artifacts>
 		HeaderName .HEADER.html
+		<Limit GET POST HEAD>
+		  Order Allow,Deny
+		  Allow from all
+		  Deny from env=bad_bot
+		</Limit>
 	</Directory>
 
 	# for watching service logfiles
@@ -190,6 +204,11 @@ Use https-redirect www.diffoscope.org
 	    AllowOverride None
 	    Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
 	    Require all granted
+		<Limit GET POST HEAD>
+		  Order Allow,Deny
+		  Allow from all
+		  Deny from env=bad_bot
+		</Limit>
 	</Directory>
 	# Use the sso.debian.org CA to validate client certificates
 	# Keep these files up to date with update-debsso-ca



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/c2549849f3bc8ff19f4aeca3d727bdb1a2711495

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/c2549849f3bc8ff19f4aeca3d727bdb1a2711495
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20181030/936a2b83/attachment-0001.html>