[Git][qa/jenkins.debian.net][master] jenkins: do not allow any bot at all into cgi-bin
Mattia Rizzolo (@mattia)
gitlab at salsa.debian.org
Fri Apr 17 10:45:21 BST 2026
Mattia Rizzolo pushed to branch master at Debian QA / jenkins.debian.net
Commits:
0a6a3f26 by Mattia Rizzolo at 2026-04-17T11:45:05+02:00
jenkins: do not allow any bot at all into cgi-bin
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
3 changed files:
- + hosts/jenkins/etc/apache2/jenkins.debian.net.conf
- hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf
- + userContent/reproducible/robots.txt
Changes:
=====================================
hosts/jenkins/etc/apache2/jenkins.debian.net.conf
=====================================
@@ -0,0 +1,313 @@
+# vim: set noexpandtab:
+
+<Macro https-redirect $name>
+ <VirtualHost *:80>
+ ServerName $name
+ ServerAdmin holger at layer-acht.org
+ Redirect permanent / https://$name/
+ </VirtualHost>
+</Macro>
+
+<Macro common-directives-ssl $name>
+ SSLEngine on
+ SSLCertificateKeyFile /var/lib/dehydrated/certs/$name/privkey.pem
+ SSLCertificateFile /var/lib/dehydrated/certs/$name/fullchain.pem
+ SSLCipherSuite HIGH:!aNULL:!eNULL:!EXP:!LOW:!MD5
+ SSLHonorCipherOrder on
+ Header always add Strict-Transport-Security "max-age=15552000"
+</Macro>
+
+<Macro block-bots>
+ #block bad bots with a 403
+ SetEnvIf User-Agent "AhrefsBot" bad_bot
+ SetEnvIf User-Agent "Amazonbot" bad_bot
+ SetEnvIf User-Agent "Baiduspider" bad_bot
+ SetEnvIf User-Agent "bingbot" bad_bot
+ SetEnvIf User-Agent "DotBot" bad_bot
+ SetEnvIf User-Agent "Exabot" bad_bot
+ SetEnvIf User-Agent "Experibot" bad_bot
+ SetEnvIf User-Agent "facebookexternalhit" bad_bot
+ SetEnvIf User-Agent "FlipboardProxy" bad_bot
+ SetEnvIf User-Agent "Googlebot" bad_bot
+ SetEnvIf User-Agent "GoogleOther" bad_bot
+ SetEnvIf User-Agent "GPTBot" bad_bot
+ SetEnvIf User-Agent "ltx71" bad_bot
+ SetEnvIf User-Agent "mediawords" bad_bot
+ SetEnvIf User-Agent "MetaURI" bad_bot
+ SetEnvIf User-Agent "MJ12bot" bad_bot
+ SetEnvIf User-Agent "SemrushBot" bad_bot
+ SetEnvIf User-Agent "Slackbot" bad_bot
+ SetEnvIf User-Agent "Sogou" bad_bot
+ SetEnvIf User-Agent "Twitterbot" bad_bot
+ SetEnvIf User-Agent "yacybot" bad_bot
+ SetEnvIf User-Agent "ZoomBot" bad_bot
+</Macro>
+
+<Macro common-directives $name>
+ ServerName $name
+ ServerAdmin holger at layer-acht.org
+
+ <Directory />
+ Options FollowSymLinks
+ AllowOverride None
+ <Limit GET POST HEAD>
+ Order Allow,Deny
+ Allow from all
+ Deny from env=bad_bot
+ </Limit>
+ </Directory>
+ <Directory /var/www/>
+ Options Indexes FollowSymLinks MultiViews
+ AllowOverride None
+ Require all granted
+ AddType text/plain .log
+ <Limit GET POST HEAD>
+ Order Allow,Deny
+ Allow from all
+ Deny from env=bad_bot
+ </Limit>
+ </Directory>
+ <Directory /var/lib/jenkins/userContent>
+ Options Indexes FollowSymLinks MultiViews
+ AllowOverride None
+ Require all granted
+ AddType text/plain .log
+ <Limit GET POST HEAD>
+ Order Allow,Deny
+ Allow from all
+ Deny from env=bad_bot
+ </Limit>
+ </Directory>
+ <Directory /srv/reproducible-builds.org/lfs>
+ Options Indexes FollowSymlinks MultiViews
+ AllowOverride None
+ Require all granted
+ <Limit GET POST HEAD>
+ Order Allow,Deny
+ Allow from all
+ Deny from env=bad_bot
+ </Limit>
+ </Directory>
+
+ <FilesMatch "\.gz$">
+ AddEncoding gzip .gz
+ ForceType text/plain
+ </FilesMatch>
+
+ RewriteEngine on
+ ProxyRequests Off
+
+ ErrorLog ${APACHE_LOG_DIR}/error.log
+ # Possible values include: debug, info, notice, warn, error, crit,
+ # alert, emerg.
+ LogLevel warn
+ CustomLog ${APACHE_LOG_DIR}/access.log combined
+</Macro>
+
+<Macro r-b-artifacts $base>
+ <Directory /var/lib/jenkins/userContent/reproducible/$base/artifacts>
+ HeaderName .HEADER.html
+ <Limit GET POST HEAD>
+ Order Allow,Deny
+ Allow from all
+ Deny from env=bad_bot
+ </Limit>
+ </Directory>
+</Macro>
+
+Use https-redirect jenkins.debian.net
+Use https-redirect reproducible.debian.net
+Use https-redirect reproducible-builds.org
+Use https-redirect www.reproducible-builds.org
+Use https-redirect diffoscope.org
+Use https-redirect www.diffoscope.org
+
+<VirtualHost *:80>
+ Use common-directives tests.reproducible-builds.org
+ RedirectMatch permanent "^/((?!debian/repository/debian/).*)" https://tests.reproducible-builds.org/$1
+ DocumentRoot /var/lib/jenkins/userContent/reproducible
+</VirtualHost>
+
+<VirtualHost *:443>
+ Use common-directives diffoscope.org
+ Use common-directives-ssl diffoscope.org
+ DocumentRoot /srv/diffoscope.org/www
+ AddDefaultCharset utf-8
+
+ Alias /archive /srv/reproducible-builds.org/lfs/releases/diffoscope
+ <Directory /srv/diffoscope.org/www>
+ AllowOverride None
+ Require all granted
+ </Directory>
+</VirtualHost>
+
+<VirtualHost *:443>
+ Use common-directives jenkins.debian.net
+ Use common-directives-ssl jenkins.debian.net
+ Use block-bots
+
+ DocumentRoot /var/www
+ AddDefaultCharset utf-8
+
+
+ Alias /userContent /var/lib/jenkins/userContent
+ Alias /robots.txt /var/lib/jenkins/userContent/robots.txt
+
+ # a bunch of redirects to point people to https://reproducible.debian.net
+ RewriteCond %{REQUEST_URI} ^/userContent/reproducible.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/reproducible.json$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_issues.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_notes.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_schedule.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_last_24h.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_last_48h.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_all_abc.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_dd-list.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_stats.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_pkg_sets.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_reproducible.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_FTBR_with_buildinfo.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_FTBR.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_FTBFS.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_404.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_not_for_us.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/index_blacklisted.html$ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/rb-pkg/ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/buildinfo/ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/dbd/ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/issues/ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/notes/ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/artifacts/ [or]
+ RewriteCond %{REQUEST_URI} ^/userContent/rbuild/
+ RewriteRule ^/userContent/(.*) https://reproducible.debian.net/debian/$1 [R=301,L]
+
+ RequestHeader set X-Forwarded-Port "443"
+ RequestHeader set X-Forwarded-Proto "https"
+ <Proxy *>
+ Require all granted
+ </Proxy>
+ ProxyPreserveHost on
+ AllowEncodedSlashes NoDecode
+ # proxy everything but a few urls
+ ProxyPass /munin !
+ ProxyPass /munin-cgi !
+ ProxyPass /server-status !
+ # map /d-i-preseed-cfgs to /UserContent/d-i-preseed-cfgs
+ ProxyPass /d-i-preseed-cfgs/ http://localhost:8080/userContent/d-i-preseed-cfgs/
+ ProxyPass /userContent !
+ ProxyPass /robots.txt !
+ ProxyPass /cli !
+ ProxyPass / http://localhost:8080/ nocanon
+ ProxyPassReverse / http://localhost:8080/
+</VirtualHost>
+
+
+<VirtualHost *:443>
+ Use common-directives tests.reproducible-builds.org
+ Use common-directives-ssl tests.reproducible-builds.org
+ Use block-bots
+
+ DocumentRoot /var/lib/jenkins/userContent/reproducible
+ AddDefaultCharset utf-8
+
+ Include reproduciblemap.conf
+
+ Use r-b-artifacts debian
+ Use r-b-artifacts debian/live_build
+
+ # for watching service logfiles
+ ScriptAlias /cgi-bin /srv/jenkins/bin/cgi-bin
+ <Directory "/srv/jenkins/bin/cgi-bin">
+ AllowOverride None
+ Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
+ Require all granted
+ <Limit GET POST HEAD>
+ Order Allow,Deny
+ Allow from all
+ Deny from env=bad_bot
+ </Limit>
+ </Directory>
+ # Use the sso.debian.org CA to validate client certificates
+ # Keep these files up to date with update-debsso-ca
+ SSLCACertificateFile /etc/apache2/ssl/debsso/debsso.crt
+ SSLCARevocationCheck chain
+ SSLCARevocationFile /etc/apache2/ssl/debsso/debsso.crl
+
+ # Alternative with salsa.debian.org OpenIDc
+ OIDCProviderMetadataURL https://salsa.debian.org/.well-known/openid-configuration
+ OIDCRedirectURI https://tests.reproducible-builds.org/auth/secure
+ Include oidc_secrets.conf
+
+ <Location /auth>
+ AuthType openid-connect
+ Require valid-user
+ </Location>
+ ScriptAlias /auth/schedule /srv/jenkins/bin/cgi-bin/schedule
+ <Location /cgi-bin/schedule>
+ # Export data about the certificate to the environment
+ SSLOptions +StdEnvVars
+ # Allow access if one does not have a valid certificate,
+ # so we can show a decent error message
+ SSLVerifyClient optional
+ </Location>
+ Redirect /cgi-bin/schedule /auth/schedule
+
+ <Proxy *>
+ Require all granted
+ </Proxy>
+
+</VirtualHost>
+
+
+<VirtualHost *:443>
+ Use common-directives reproducible-builds.org
+ Use common-directives-ssl reproducible-builds.org
+
+ DocumentRoot /srv/reproducible-builds.org/www
+ AddDefaultCharset utf-8
+
+ Alias /website.git /srv/reproducible-builds.org/git/website.git
+ Alias /specs /var/lib/jenkins/userContent/reproducible/specs
+ Alias /style /srv/reproducible-builds.org/style
+ Alias /_lfs /srv/reproducible-builds.org/lfs
+
+ RewriteEngine on
+ RewriteRule /howto($|/.*) /docs/ [R=permanent]
+
+ <Directory /srv/reproducible-builds.org/www>
+ AllowOverride None
+ Require all granted
+ </Directory>
+ <Directory /srv/reproducible-builds.org/style>
+ AllowOverride None
+ Require all granted
+ </Directory>
+ <Directory /srv/reproducible-builds.org/git>
+ Options Indexes
+ AllowOverride None
+ Require all granted
+ </Directory>
+</VirtualHost>
+
+
+<VirtualHost *:443>
+ Use common-directives www.reproducible-builds.org
+ Use common-directives-ssl www.reproducible-builds.org
+
+ # just redirect everything to non-www
+ Redirect permanent / https://reproducible-builds.org/
+</VirtualHost>
+<VirtualHost *:443>
+ Use common-directives www.diffoscope.org
+ Use common-directives-ssl www.diffoscope.org
+
+ # just redirect everything to non-www
+ Redirect permanent / https://diffoscope.org/
+</VirtualHost>
+<VirtualHost *:443>
+ Use common-directives reproducible.debian.net
+ Use common-directives-ssl reproducible.debian.net
+
+ # just redirect everything to the new hostname
+ Redirect permanent / https://tests.reproducible-builds.org/
+</VirtualHost>
=====================================
hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf
=====================================
@@ -227,6 +227,7 @@ Use https-redirect www.diffoscope.org
<Directory "/srv/jenkins/bin/cgi-bin">
AllowOverride None
Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
+ BrowserMatchNoCase "bot" bad_bot
<RequireAll>
Require all granted
Require not env bad_bot
=====================================
userContent/reproducible/robots.txt
=====================================
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /cgi-bin/
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/0a6a3f26e93115f613ed77b177dc081d62857934
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/0a6a3f26e93115f613ed77b177dc081d62857934
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20260417/a824d504/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list