[Git][qa/jenkins.debian.net][master] reproduce.debian.net: setup another instance on o4 and update installation docs
Holger Levsen (@holger)
gitlab at salsa.debian.org
Sat Nov 30 17:36:30 GMT 2024
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
5bfdbaa3 by Holger Levsen at 2024-11-30T18:36:14+01:00
reproduce.debian.net: setup another instance on o4 and update installation docs
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
7 changed files:
- hosts/osuosl4-amd64/etc/nginx/nginx.conf
- + hosts/osuosl4-amd64/opt/etc/rebuilderd-sync.conf
- + hosts/osuosl4-amd64/opt/etc/rebuilderd-worker.conf
- + hosts/osuosl4-amd64/opt/usr/libexec/rebuilderd/rebuilder-debian.sh
- + hosts/osuosl4-amd64/var/www/html/index.html
- hosts/osuosl5-amd64/etc/nginx/nginx.conf
- hosts/osuosl5-amd64/var/www/html/rebuilderd-setup.html
Changes:
=====================================
hosts/osuosl4-amd64/etc/nginx/nginx.conf
=====================================
@@ -13,35 +13,59 @@ http {
server_tokens off;
server {
- listen 80;
- server_name localhost;
+ server_name osuosl4-amd64.debian.net;
+ index index.html index.htm
+ root /var/www/html;
+
+ listen 443 ssl;
+ ssl_certificate /var/lib/dehydrated/certs/osuosl4-amd64.debian.net/fullchain.pem;
+ ssl_certificate_key /var/lib/dehydrated/certs/osuosl4-amd64.debian.net/privkey.pem;
+
+ ssl_session_cache shared:le_nginx_SSL:10m;
+ ssl_session_timeout 1440m;
+ ssl_session_tickets off;
+ ssl_protocols TLSv1.2 TLSv1.3;
+ ssl_prefer_server_ciphers off;
+ ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-C
+ HACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384";
+
- location /snapshot {
- alias /snapshot;
- autoindex on;
+ location /.well-known/acme-challenge/ {
+ alias /var/lib/dehydrated/acme-challenges/;
+ disable_symlinks off;
+ autoindex off;
}
- location /snapshot/mr {
- include uwsgi_params;
- rewrite /snapshot/(.+) /$1 break;
- uwsgi_pass unix:/var/run/snapshot/snapshot.sock;
+ location / {
+ # First attempt to serve request as file, then
+ # as directory, then fall back to displaying a 404.
+ try_files $uri $uri/ =404;
}
- location /snapshot-experiment {
- alias /srv/data/vagrant/snapshots;
- autoindex on;
+
+
+ location /api/ {
+ proxy_pass http://127.0.0.1:8484;
}
+ }
-location ~ /rebuilder-snapshot/api(.*) {
- proxy_pass http://127.0.0.1:5001/$1;
- include proxy_params;
-}
-location /rebuilder-snapshot/ {
- alias /srv/data/rebuilder-snapshot/;
- autoindex on;
-}
+ server {
+ listen 80;
+ server_name osuosl4-amd64.debian.net;
+
+ location /.well-known/acme-challenge/ {
+ alias /var/lib/dehydrated/acme-challenges/;
+ disable_symlinks off;
+ autoindex off;
+ }
+ location / {
+ return 301 https://$host$request_uri;
+ }
+ return 404;
}
+
+
}
=====================================
hosts/osuosl4-amd64/opt/etc/rebuilderd-sync.conf
=====================================
@@ -0,0 +1,53 @@
+## rebuild all of archlinux core
+[profile."archlinux-core"]
+distro = "archlinux"
+suite = "core"
+architectures = ["x86_64"]
+source = "https://ftp.halifax.rwth-aachen.de/archlinux/$repo/os/$arch"
+
+## rebuild community packages of specific maintainers, or allow-list packages by name.
+## If no filter is set, all packages are imported, if both filters are set the package only
+## has to match one of them to be included.
+#[profile."archlinux-community"]
+#distro = "archlinux"
+#suite = "community"
+#architectures = ["x86_64"]
+#source = "https://ftp.halifax.rwth-aachen.de/archlinux/$repo/os/$arch"
+#maintainers = ["somebody"]
+#pkgs = ["some-pkg", "python-*"]
+#excludes = ["tensorflow*"]
+
+[profile."debian-main"]
+distro = "debian"
+suite = "main"
+architectures = ["amd64"]
+#releases = ["buster", "sid"]
+releases = ["trixie"]
+source = "http://deb.debian.org/debian"
+
+[profile."debian-anarchism"]
+distro = "debian"
+suite = "main"
+architectures = ["amd64"]
+releases = ["sid"]
+pkgs = ["anarchism"]
+source = "http://deb.debian.org/debian"
+
+# a set of packages to test certain edge-cases
+[profile."debian-misc"]
+distro = "debian"
+suite = "main"
+architectures = ["amd64"]
+releases = ["sid"]
+pkgs = ["anarchism", "binutils-arm-none-eabi", "libglib2.0-bin", "libglib2.0-dev", "sniffglue", "librust-sniffglue-dev", "dfrs", "librust-dfrs-dev"]
+source = "http://deb.debian.org/debian"
+
+[profile."tails"]
+distro = "tails"
+suite = "stable"
+source = "https://mirrors.wikimedia.org/tails/"
+
+[profile."tails-alpha"]
+distro = "tails"
+suite = "alpha"
+source = "https://mirrors.wikimedia.org/tails/"
=====================================
hosts/osuosl4-amd64/opt/etc/rebuilderd-worker.conf
=====================================
@@ -0,0 +1,33 @@
+## The rebuilderd to connect to
+endpoint = "http://127.0.0.1:8484"
+## The server would either allowlist our key or require a signup secret
+#signup_secret = "your_signup_key"
+
+[build]
+#timeout = 86400 # 24 hours
+## Set a maximum build log limit in bytes (default: none).
+## When reaching this limit the log is truncated but the rebuilder backend is *not* terminated.
+max_bytes = 10485760 # 10 MiB
+## By default build output is forwarded to stdout/stderr.
+## This can be disabled by settings this to true.
+#silent = true
+
+[diffoscope]
+## Generate and attach diffs with diffoscope when rebuilding
+enabled = false
+## Pass additional arguments to diffoscope. Use wisely, some options might not work well.
+#args = ["--max-container-depth", "2", "--fuzzy-threshold", "0"]
+## Set a timeout in seconds after which diffoscope is terminated (default: 3600)
+#timeout = 600 # 10 minutes
+## Set a maximum diffoscope output limit in bytes (default: none).
+## When reaching this limit, diffoscope is terminated and the output is truncated.
+max_bytes = 41943040 # 40 MiB
+
+[backend."archlinux"]
+path = "/usr/libexec/rebuilderd/rebuilder-archlinux.sh"
+
+[backend."debian"]
+path = "/opt/usr/libexec/rebuilderd/rebuilder-debian.sh"
+
+[backend."tails"]
+path = "/usr/libexec/rebuilderd/rebuilder-tails.sh"
=====================================
hosts/osuosl4-amd64/opt/usr/libexec/rebuilderd/rebuilder-debian.sh
=====================================
@@ -0,0 +1,11 @@
+#!/bin/sh
+set -xe
+cd "$(dirname "$1")"
+
+mkdir -p etc/apt
+mkdir -p var/lib/apt/lists/
+echo 'deb-src [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] https://deb.debian.org/debian trixie main' > etc/apt/sources.list
+apt-get -o Dir=. update
+apt-get -o Dir=. source --download-only "$(basename "$1" | cut -d_ -f1)"
+
+/usr/bin/debrebuild --buildresult="${REBUILDERD_OUTDIR}" --builder=sbuild+unshare --cache=/srv/rebuilderd/cache -- "${1}"
=====================================
hosts/osuosl4-amd64/var/www/html/index.html
=====================================
@@ -0,0 +1,197 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>
+ osuosl4-amd64.debian.net - trixie/amd64
+ </title>
+ <style>
+ body {
+ font-family: sans-serif;
+ }
+ h1 {
+ font-size: 24px;
+ }
+
+ #search-results {
+ margin: 20px 0;
+ }
+
+ pre {
+ margin: 0;
+ }
+
+ #search-results pre {
+ margin: 12px 0;
+ }
+
+ .status {
+ font-weight: bold;
+ }
+
+ .good {
+ color: green;
+ }
+
+ .bad {
+ color: red;
+ }
+
+ .unknown {
+ color: #957fff;
+ }
+
+ footer {
+ font-size: small;
+ margin: 30px 0;
+ }
+ </style>
+ <script>
+ document.addEventListener('DOMContentLoaded', function() {
+ // package search
+ function spanWith(text) {
+ let s = document.createElement('span');
+ s.textContent = text;
+ return s;
+ }
+
+ function linkTo(href, text) {
+ let a = document.createElement('a');
+ a.href = href;
+ a.textContent = text;
+ return a;
+ }
+
+ function searchPkgs(query) {
+ let div = document.getElementById('search-results');
+ let url = '/api/v0/pkgs/list?' + new URLSearchParams({
+ name: query,
+ distro: 'debian',
+ });
+ fetch(url)
+ .then(response => response.json())
+ .then(data => {
+ // clear children
+ div.innerHTML = '';
+
+ data.map(pkg => {
+ let build_id = pkg.build_id;
+ let r = document.createElement('pre');
+
+ r.appendChild(spanWith('['));
+ let status = pkg['status'];
+ let statusSpan = spanWith(status);
+ statusSpan.className += ' status';
+ if (status == 'GOOD') {
+ statusSpan.className += ' good';
+ } else if (status == 'BAD') {
+ statusSpan.className += ' bad';
+ } else {
+ statusSpan.className += ' unknown';
+ }
+ r.appendChild(statusSpan);
+ r.appendChild(spanWith(`] ${pkg['name']} ${pkg['version']}\t\t`));
+
+ if (build_id) {
+ r.appendChild(spanWith(' ['));
+ r.appendChild(linkTo(`/api/v0/builds/${build_id}/log`, 'log'));
+ r.appendChild(spanWith(']'));
+ }
+
+ if (pkg.has_attestation) {
+ r.appendChild(spanWith(' ['));
+ r.appendChild(linkTo(`/api/v0/builds/${build_id}/attestation`, 'attestation'));
+ r.appendChild(spanWith(']'));
+ }
+
+ if (pkg.has_diffoscope) {
+ r.appendChild(spanWith(' ['));
+ r.appendChild(linkTo(`/api/v0/builds/${build_id}/diffoscope`, 'diffoscope'));
+ r.appendChild(spanWith(']'));
+ }
+
+ div.appendChild(r);
+ });
+ });
+ }
+
+ if (location.hash) {
+ searchPkgs(location.hash.substr(1));
+ }
+
+ document.getElementById('search').addEventListener('submit', function(e) {
+ e.preventDefault();
+ let query = document.getElementById('search-query').value;
+ location.href = `#${query}`;
+ searchPkgs(query);
+ });
+
+ // display stats
+ function updateStats(data) {
+ let div = document.getElementById('stats');
+ // clear children
+ div.innerHTML = '';
+ // add rows
+ data.map(row => {
+ let r = document.createElement('pre');
+ let key = row[0] + ': ';
+ r.textContent = key.padEnd(20) + row[1];
+ div.appendChild(r);
+ });
+ }
+
+ function fetchStats() {
+ fetch('/api/v0/dashboard')
+ .then(response => response.json())
+ .then(data => {
+ let div = document.getElementById('stats');
+ let main = data['suites']['main'];
+
+ let good = main['good'];
+ let bad = main['bad'];
+ let unknown = main['unknown'];
+
+ updateStats([
+ ['good', good],
+ ['bad', bad],
+ ['unknown', unknown],
+ ['build progress', (100 / (good + unknown + bad) * (good + bad)).toFixed(2) + '%'],
+ ['repro', (100 / (good + bad) * good).toFixed(2) + '%'],
+ ]);
+ });
+ }
+
+ setInterval(fetchStats, 60 * 1000);
+ fetchStats();
+ });
+ </script>
+ </head>
+ <body>
+ <h1>
+ osuosl4-amd64.debian.net
+ </h1>
+ <h3>
+ trixie/amd64
+ </h3>
+
+ <div id="stats">Loading stats...</div>
+
+ <div>
+ <h3>Search</h3>
+ <form id="search">
+ <input type="text" id="search-query" placeholder="2ping">
+ <input type="submit" value="Search">
+ </form>
+ </div>
+ <div id="search-results">
+ </div>
+
+ <footer>
+ pew pew, <a href="https://github.com/kpcyrd/rebuilderd">rebuilderd</a> using <a href="https://tracker.debian.org/pkg/devscripts">debrebuild</a> to reproduce what Debian distributes via <code>deb.debian.org</code> ♥️
+ </br>
+ <!-- a href="/stats/">Statistics about BAD packages.</a> -->
+ </br>
+ <small>This setup is in it's infancy. (= don't take numbers seriously unless...!?)</small>
+ </footer>
+ </body>
+</html>
=====================================
hosts/osuosl5-amd64/etc/nginx/nginx.conf
=====================================
@@ -14,7 +14,7 @@ http {
server {
server_name reproduce.debian.net;
- index index.html index.htm index.nginx-debian.html;
+ index index.html index.htm
root /var/www/html;
listen 443 ssl;
=====================================
hosts/osuosl5-amd64/var/www/html/rebuilderd-setup.html
=====================================
@@ -25,15 +25,28 @@
</h1>
<pre>
-(version 0.1, based on previous notes but as such untested. feedback much welcome.)
+(version 0.2, feedback much welcome.)
USER=rebuilderd
+# sudo adduser $USER
REBUILDERD_HOSTNAME=rebuilderd.example.org
# to get a working debrebuild (from devscripts) and sbuild with unshare backend
sudo apt install devscripts sbuild uidmap
+sudo apt install apt-utils dpkg-dev mmdebstrap
# todo: check if these are really needed
-sudo apt install apt-utils dpkg-dev equivs mmdebstrap python3-pycurl libwww-perl libjson-perl wget libstring-shellquote-perl python3-requests libwww-perl libjson-perl python3-debian debian-keyring debootstrap libarchive13t64 qemu-user-static genext2fs ncompress qemu-user squashfs-tools-ng arch-test fakechroot libdistro-info-perl
+# apt install equivs python3-pycurl libwww-perl libjson-perl wget libstring-shellquote-perl python3-requests libwww-perl libjson-perl python3-debian debian-keyring debootstrap libarchive13t64 qemu-user-static genext2fs ncompress qemu-user squashfs-tools-ng arch-test fakechroot libdistro-info-perl
+
+#
+# the filesystem layout used was
+#
+# sudo vcreate -L 1TiB rebuilderd
+# sudo lvcreate -L 1T lvm -n tmp
+# sudo mkfs.ext4 /dev/mapper/lvm-tmp
+# sudo mkfs.ext4 /dev/mapper/lvm-rebuilderd
+# echo "/dev/mapper/lvm-tmp /tmp ext4 defaults,noatime,mode=1777 0 0" | sudo tee -a /etc/fstab
+# echo "/dev/mapper/lvm-rebuilderd /srv/rebuilderd ext4 defaults 0 1" | sudo tee -a /etc/fstab
+# sudo mount /srv/rebuilderd
sudo mkdir -p /srv/rebuilderd/cache
sudo mkdir -p /srv/rebuilderd/results
@@ -41,7 +54,7 @@ sudo chown -R $USER:$USER /srv/rebuilderd
# make sure $USER is mentioned in /etc/sub(uid|gid)
# (this is only the case for installations done with bookworm or later)
-for i in uid git ; do
+for i in uid gid ; do
grep $USER /etc/sub$i || echo $USER must be added to /etc/sub$i manually
done
@@ -69,22 +82,23 @@ apt-get -o Dir=. update
apt-get -o Dir=. source --download-only "$(basename "$1" | cut -d_ -f1)"
/usr/bin/debrebuild --buildresult="${REBUILDERD_OUTDIR}" --builder=sbuild+unshare --cache=/srv/rebuilderd/cache -- "${1}"
+# Note: the distro, eg trixie, has to be defined in /opt/usr/libexec/rebuilderd/rebuilder-debian.sh
+# and /opt/etc/rebuilderd-sync.conf
+
#
# FIXME: this should be done using services
#
# then start rebuilderd as $USER in screen:
/opt/usr/bin/rebuilderd -c /opt/etc/rebuilderd.conf
-# then as $USER tell it to schedule some work:
-env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8484 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync.conf debian-main
-# then start two workers as $USER in screen:
+# run this in screen as $USER to update the list of packages frequently
+while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8484 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync.conf debian-main ; echo sleeping 3h now ; sleep 3h ; done
+# finally start some workers as $USER in screen:
cd
mkdir 1 2
cd 1
/opt/usr/bin/rebuilderd-worker -c /opt/etc/rebuilderd-worker.conf connect
cd 2
/opt/usr/bin/rebuilderd-worker -c /opt/etc/rebuilderd-worker.conf connect
-# run this in screen to update the list of packages frequently
-while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8484 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync.conf debian-main ; echo sleeping 3h now ; sleep 3h ; done
#
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/5bfdbaa3380d5bfa035ec9d521e221c8ee8f361f
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/5bfdbaa3380d5bfa035ec9d521e221c8ee8f361f
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20241130/26643964/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list