[Git][qa/jenkins.debian.net][master] reproduce.debian.net: setup another instance on o4 and update installation docs

Holger Levsen (@holger) gitlab at salsa.debian.org
Sat Nov 30 17:36:30 GMT 2024



Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
5bfdbaa3 by Holger Levsen at 2024-11-30T18:36:14+01:00
reproduce.debian.net: setup another instance on o4 and update installation docs

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


7 changed files:

- hosts/osuosl4-amd64/etc/nginx/nginx.conf
- + hosts/osuosl4-amd64/opt/etc/rebuilderd-sync.conf
- + hosts/osuosl4-amd64/opt/etc/rebuilderd-worker.conf
- + hosts/osuosl4-amd64/opt/usr/libexec/rebuilderd/rebuilder-debian.sh
- + hosts/osuosl4-amd64/var/www/html/index.html
- hosts/osuosl5-amd64/etc/nginx/nginx.conf
- hosts/osuosl5-amd64/var/www/html/rebuilderd-setup.html


Changes:

=====================================
hosts/osuosl4-amd64/etc/nginx/nginx.conf
=====================================
@@ -13,35 +13,59 @@ http {
     server_tokens       off;
 
     server {
-        listen          80;
-        server_name     localhost;
+        server_name     osuosl4-amd64.debian.net;
+        index index.html index.htm
+        root /var/www/html;
+
+        listen 443 ssl;
+        ssl_certificate /var/lib/dehydrated/certs/osuosl4-amd64.debian.net/fullchain.pem;
+        ssl_certificate_key /var/lib/dehydrated/certs/osuosl4-amd64.debian.net/privkey.pem;
+
+        ssl_session_cache shared:le_nginx_SSL:10m;
+        ssl_session_timeout 1440m;
+        ssl_session_tickets off;
+        ssl_protocols TLSv1.2 TLSv1.3;
+        ssl_prefer_server_ciphers off;
+        ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-C
+        HACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384";
+
 
-        location /snapshot {
-            alias       /snapshot;
-            autoindex   on;
+        location /.well-known/acme-challenge/ {
+            alias /var/lib/dehydrated/acme-challenges/;
+            disable_symlinks off;
+            autoindex off;
         }
 
-        location /snapshot/mr {
-            include     uwsgi_params;
-            rewrite     /snapshot/(.+) /$1 break;
-            uwsgi_pass  unix:/var/run/snapshot/snapshot.sock;
+        location / {
+            # First attempt to serve request as file, then
+            # as directory, then fall back to displaying a 404.
+            try_files $uri $uri/ =404;
         }
-        location /snapshot-experiment {
-            alias       /srv/data/vagrant/snapshots;
-            autoindex   on;
+
+
+        location /api/ {
+            proxy_pass http://127.0.0.1:8484;
         }
 
+    }
 
-location ~ /rebuilder-snapshot/api(.*) {
-	proxy_pass  	http://127.0.0.1:5001/$1;
-	include		proxy_params;
-}
 
-location 		/rebuilder-snapshot/ {
-	alias		/srv/data/rebuilder-snapshot/;
-	autoindex	on;
-}
+    server {
+        listen          80;
+        server_name     osuosl4-amd64.debian.net;
+
+        location /.well-known/acme-challenge/ {
+            alias /var/lib/dehydrated/acme-challenges/;
+            disable_symlinks off;
+            autoindex off;
+        }
 
+        location / {
+            return 301 https://$host$request_uri;
+        }
 
+        return 404;
     }
+
+
 }


=====================================
hosts/osuosl4-amd64/opt/etc/rebuilderd-sync.conf
=====================================
@@ -0,0 +1,53 @@
+## rebuild all of archlinux core
+[profile."archlinux-core"]
+distro = "archlinux"
+suite = "core"
+architectures = ["x86_64"]
+source = "https://ftp.halifax.rwth-aachen.de/archlinux/$repo/os/$arch"
+
+## rebuild community packages of specific maintainers, or allow-list packages by name.
+## If no filter is set, all packages are imported, if both filters are set the package only
+## has to match one of them to be included.
+#[profile."archlinux-community"]
+#distro = "archlinux"
+#suite = "community"
+#architectures = ["x86_64"]
+#source = "https://ftp.halifax.rwth-aachen.de/archlinux/$repo/os/$arch"
+#maintainers = ["somebody"]
+#pkgs = ["some-pkg", "python-*"]
+#excludes = ["tensorflow*"]
+
+[profile."debian-main"]
+distro = "debian"
+suite = "main"
+architectures = ["amd64"]
+#releases = ["buster", "sid"]
+releases = ["trixie"]
+source = "http://deb.debian.org/debian"
+
+[profile."debian-anarchism"]
+distro = "debian"
+suite = "main"
+architectures = ["amd64"]
+releases = ["sid"]
+pkgs = ["anarchism"]
+source = "http://deb.debian.org/debian"
+
+# a set of packages to test certain edge-cases
+[profile."debian-misc"]
+distro = "debian"
+suite = "main"
+architectures = ["amd64"]
+releases = ["sid"]
+pkgs = ["anarchism", "binutils-arm-none-eabi", "libglib2.0-bin", "libglib2.0-dev", "sniffglue", "librust-sniffglue-dev", "dfrs", "librust-dfrs-dev"]
+source = "http://deb.debian.org/debian"
+
+[profile."tails"]
+distro = "tails"
+suite = "stable"
+source = "https://mirrors.wikimedia.org/tails/"
+
+[profile."tails-alpha"]
+distro = "tails"
+suite = "alpha"
+source = "https://mirrors.wikimedia.org/tails/"


=====================================
hosts/osuosl4-amd64/opt/etc/rebuilderd-worker.conf
=====================================
@@ -0,0 +1,33 @@
+## The rebuilderd to connect to
+endpoint = "http://127.0.0.1:8484"
+## The server would either allowlist our key or require a signup secret
+#signup_secret = "your_signup_key"
+
+[build]
+#timeout = 86400 # 24 hours
+## Set a maximum build log limit in bytes (default: none).
+## When reaching this limit the log is truncated but the rebuilder backend is *not* terminated.
+max_bytes = 10485760 # 10 MiB
+## By default build output is forwarded to stdout/stderr.
+## This can be disabled by settings this to true.
+#silent = true
+
+[diffoscope]
+## Generate and attach diffs with diffoscope when rebuilding
+enabled = false
+## Pass additional arguments to diffoscope. Use wisely, some options might not work well.
+#args = ["--max-container-depth", "2", "--fuzzy-threshold", "0"]
+## Set a timeout in seconds after which diffoscope is terminated (default: 3600)
+#timeout = 600 # 10 minutes
+## Set a maximum diffoscope output limit in bytes (default: none).
+## When reaching this limit, diffoscope is terminated and the output is truncated.
+max_bytes = 41943040 # 40 MiB
+
+[backend."archlinux"]
+path = "/usr/libexec/rebuilderd/rebuilder-archlinux.sh"
+
+[backend."debian"]
+path = "/opt/usr/libexec/rebuilderd/rebuilder-debian.sh"
+
+[backend."tails"]
+path = "/usr/libexec/rebuilderd/rebuilder-tails.sh"


=====================================
hosts/osuosl4-amd64/opt/usr/libexec/rebuilderd/rebuilder-debian.sh
=====================================
@@ -0,0 +1,11 @@
+#!/bin/sh
+set -xe
+cd "$(dirname "$1")"
+
+mkdir -p etc/apt
+mkdir -p var/lib/apt/lists/
+echo 'deb-src [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] https://deb.debian.org/debian trixie main' > etc/apt/sources.list
+apt-get -o Dir=. update
+apt-get -o Dir=. source --download-only "$(basename "$1" | cut -d_ -f1)"
+
+/usr/bin/debrebuild --buildresult="${REBUILDERD_OUTDIR}" --builder=sbuild+unshare --cache=/srv/rebuilderd/cache -- "${1}"


=====================================
hosts/osuosl4-amd64/var/www/html/index.html
=====================================
@@ -0,0 +1,197 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <title>
+		osuosl4-amd64.debian.net - trixie/amd64
+        </title>
+        <style>
+        body {
+            font-family: sans-serif;
+        }
+        h1 {
+            font-size: 24px;
+        }
+
+        #search-results {
+            margin: 20px 0;
+        }
+
+        pre {
+            margin: 0;
+        }
+
+        #search-results pre {
+            margin: 12px 0;
+        }
+
+        .status {
+            font-weight: bold;
+        }
+
+        .good {
+            color: green;
+        }
+
+        .bad {
+            color: red;
+        }
+
+        .unknown {
+            color: #957fff;
+        }
+
+        footer {
+            font-size: small;
+            margin: 30px 0;
+        }
+        </style>
+        <script>
+        document.addEventListener('DOMContentLoaded', function() {
+            // package search
+            function spanWith(text) {
+                let s = document.createElement('span');
+                s.textContent = text;
+                return s;
+            }
+
+            function linkTo(href, text) {
+                let a = document.createElement('a');
+                a.href = href;
+                a.textContent = text;
+                return a;
+            }
+
+            function searchPkgs(query) {
+                let div = document.getElementById('search-results');
+                let url = '/api/v0/pkgs/list?' + new URLSearchParams({
+                    name: query,
+                    distro: 'debian',
+                });
+                fetch(url)
+                    .then(response => response.json())
+                    .then(data => {
+                        // clear children
+                        div.innerHTML = '';
+
+                        data.map(pkg => {
+                            let build_id = pkg.build_id;
+                            let r = document.createElement('pre');
+
+                            r.appendChild(spanWith('['));
+                            let status = pkg['status'];
+                            let statusSpan = spanWith(status);
+                            statusSpan.className += ' status';
+                            if (status == 'GOOD') {
+                                statusSpan.className += ' good';
+                            } else if (status == 'BAD') {
+                                statusSpan.className += ' bad';
+                            } else {
+                                statusSpan.className += ' unknown';
+                            }
+                            r.appendChild(statusSpan);
+                            r.appendChild(spanWith(`] ${pkg['name']} ${pkg['version']}\t\t`));
+
+                            if (build_id) {
+                                r.appendChild(spanWith(' ['));
+                                r.appendChild(linkTo(`/api/v0/builds/${build_id}/log`, 'log'));
+                                r.appendChild(spanWith(']'));
+                            }
+
+                            if (pkg.has_attestation) {
+                                r.appendChild(spanWith(' ['));
+                                r.appendChild(linkTo(`/api/v0/builds/${build_id}/attestation`, 'attestation'));
+                                r.appendChild(spanWith(']'));
+                            }
+
+                            if (pkg.has_diffoscope) {
+                                r.appendChild(spanWith(' ['));
+                                r.appendChild(linkTo(`/api/v0/builds/${build_id}/diffoscope`, 'diffoscope'));
+                                r.appendChild(spanWith(']'));
+                            }
+
+                            div.appendChild(r);
+                        });
+                    });
+            }
+
+            if (location.hash) {
+                searchPkgs(location.hash.substr(1));
+            }
+
+            document.getElementById('search').addEventListener('submit', function(e) {
+                e.preventDefault();
+                let query = document.getElementById('search-query').value;
+                location.href = `#${query}`;
+                searchPkgs(query);
+            });
+
+            // display stats
+            function updateStats(data) {
+                let div = document.getElementById('stats');
+                // clear children
+                div.innerHTML = '';
+                // add rows
+                data.map(row => {
+                    let r = document.createElement('pre');
+                    let key = row[0] + ': ';
+                    r.textContent = key.padEnd(20) + row[1];
+                    div.appendChild(r);
+                });
+            }
+
+            function fetchStats() {
+                fetch('/api/v0/dashboard')
+                    .then(response => response.json())
+                    .then(data => {
+                        let div = document.getElementById('stats');
+                        let main = data['suites']['main'];
+
+                        let good = main['good'];
+                        let bad = main['bad'];
+                        let unknown = main['unknown'];
+
+                        updateStats([
+                            ['good', good],
+                            ['bad', bad],
+                            ['unknown', unknown],
+                            ['build progress', (100 / (good + unknown + bad) * (good + bad)).toFixed(2) + '%'],
+                            ['repro', (100 / (good + bad) * good).toFixed(2) + '%'],
+                        ]);
+                    });
+            }
+
+            setInterval(fetchStats, 60 * 1000);
+            fetchStats();
+        });
+        </script>
+    </head>
+    <body>
+        <h1>
+		osuosl4-amd64.debian.net
+        </h1>
+        <h3>
+		trixie/amd64
+        </h3>
+
+        <div id="stats">Loading stats...</div>
+
+        <div>
+            <h3>Search</h3>
+            <form id="search">
+                <input type="text" id="search-query" placeholder="2ping">
+                <input type="submit" value="Search">
+            </form>
+        </div>
+        <div id="search-results">
+        </div>
+
+        <footer>
+            pew pew, <a href="https://github.com/kpcyrd/rebuilderd">rebuilderd</a> using <a href="https://tracker.debian.org/pkg/devscripts">debrebuild</a> to reproduce what Debian distributes via <code>deb.debian.org</code> ♥️
+	    </br>
+	    <!-- a href="/stats/">Statistics about BAD packages.</a> -->
+	    </br>
+	    <small>This setup is in it's infancy. (= don't take numbers seriously unless...!?)</small>
+        </footer>
+    </body>
+</html>


=====================================
hosts/osuosl5-amd64/etc/nginx/nginx.conf
=====================================
@@ -14,7 +14,7 @@ http {
 
     server {
         server_name     reproduce.debian.net;
-        index index.html index.htm index.nginx-debian.html;
+        index index.html index.htm
         root /var/www/html;
 
         listen 443 ssl;


=====================================
hosts/osuosl5-amd64/var/www/html/rebuilderd-setup.html
=====================================
@@ -25,15 +25,28 @@
         </h1>
 
 	    <pre>
-(version 0.1, based on previous notes but as such untested. feedback much welcome.)
+(version 0.2, feedback much welcome.)
 
 USER=rebuilderd
+# sudo adduser $USER
 REBUILDERD_HOSTNAME=rebuilderd.example.org
 
 # to get a working debrebuild (from devscripts) and sbuild with unshare backend
 sudo apt install devscripts sbuild uidmap
+sudo apt install apt-utils dpkg-dev mmdebstrap 
 # todo: check if these are really needed
-sudo apt install apt-utils dpkg-dev equivs mmdebstrap python3-pycurl libwww-perl libjson-perl wget libstring-shellquote-perl python3-requests libwww-perl libjson-perl python3-debian debian-keyring debootstrap libarchive13t64 qemu-user-static genext2fs ncompress qemu-user squashfs-tools-ng arch-test fakechroot libdistro-info-perl
+# apt install equivs python3-pycurl libwww-perl libjson-perl wget libstring-shellquote-perl python3-requests libwww-perl libjson-perl python3-debian debian-keyring debootstrap libarchive13t64 qemu-user-static genext2fs ncompress qemu-user squashfs-tools-ng arch-test fakechroot libdistro-info-perl
+
+# 
+# the filesystem layout used was
+#
+# sudo vcreate -L 1TiB rebuilderd
+# sudo lvcreate -L 1T lvm -n tmp
+# sudo mkfs.ext4 /dev/mapper/lvm-tmp
+# sudo mkfs.ext4 /dev/mapper/lvm-rebuilderd
+# echo "/dev/mapper/lvm-tmp /tmp ext4 defaults,noatime,mode=1777   0  0" | sudo tee -a /etc/fstab
+# echo "/dev/mapper/lvm-rebuilderd /srv/rebuilderd ext4 defaults 0 1" | sudo tee -a /etc/fstab
+# sudo mount /srv/rebuilderd
 
 sudo mkdir -p /srv/rebuilderd/cache
 sudo mkdir -p /srv/rebuilderd/results
@@ -41,7 +54,7 @@ sudo chown -R $USER:$USER /srv/rebuilderd
 
 # make sure $USER is mentioned in /etc/sub(uid|gid)
 # (this is only the case for installations done with bookworm or later)
-for i in uid git ; do
+for i in uid gid ; do
   grep $USER /etc/sub$i || echo $USER must be added to /etc/sub$i manually
 done
 
@@ -69,22 +82,23 @@ apt-get -o Dir=. update
 apt-get -o Dir=. source --download-only "$(basename "$1" | cut -d_ -f1)"
 /usr/bin/debrebuild --buildresult="${REBUILDERD_OUTDIR}" --builder=sbuild+unshare --cache=/srv/rebuilderd/cache -- "${1}"
 
+# Note: the distro, eg trixie, has to be defined in /opt/usr/libexec/rebuilderd/rebuilder-debian.sh
+#       and /opt/etc/rebuilderd-sync.conf
+
 #
 # FIXME: this should be done using services
 #
 # then start rebuilderd as $USER in screen:
 /opt/usr/bin/rebuilderd -c /opt/etc/rebuilderd.conf
-# then as $USER tell it to schedule some work:
-env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8484 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync.conf debian-main
-# then start two workers as $USER in screen:
+# run this in screen as $USER to update the list of packages frequently
+while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8484 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync.conf debian-main ; echo sleeping 3h now ; sleep 3h ; done
+# finally start some workers as $USER in screen:
 cd
 mkdir 1 2
 cd 1
 /opt/usr/bin/rebuilderd-worker -c /opt/etc/rebuilderd-worker.conf connect
 cd 2
 /opt/usr/bin/rebuilderd-worker -c /opt/etc/rebuilderd-worker.conf connect
-# run this in screen to update the list of packages frequently
-while true ; do env REBUILDERD_COOKIE_PATH=~/.local/share/rebuilderd-auth-cookie /opt/usr/bin/rebuildctl -H http://127.0.0.1:8484 pkgs sync-profile --sync-config /opt/etc/rebuilderd-sync.conf debian-main ; echo sleeping 3h now ; sleep 3h ; done
 
 
 #



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/5bfdbaa3380d5bfa035ec9d521e221c8ee8f361f

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/5bfdbaa3380d5bfa035ec9d521e221c8ee8f361f
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20241130/26643964/attachment-0001.htm>


More information about the Qa-jenkins-scm mailing list