[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] reproducible Debian: drop my rebuilder prototype as its not useful anymore
Holger Levsen (@holger)
gitlab at salsa.debian.org
Thu Sep 30 18:04:35 BST 2021
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
3814aa00 by Holger Levsen at 2021-09-30T19:04:15+02:00
reproducible Debian: drop my rebuilder prototype as its not useful anymore
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
8 changed files:
- TODO
- bin/common-functions.sh
- bin/jenkins_node_wrapper.sh
- − bin/rb-debrebuild
- − bin/reproducible_debian_rebuilder_prototype.sh
- bin/reproducible_html_nodes_info.sh
- bin/reproducible_maintenance.sh
- job-cfg/reproducible.yaml
Changes:
=====================================
TODO
=====================================
@@ -51,6 +51,8 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
=== Debian rebuilder 'thing'
+(these notes are probably mostly obsolete now.)
+
* prototype only dealing with bullseye and amd64 (and arch all packages)
** 1st step: download a specific .buildinfo file and sbuild it - done
** 2nd step: choose a random package from bullseye and fetch the .buildinfo file from builtin-pho and build it - done
=====================================
bin/common-functions.sh
=====================================
@@ -224,7 +224,7 @@ jenkins_zombie_check() {
# related however is that I issued a reboot (via running
# /sbin/reboot) right before the zombies appeared...
#
- ZOMBIES="$(ls -1d /var/lib/jenkins/jobs/* | egrep 'strip-nondeterminism|reprotest|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|chroot-installation_buster_.*_upgrade_to_sid|piuparts_.*_jessie|udd_stretch|d-i_pu-build|debsums-tests_stretch|debian-archive-keyring-tests_stretch|chroot-installation_jessie|chroot-installation_.*education-lang-|kirkwoot|rebootstrap_.*_gcc[5-8]($|_)|brcm47xx|rebootstrap_kfreebsd|diffoscope_from_git_|disorderfs_from_git_master|diffoscope_pypi|diffoscope_freebsd|diffoscope_netbsd|diffoscope_macports|diffoscope_archlinux|openwrt-target-ath97|profitbricks|pool_buildinfos_suites|^g-i|reproducible_compare_Debian_sha1sums|bbx15|cb3a|ff2a|ff2b|jtk1a|jtk1b|odxu4a|odxu4b|odu3a|opi2a|opi2c|p64b|p64c|ar71xx|reproducible_debian_live_build$|chroot-installation_stretch|chroot-installation_bullseye*upgrade_to_sid' || true)"
+ ZOMBIES="$(ls -1d /var/lib/jenkins/jobs/* | egrep 'strip-nondeterminism|reprotest|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|chroot-installation_buster_.*_upgrade_to_sid|piuparts_.*_jessie|udd_stretch|d-i_pu-build|debsums-tests_stretch|debian-archive-keyring-tests_stretch|chroot-installation_jessie|chroot-installation_.*education-lang-|kirkwoot|rebootstrap_.*_gcc[5-8]($|_)|brcm47xx|rebootstrap_kfreebsd|diffoscope_from_git_|disorderfs_from_git_master|diffoscope_pypi|diffoscope_freebsd|diffoscope_netbsd|diffoscope_macports|diffoscope_archlinux|openwrt-target-ath97|profitbricks|pool_buildinfos_suites|^g-i|reproducible_compare_Debian_sha1sums|bbx15|cb3a|ff2a|ff2b|jtk1a|jtk1b|odxu4a|odxu4b|odu3a|opi2a|opi2c|p64b|p64c|ar71xx|reproducible_debian_live_build$|chroot-installation_stretch|chroot-installation_bullseye*upgrade_to_sid|rebuilder_prototype' || true)"
if [ ! -z "$ZOMBIES" ] ; then
DIRTY=true
figlet 'zombies!!!'
=====================================
bin/jenkins_node_wrapper.sh
=====================================
@@ -79,8 +79,6 @@ elif [[ "$1" =~ ^chroot-installation_.* ]]; then
shift ; exec /srv/jenkins/bin/chroot-installation.sh $@ ; croak "Exec failed"
elif [ "$*" = "reproducible_nodes_info" ] ; then
exec /srv/jenkins/bin/reproducible_info.sh ; croak "Exec failed";
-elif [ "$1" = "reproducible_debian_rebuilder_prototype" ] ; then
- exec /srv/jenkins/bin/reproducible_debian_rebuilder_prototype.sh ; croak "Exec failed";
elif [ "$1" = "reproducible_pool_buildinfos" ] ; then
exec /srv/jenkins/bin/reproducible_pool_buildinfos.sh ; croak "Exec failed";
elif [ "$1" = "reproducible_html_builtin-pho" ] ; then
=====================================
bin/rb-debrebuild deleted
=====================================
@@ -1,1280 +0,0 @@
-#!/usr/bin/perl
-#
-# Copyright © 2014-2020 Johannes Schauer <j.schauer at email.de>
-# Copyright © 2020 Niels Thykier <niels at thykier.net>
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-
-use strict;
-use warnings;
-use autodie;
-
-use Getopt::Long qw(:config gnu_getopt no_bundling no_auto_abbrev);
-
-use Dpkg::Control;
-use Dpkg::Index;
-use Dpkg::Deps;
-use Dpkg::Source::Package;
-use File::Temp qw(tempdir);
-use File::Path qw(make_path);
-use File::HomeDir;
-use JSON::PP;
-use Time::Piece;
-use File::Basename;
-use List::Util qw(any none);
-
-my $progname;
-
-BEGIN {
- $progname = basename($0);
- eval { require String::ShellQuote; };
- if ($@) {
- if ($@ =~ /^Can\'t locate String\/ShellQuote\.pm/) {
- die
-"$progname: you must have the libstring-shellquote-perl package installed\n"
- . "to use this script";
- } else {
- die
-"$progname: problem loading the String::ShellQuote module:\n $@\n"
- . "Have you installed the libstring-shellquote-perl package?";
- }
- }
-
- eval {
- require LWP::Simple;
- require LWP::UserAgent;
- require URI::Escape; # libwww-perl depends on liburi-perl
- no warnings;
- $LWP::Simple::ua
- = LWP::UserAgent->new(agent => 'LWP::UserAgent/debrebuild');
- $LWP::Simple::ua->env_proxy();
- };
- if ($@) {
- if ($@ =~ m/Can\'t locate LWP/) {
- die "$progname: you must have the libwww-perl package installed\n"
- . "to use this script";
- } else {
- die "$progname: problem loading the LWP and URI modules:\n $@\n"
- . "Have you installed the libwww-perl package?";
- }
- }
-
-}
-
-my $respect_build_path = 1;
-my $use_tor = 0;
-my $outdir = './';
-my $builder = 'none';
-my $timestamp = '';
-
-my %OPTIONS = (
- 'help|h' => sub { usage(0); },
- 'use-tor-proxy!' => \$use_tor,
- 'respect-build-path!' => \$respect_build_path,
- 'buildresult=s' => \$outdir,
- 'builder=s' => \$builder,
- 'timestamp|t=s' => \$timestamp,
-);
-
-sub usage {
- my ($exit_code) = @_;
- $exit_code //= 0;
- print <<EOF;
-Usage: $progname [options] <buildinfo>
- $progname <--help|-h>
-
-Given a buildinfo file from a Debian package, generate instructions for
-attempting to reproduce the binary packages built from the associated source
-and build information.
-
-Options:
- --help, -h Show this help and exit
- --[no-]use-tor-proxy Whether to fetch resources via tor (socks://127.0.0.1:9050)
- Assumes "apt-transport-tor" is installed both in host + chroot
- --[no-]respect-build-path Whether to setup the build to use the Build-Path from the
- provided .buildinfo file.
- --buildresults Directory for the build artifacts (default: ./)
- --builder=BUILDER Which building software should be used. Possible values are
- none, sbuild, mmdebstrap, dpkg and sbuild+unshare. The default
- is none. See section BUILDER for details.
- --timestamp, -t The required unstable main timestamps from snapshot.d.o if you
- already know them, separated by commas, or one of the values
- "first_seen" or "metasnap". See section TIMESTAMPS.
-
-Note: $progname can parse buildinfo files with and without a GPG signature. However,
-the signature (if present) is discarded as debrebuild does not support verifying
-it. If the authenticity or integrity of the buildinfo files are important to
-you, checking these need to be done before invoking $progname, for example by using
-dscverify.
-
-EXAMPLES
-
- \$ $progname --buildresults=./artifacts --builder=mmdebstrap hello_2.10-2_amd64.buildinfo
-
-BUILDERS
-
-debrebuild can use different backends to perform the actual package rebuild.
-The desired backend is chosen using the --builder option. The default is
-"none".
-
- none Dry-run mode. No build is performed.
- sbuild Use sbuild to build the package. This requires sbuild to be
- setup with schroot chroots of Debian stable distributions.
- mmdebstrap Use mmdebstrap to build the package. This requires no
- setup and no superuser privileges.
- dpkg Directly run apt-get and dpkg-buildpackage on the current
- system without chroot. This requires root privileges..
- sbuild+unshare Use sbuild with the unshare backend. This will create the
- chroot and perform the build without superuser privileges
- and without any setup.
-
-TIMESTAMPS
-
-The --timestamp option allows one to skip the step of figuring out the correct
-set of required timestamps by listing them separated by commas in the same
-format used in the snapshot.d.o URL. The default is to use the "first_seen"
-attribute from the snapshot.d.o API and download multiple Packages files until
-all required timestamps are found. To explicitly select this mode, use
---timestamp=first_seen. Lastly, the metasnap.d.n service can be used to figure
-out the right set of timestamps. This mode can be selected by using
---timestamp=metasnap. In contrast to the "first_seen" mode, the metasnap.d.n
-service will always return a minimal set of timestamps if the package versions
-were at some point part of Debian unstable main.
-
-UNSHARE
-
-Before kernel 5.10.1 or before Debian 11 (Bullseye), unprivileged user
-namespaces were disabled in Debian for security reasons. Refer to Debian bug
-#898446 for details. To enable user namespaces, run:
-
- \$ sudo sysctl -w kernel.unprivileged_userns_clone=1
-
-The sbuild+unshare builder requires and the mmdebstrap builder benefits from
-having unprivileged user namespaces activated. On Ubuntu they are enabled by
-default.
-
-LIMITATIONS
-
-Currently, the code assumes that all packages were at some point part of Debian
-unstable main. This fails for packages from Debian ports, packages from
-experimental as well as for locally built packages or packages from third
-party repositories. Enabling support for Debian ports and experimental is
-conceptually possible and only needs somebody implementing it.
-
-EOF
-
- exit($exit_code);
-}
-
-GetOptions(%OPTIONS);
-
-my $buildinfo = shift @ARGV;
-if (not defined($buildinfo)) {
- print STDERR "ERROR: Missing mandatory buildinfo filename\n";
- print STDERR "\n";
- usage(1);
-}
-if ($buildinfo eq '--help' or $buildinfo eq '-h') {
- usage(0);
-}
-
-if ($buildinfo =~ m/^-/) {
- print STDERR "ERROR: Unsupported option $buildinfo\n";
- print STDERR "\n";
- usage(1);
-}
-
-if (@ARGV) {
- print STDERR "ERROR: This program requires exactly argument!\n";
- print STDERR "\n";
- usage(1);
-}
-
-my $base_mirror = "http://snapshot.debian.org/archive/debian";
-if ($use_tor) {
- $base_mirror = "tor+http://snapshot.debian.org/archive/debian";
- eval {
- $LWP::Simple::ua->proxy([qw(http https)] => 'socks://127.0.0.1:9050');
- };
- if ($@) {
- if ($@ =~ m/Can\'t locate LWP/) {
- die
-"Unable to use tor: the liblwp-protocol-socks-perl package is not installed\n";
- } else {
- die "Unable to use tor: Couldn't load socks proxy support: $@\n";
- }
- }
-}
-
-# buildinfo support in libdpkg-perl (>= 1.18.11)
-my $cdata = Dpkg::Control->new(type => CTRL_FILE_BUILDINFO, allow_pgp => 1);
-
-if (not $cdata->load($buildinfo)) {
- die "cannot load $buildinfo\n";
-}
-
-if ($cdata->get_option('is_pgp_signed')) {
- print
-"$buildinfo contained a GPG signature; it has NOT been validated (debrebuild does not support this)!\n";
-} else {
- print "$buildinfo was unsigned\n";
-}
-
-my @architectures = split /\s+/, $cdata->{"Architecture"};
-my $build_source = (scalar(grep /^source$/, @architectures)) == 1;
-my $build_archall = (scalar(grep /^all$/, @architectures)) == 1;
- at architectures = grep { !/^source$/ && !/^all$/ } @architectures;
-if (scalar @architectures > 1) {
- die "more than one architecture in Architecture field\n";
-}
-my $build_archany = (scalar @architectures) == 1;
-
-my $build_arch = $cdata->{"Build-Architecture"};
-if (not defined($build_arch)) {
- die "need Build-Architecture field\n";
-}
-my $host_arch = $cdata->{"Host-Architecture"};
-if (not defined($host_arch)) {
- $host_arch = $build_arch;
-}
-
-my $srcpkgname = $cdata->{Source};
-my $srcpkgver = $cdata->{Version};
-my $srcpkgbinver
- = $cdata->{Version}; # this version will include the binmu suffix
-if ($srcpkgname =~ / /) {
- # In some cases such as binNMUs, the source field contains a version in
- # the form:
- # mscgen (0.20)
- ($srcpkgname, $srcpkgver) = split / /, $srcpkgname, 2;
- # Add a simple control check to avoid the worst surprises and stop obvious
- # cases of garbage-in-garbage-out.
- die("Unexpected source package name: ${srcpkgname}\n")
- if $srcpkgname =~ m{[ \t_/\(\)<>!\n%&\$\#\@]};
- # remove the surrounding parenthesis from the version
- $srcpkgver =~ s/^\((.*)\)$/$1/;
-}
-
-my $new_buildinfo;
-{
- my $arch;
- if ($build_archany) {
- $arch = $host_arch;
- } elsif ($build_archall) {
- $arch = 'all';
- } else {
- die "nothing to build\n";
- }
- $new_buildinfo = "$outdir/${srcpkgname}_${srcpkgbinver}_$arch.buildinfo";
-}
-if (-e $new_buildinfo) {
- my ($dev1, $ino1) = (lstat $buildinfo)[0, 1]
- or die "cannot lstat $buildinfo: $!\n";
- my ($dev2, $ino2) = (lstat $new_buildinfo)[0, 1]
- or die "cannot lstat $new_buildinfo: $!\n";
- if ($dev1 == $dev2 && $ino1 == $ino2) {
- die "refusing to overwrite the input buildinfo file\n";
- }
-}
-
-my $inst_build_deps = $cdata->{"Installed-Build-Depends"};
-if (not defined($inst_build_deps)) {
- die "need Installed-Build-Depends field\n";
-}
-my $custom_build_path = $respect_build_path ? $cdata->{'Build-Path'} : undef;
-
-if (defined($custom_build_path)) {
- if ($custom_build_path =~ m{['`\$\\"\(\)<>#]|(?:\a|/)[.][.](?:\z|/)}) {
- warn(
-"Retry build with --no-respect-build-path to ignore the Build-Path field.\n"
- );
- die(
-"Refusing to use $custom_build_path as Build-Path: Looks too special to be true"
- );
- }
-
- if ($custom_build_path eq '' or $custom_build_path !~ m{^/}) {
- warn(
-"Retry build with --no-respect-build-path to ignore the Build-Path field.\n"
- );
- die(
-qq{Build-Path must be a non-empty absolute path (i.e. start with "/").\n}
- );
- }
- print "Using defined Build-Path: ${custom_build_path}\n";
-} else {
- if ($respect_build_path) {
- print
-"No Build-Path defined; not setting a defined build path for this build.\n";
- }
-}
-
-my $srcpkg = Dpkg::Source::Package->new();
-$srcpkg->{fields}{'Source'} = $srcpkgname;
-$srcpkg->{fields}{'Version'} = $srcpkgver;
-my $dsc_fname
- = (dirname($buildinfo)) . '/' . $srcpkg->get_basename(1) . ".dsc";
-
-my $environment = $cdata->{"Environment"};
-if (not defined($environment)) {
- die "need Environment field\n";
-}
-$environment =~ s/\n/ /g; # remove newlines
-$environment =~ s/^ //; # remove leading whitespace
-
-my @environment;
-foreach my $line (split /\n/, $cdata->{"Environment"}) {
- chomp $line;
- if ($line eq '') {
- next;
- }
- my ($name, $val) = split /=/, $line, 2;
- $val =~ s/^"(.*)"$/$1/;
- push @environment, "$name=$val";
-}
-
-# gather all installed build-depends and figure out the version of base-files
-my $base_files_version;
-my @inst_build_deps = ();
-$inst_build_deps
- = deps_parse($inst_build_deps, reduce_arch => 0, build_dep => 0);
-if (!defined $inst_build_deps) {
- die "deps_parse failed\n";
-}
-
-foreach my $pkg ($inst_build_deps->get_deps()) {
- if (!$pkg->isa('Dpkg::Deps::Simple')) {
- die "dependency disjunctions are not allowed\n";
- }
- if (not defined($pkg->{package})) {
- die "name undefined\n";
- }
- if (defined($pkg->{relation})) {
- if ($pkg->{relation} ne "=") {
- die "wrong relation";
- }
- if (not defined($pkg->{version})) {
- die "version undefined\n";
- }
- } else {
- die "no version";
- }
- if ($pkg->{package} eq "base-files") {
- if (defined($base_files_version)) {
- die "more than one base-files\n";
- }
- $base_files_version = $pkg->{version};
- }
- push @inst_build_deps,
- {
- name => $pkg->{package},
- architecture => $pkg->{archqual},
- version => $pkg->{version} };
-}
-
-if (!defined($base_files_version)) {
- die "no base-files\n";
-}
-
-# figure out the debian release from the version of base-files
-my $base_dist;
-
-my %base_files_map = ();
-my $di_path = '/usr/share/distro-info/debian.csv';
-eval { require Debian::DistroInfo; };
-if (!$@) {
- # libdistro-info-perl is installed
- my $di = DebianDistroInfo->new();
- foreach my $series ($di->all) {
- if (!$di->version($series)) {
- next;
- }
- $base_files_map{ $di->version($series) } = $series;
- }
-} elsif (-f $di_path) {
- # distro-info-data is installed
- open my $fh, '<', $di_path or die "cannot open $di_path: $!\n";
- my $i = 0;
- while (my $line = <$fh>) {
- chomp($line);
- $i++;
- my @cells = split /,/, $line;
- if (scalar @cells < 4) {
- die "cannot parse line $i of $di_path\n";
- }
- if (
- $i == 1
- and ( scalar @cells < 6
- or $cells[0] ne 'version'
- or $cells[1] ne 'codename'
- or $cells[2] ne 'series'
- or $cells[3] ne 'created'
- or $cells[4] ne 'release'
- or $cells[5] ne 'eol')
- ) {
- die "cannot find correct header in $di_path\n";
- }
- if ($i == 1) {
- next;
- }
- $base_files_map{ $cells[0] } = $cells[2];
- }
- close $fh;
-} else {
- # nothing is installed -- use hard-coded values
- %base_files_map = (
- "6" => "squeeze",
- "7" => "wheezy",
- "8" => "jessie",
- "9" => "stretch",
- "10" => "buster",
- "11" => "bullseye",
- "12" => "bookworm",
- "13" => "trixie",
- );
-}
-
-$base_files_version =~ s/^(\d+).*/$1/;
-
-# we subtract one from $base_files_version because we want the Debian release
-# before what is currently in unstable
-$base_dist = $base_files_map{ $base_files_version - 1 };
-
-if (!defined $base_dist) {
- die "base-files version didn't map to any Debian release\n";
-}
-
-my $src_date;
-{
- print "retrieving snapshot.d.o data for $srcpkgname $srcpkgver\n";
- my $json_url
- = "http://snapshot.debian.org/mr/package/$srcpkgname/$srcpkgver/srcfiles?fileinfo=1";
- my $content = LWP::Simple::get($json_url);
- die "cannot retrieve $json_url" unless defined $content;
- my $json = JSON::PP->new();
- # json options taken from debsnap
- my $json_text = $json->allow_nonref->utf8->relaxed->decode($content);
- die "cannot decode json" unless defined $json_text;
- foreach my $result (@{ $json_text->{result} }) {
- # FIXME - assumption: package is from Debian official (and not ports)
- my @package_from_main = grep { $_->{archive_name} eq "debian" }
- @{ $json_text->{fileinfo}->{ $result->{hash} } };
- if (scalar @package_from_main > 1) {
- die
- "more than one package with the same hash in Debian official\n";
- }
- if (scalar @package_from_main == 0) {
- die "no package with the right hash in Debian official\n";
- }
- $src_date = $package_from_main[0]->{first_seen};
- }
-}
-if (!defined($src_date)) {
- die "cannot find .dsc\n";
-}
-
-# support timestamps being separated by a comma
-my @required_timestamps = ();
-if ($timestamp eq "first_seen") {
- # nothing to do, timestamps will be figured out later
-} elsif ($timestamp eq "metasnap") {
- # acquire the required timestamps using metasnap.d.n
- print "retrieving required timestamps from metasnap.d.n\n";
- my $ua = LWP::UserAgent->new(timeout => 10);
- $ua->env_proxy;
- my @pkgs = ();
- foreach my $pkg (@inst_build_deps) {
- my $pkg_name = $pkg->{name};
- my $pkg_ver = $pkg->{version};
- my $pkg_arch = $pkg->{architecture};
- if (defined $pkg_arch) {
- push @pkgs,
- URI::Escape::uri_escape("$pkg_name:$pkg_arch=$pkg_ver");
- } else {
- push @pkgs, URI::Escape::uri_escape("$pkg_name=$pkg_ver");
- }
- }
- my $response
- = $ua->get('https://metasnap.debian.net/cgi-bin/api'
- . '?archive=debian'
- . "&pkgs="
- . (join "%2C", @pkgs)
- . "&arch=$build_arch"
- . '&suite=unstable'
- . '&comp=main');
- if (!$response->is_success) {
- die "request to metasnap.d.n failed: $response->status_line";
- }
- foreach my $line (split /\n/, $response->decoded_content) {
- my ($arch, $t) = split / /, $line, 2;
- if ($arch ne $build_arch) {
- die
-"debrebuild is currently unable to handle multiple architectures";
- }
- push @required_timestamps, $t;
- }
-} else {
- @required_timestamps = split(/,/, $timestamp);
-}
-
-# setup a temporary apt directory
-
-my $tempdir = tempdir(CLEANUP => 1);
-
-foreach my $d ((
- '/etc/apt', '/etc/apt/apt.conf.d',
- '/etc/apt/preferences.d', '/etc/apt/trusted.gpg.d',
- '/etc/apt/sources.list.d', '/var/lib/apt/lists/partial',
- '/var/cache/apt/archives/partial', '/var/lib/dpkg',
- )
-) {
- make_path("$tempdir/$d");
-}
-
-# We use the Build-Date field as a heuristic to find a good date for the
-# stable release. If we would get the stable release from deb.debian.org
-# instead, then packages might be newer than in unstable of the past because
-# of point releases. The date from the source package will also work in most
-# cases but will fail for binNMU buildinfo files where the source package
-# might even come from years in the past
-my $build_date;
-{
- local $ENV{LC_ALL} = 'C';
- my $tp
- = Time::Piece->strptime($cdata->{'Build-Date'}, '%a, %d %b %Y %T %z');
- $build_date = $tp->strftime("%Y%m%dT%H%M%SZ");
-}
-
-sub get_sources_list() {
- my @result = ();
- push @result, "deb $base_mirror/$build_date/ $base_dist main";
- push @result, "deb-src $base_mirror/$src_date/ unstable main";
- foreach my $ts (@required_timestamps) {
- push @result, "deb $base_mirror/$ts/ unstable main";
- }
- return @result;
-}
-
-open(FH, '>', "$tempdir/etc/apt/sources.list");
-print FH (join "\n", get_sources_list) . "\n";
-close FH;
-# FIXME - document what's dpkg's status for
-# Create dpkg status
-open(FH, '>', "$tempdir/var/lib/dpkg/status");
-close FH; #empty file
-# Create apt.conf
-my $aptconf = "$tempdir/etc/apt/apt.conf";
-open(FH, '>', $aptconf);
-
-# We create an apt.conf and pass it to apt via the APT_CONFIG environment
-# variable instead of passing all options via the command line because
-# otherwise apt will read the system's config first and might get unwanted
-# configuration options from there. See apt.conf(5) for the order in which
-# configuration options are read.
-#
-# While we are at it, we also set all other options through our custom
-# apt.conf.
-#
-# Apt::Architecture has to be set because otherwise apt will default to the
-# architecture apt was compiled for.
-#
-# Apt::Architectures has to be set or otherwise apt will use dpkg to find all
-# foreign architectures of the system running apt.
-#
-# Dir::State::status has to be set even though Dir is set because Dir::State
-# is set to var/lib/apt, so Dir::State::status would be below that but really
-# isn't and without an absolute path, Dir::State::status would be constructed
-# from Dir + Dir::State + Dir::State::status. This has been fixed in apt
-# commit 475f75506db48a7fa90711fce4ed129f6a14cc9a.
-#
-# Acquire::Check-Valid-Until has to be set to false because the snapshot
-# timestamps might be too far in the past to still be valid. This could be
-# fixed by a solution to https://bugs.debian.org/763419
-#
-# Acquire::Languages has to be set to prevent downloading of translations from
-# the mirrors.
-#
-# Binary::apt-get::Acquire::AllowInsecureRepositories has to be set to false
-# so that apt-get update fails if repositories cannot be authenticated. The
-# default value of this option will change to true with apt from Debian
-# Buster.
-#
-# We need APT::Get::allow-downgrades set to true, because even if we choose a
-# base distribution that was released before the state that "unstable"
-# currently is in, the package versions in that stable release might be newer
-# than what is in unstable due to security fixes. Choosing a stable release
-# from an older snapshot timestamp would fix this problem but would defeat the
-# purpose of a base distribution for builders like sbuild which can take
-# advantage of existing chroot environments.
-
-print FH <<EOF;
-Apt {
- Architecture "$build_arch";
- Architectures "$build_arch";
-};
-
-Dir "$tempdir";
-Dir::State::status "$tempdir/var/lib/dpkg/status";
-Acquire::Languages "none";
-Binary::apt-get::Acquire::AllowInsecureRepositories "false";
-EOF
-my @common_aptopts = (
- 'Acquire::Check-Valid-Until "false";',
- 'Acquire::http::Dl-Limit "1000";',
- 'Acquire::https::Dl-Limit "1000";',
- 'Acquire::Retries "5";',
- 'APT::Get::allow-downgrades "true";',
-);
-foreach my $line (@common_aptopts) {
- print FH "$line\n";
-}
-close FH;
-
-# add the removed keys because they are not returned by Dpkg::Vendor
-# we don't need the Ubuntu vendor now but we already put the comments to
-# possibly extend this script to other Debian derivatives
-my @keyrings = ();
-my $debianvendor = Dpkg::Vendor::Debian->new();
-push @keyrings, $debianvendor->run_hook('archive-keyrings');
-push @keyrings, $debianvendor->run_hook('archive-keyrings-historic');
-#my $ubuntuvendor = Dpkg::Vendor::Ubuntu->new();
-#push @keyrings, $ubuntuvendor->run_hook('archive-keyrings');
-#push @keyrings, $ubuntuvendor->run_hook('archive-keyrings-historic');
-
-foreach my $keyring (@keyrings) {
- my $base = basename $keyring;
- print "$keyring\n";
- if (-f $keyring) {
- print "linking $tempdir/etc/apt/trusted.gpg.d/$base to $keyring\n";
- symlink $keyring, "$tempdir/etc/apt/trusted.gpg.d/$base";
- }
-}
-
-$ENV{'APT_CONFIG'} = $aptconf;
-
-0 == system 'apt-get', 'update' or die "apt-get update failed\n";
-
-sub dpkg_index_key_func {
- return
- $_[0]->{Package} . ' '
- . $_[0]->{Version} . ' '
- . $_[0]->{Architecture};
-}
-
-sub parse_all_packages_files {
- my $dpkg_index = Dpkg::Index->new(get_key_func => \&dpkg_index_key_func);
-
- open(my $fd, '-|', 'apt-get', 'indextargets', '--format', '$(FILENAME)',
- 'Created-By: Packages');
- while (my $fname = <$fd>) {
- chomp $fname;
- print "parsing $fname...\n";
- open(my $fd2, '-|', '/usr/lib/apt/apt-helper', 'cat-file', $fname);
- $dpkg_index->parse($fd2, "pipe") or die "cannot parse Packages file\n";
- close($fd2);
- }
- close($fd);
- return $dpkg_index;
-}
-
-my $index = parse_all_packages_files();
-if (scalar @required_timestamps == 0) {
- # go through all packages in the Installed-Build-Depends field and find out
- # the timestamps at which they were first seen each
- my %notfound_timestamps;
-
- my %missing;
-
- foreach my $pkg (@inst_build_deps) {
- my $pkg_name = $pkg->{name};
- my $pkg_ver = $pkg->{version};
- my $pkg_arch = $pkg->{architecture};
-
- # check if we really need to acquire this package from snapshot.d.o or if
- # it already exists in the cache
- if (defined $pkg->{architecture}) {
- if ($index->get_by_key("$pkg_name $pkg_ver $pkg_arch")) {
- print "skipping $pkg_name $pkg_ver\n";
- next;
- }
- } else {
- if ($index->get_by_key("$pkg_name $pkg_ver $build_arch")) {
- $pkg->{architecture} = $build_arch;
- print "skipping $pkg_name $pkg_ver\n";
- next;
- }
- if ($index->get_by_key("$pkg_name $pkg_ver all")) {
- $pkg->{architecture} = "all";
- print "skipping $pkg_name $pkg_ver\n";
- next;
- }
- }
-
- print "retrieving snapshot.d.o data for $pkg_name $pkg_ver\n";
- my $json_url
- = "http://snapshot.debian.org/mr/binary/$pkg_name/$pkg_ver/binfiles?fileinfo=1";
- my $content = LWP::Simple::get($json_url);
- die "cannot retrieve $json_url" unless defined $content;
- my $json = JSON::PP->new();
- # json options taken from debsnap
- my $json_text = $json->allow_nonref->utf8->relaxed->decode($content);
- die "cannot decode json" unless defined $json_text;
- my $pkg_hash;
- if (scalar @{ $json_text->{result} } == 1) {
- # if there is only a single result, then the package must either be
- # Architecture:all, be the build architecture or match the requested
- # architecture
- $pkg_hash = ${ $json_text->{result} }[0]->{hash};
- $pkg->{architecture}
- = ${ $json_text->{result} }[0]->{architecture};
- # if a specific architecture was requested, it should match
- if (defined $pkg_arch && $pkg_arch ne $pkg->{architecture}) {
- die
-"package $pkg_name was explicitly requested for $pkg_arch but only $pkg->{architecture} was found\n";
- }
- # if no specific architecture was requested, it should be the build
- # architecture
- if ( !defined $pkg_arch
- && $build_arch ne $pkg->{architecture}
- && "all" ne $pkg->{architecture}) {
- die
-"package $pkg_name was implicitly requested for $pkg_arch but only $pkg->{architecture} was found\n";
- }
- # Ensure that $pkg_arch is defined from here as we want to look it up
- # later in a Packages file from snapshot.d.o if it is not in the
- # current Packages file
- $pkg_arch = $pkg->{architecture};
- } else {
- # Since the package occurs more than once, we expect it to be of
- # Architecture:any
- #
- # If no specific architecture was requested, look for the build
- # architecture
- if (!defined $pkg_arch) {
- $pkg_arch = $build_arch;
- }
- foreach my $result (@{ $json_text->{result} }) {
- if ($result->{architecture} eq $pkg_arch) {
- $pkg_hash = $result->{hash};
- last;
- }
- }
- if (!defined($pkg_hash)) {
- die "cannot find package in architecture $pkg_arch\n";
- }
- # we now know that this package is not architecture:all but has a
- # concrete architecture
- $pkg->{architecture} = $pkg_arch;
- }
- # FIXME - assumption: package is from Debian official (and not ports)
- my @package_from_main = grep { $_->{archive_name} eq "debian" }
- @{ $json_text->{fileinfo}->{$pkg_hash} };
- if (scalar @package_from_main > 1) {
- die
- "more than one package with the same hash in Debian official\n";
- }
- if (scalar @package_from_main == 0) {
- die "no package with the right hash in Debian official\n";
- }
- my $date = $package_from_main[0]->{first_seen};
- $pkg->{first_seen} = $date;
- $notfound_timestamps{$date} = 1;
- $missing{"${pkg_name}/${pkg_ver}/${pkg_arch}"} = 1;
- }
-
- # feed apt with timestamped snapshot.debian.org URLs until apt is able to
- # find all the required package versions. We start with the most recent
- # timestamp, check which packages cannot be found at that timestamp, add
- # the timestamp of the most recent not-found package and continue doing
- # this iteratively until all versions can be found.
-
- while (0 < scalar keys %notfound_timestamps) {
- print "left to check: " . (scalar keys %notfound_timestamps) . "\n";
- my @timestamps = map { Time::Piece->strptime($_, '%Y%m%dT%H%M%SZ') }
- (sort keys %notfound_timestamps);
- my $newest = $timestamps[$#timestamps];
- $newest = $newest->strftime("%Y%m%dT%H%M%SZ");
- push @required_timestamps, $newest;
- delete $notfound_timestamps{$newest};
-
- my $snapshot_url = "$base_mirror/$newest/";
-
- open(FH, '>>', "$tempdir/etc/apt/sources.list");
- print FH "deb ${snapshot_url} unstable main\n";
- close FH;
-
- 0 == system 'apt-get', 'update' or die "apt-get update failed\n";
-
- my $index = parse_all_packages_files();
- foreach my $pkg (@inst_build_deps) {
- my $pkg_name = $pkg->{name};
- my $pkg_ver = $pkg->{version};
- my $pkg_arch = $pkg->{architecture};
- my $first_seen = $pkg->{first_seen};
- my $cdata = $index->get_by_key("$pkg_name $pkg_ver $pkg_arch");
- if (not defined($cdata->{"Package"})) {
- # Not present yet; we hope a later snapshot URL will locate it.
- next;
- }
- delete($missing{"${pkg_name}/${pkg_ver}/${pkg_arch}"});
- if (defined $first_seen) {
- # this may delete timestamps that we actually need for some other
- # packages
- delete $notfound_timestamps{$first_seen};
- }
- }
- }
-
- if (%missing) {
- print STDERR 'Cannot locate the following packages via snapshots'
- . " or the current repo/mirror\n";
- for my $key (sort(keys(%missing))) {
- print STDERR " ${key}\n";
- }
- exit(1);
- }
-} else {
- # find out the actual package architecture for all installed build
- # dependencies without explicit architecture qualification
- foreach my $pkg (@inst_build_deps) {
- my $pkg_name = $pkg->{name};
- my $pkg_ver = $pkg->{version};
- if (defined $pkg->{architecture}) {
- next;
- }
- if ($index->get_by_key("$pkg_name $pkg_ver $build_arch")) {
- $pkg->{architecture} = $build_arch;
- next;
- }
- if ($index->get_by_key("$pkg_name $pkg_ver all")) {
- $pkg->{architecture} = "all";
- next;
- }
- die "cannot find $pkg_name $pkg_ver in index\n";
- }
-}
-
-# remove $tempdir manually to avoid any surprises
-0 == system 'apt-get', '--option',
- 'Dir::Etc::SourceList=/dev/null', '--option',
- 'Dir::Etc::SourceParts=/dev/null', 'update'
- or die "apt-get update failed\n";
-
-foreach my $f (
- '/var/cache/apt/pkgcache.bin',
- '/var/cache/apt/srcpkgcache.bin',
- '/var/lib/dpkg/status',
- '/var/lib/apt/lists/lock',
- '/etc/apt/apt.conf',
- '/etc/apt/sources.list',
- '/etc/apt/trusted.gpg.d/debian-archive-removed-keys.gpg',
- '/etc/apt/trusted.gpg.d/debian-archive-keyring.gpg'
-) {
- unlink "$tempdir/$f" or die "cannot unlink $tempdir/$f: $!\n";
-}
-
-foreach my $d (
- '/var/cache/apt/archives/partial', '/var/cache/apt/archives',
- '/var/cache/apt', '/var/cache',
- '/var/lib/dpkg', '/var/lib/apt/lists/auxfiles',
- '/var/lib/apt/lists/partial', '/var/lib/apt/lists',
- '/var/lib/apt', '/var/lib',
- '/var', '/etc/apt/sources.list.d',
- '/etc/apt/trusted.gpg.d', '/etc/apt/preferences.d',
- '/etc/apt/apt.conf.d', '/etc/apt',
- '/etc', ''
-) {
- rmdir "$tempdir/$d" or die "cannot rmdir $d: $!\n";
-}
-
-!-e $tempdir or die "failed to remove $tempdir\n";
-
-if ($builder ne "none") {
- if (!-e $outdir) {
- make_path($outdir);
- }
-}
-
-my $build = '';
-my $changesarch = '';
-if ($build_archany and $build_archall) {
- $build = "binary";
- $changesarch = $host_arch;
-} elsif ($build_archany and !$build_archall) {
- $build = "any";
- $changesarch = $host_arch;
-} elsif (!$build_archany and $build_archall) {
- $build = "all";
- $changesarch = 'all';
-} else {
- die "nothing to build\n";
-}
-
-my @install = ();
-foreach my $pkg (@inst_build_deps) {
- my $pkg_name = $pkg->{name};
- my $pkg_ver = $pkg->{version};
- my $pkg_arch = $pkg->{architecture};
- if (any { $_ eq $builder } ('mmdebstrap', 'none', 'dpkg')) {
- if ($pkg_arch eq "all" || $pkg_arch eq $build_arch) {
- push @install, "$pkg_name=$pkg_ver";
- } else {
- push @install, "$pkg_name:$pkg_arch=$pkg_ver";
- }
- } elsif (any { $_ eq $builder } ('sbuild', 'sbuild+unshare')) {
- if ($pkg_arch eq "all" || $pkg_arch eq $build_arch) {
- push @install, "$pkg_name (= $pkg_ver)";
- } else {
- push @install, "$pkg_name:$pkg_arch (= $pkg_ver)";
- }
- } else {
- die "unsupported builder: $builder\n";
- }
-}
-
-if ($builder eq "none") {
- print "\n";
- print "Manual installation and build\n";
- print "-----------------------------\n";
- print "\n";
- print
- "The following sources.list contains all the required repositories:\n";
- print "\n";
- print(join "\n", get_sources_list);
- print "\n";
- print "You can manually install the right dependencies like this:\n";
- print "\n";
- print "apt-get install --no-install-recommends";
-
- # Release files from snapshots.d.o have often expired by the time
- # we fetch them. Include the option to work around that to assist
- # the user.
- print " -oAcquire::Check-Valid-Until=false";
- foreach my $pkg (@install) {
- print " $pkg";
- }
- print "\n";
- print "\n";
- print "And then build your package:\n";
- print "\n";
- if ($custom_build_path) {
- require Cwd;
- my $custom_build_parent_dir = dirname($custom_build_path);
- my $dsc_path = Cwd::realpath($dsc_fname)
- // die("Cannot resolve ${dsc_fname}: $!\n");
- print "mkdir -p \"${custom_build_parent_dir}\"\n";
- print qq{dpkg-source -x "${dsc_path}" "${custom_build_path}"\n};
- print "cd \"$custom_build_path\"\n";
- } else {
- print qq{dpkg-source -x "${dsc_fname}"\n};
- print "cd packagedirectory\n";
- }
- print "\n";
- if ($cdata->{"Binary-Only-Changes"}) {
- print( "Since this is a binNMU, you must put the following "
- . "lines at the top of debian/changelog:\n\n");
- print($cdata->{"Binary-Only-Changes"});
- }
- print "\n";
- print( "$environment dpkg-buildpackage -uc "
- . "--host-arch=$host_arch --build=$build\n");
-} elsif ($builder eq "dpkg") {
- if ("$build_arch\n" ne `dpkg --print-architecture`) {
- die "must be run on $build_arch\n";
- }
-
- if ($> != 0) {
- die "you must be root for the dpkg builder\n";
- }
-
- if (-e $custom_build_path) {
- die "$custom_build_path exists -- refusing to overwrite\n";
- }
-
- my $sources = '/etc/apt/sources.list.d/debrebuild.list';
- if (-e $sources) {
- die "$sources already exists -- refusing to overwrite\n";
- }
- open(FH, '>', $sources) or die "cannot open $sources: $!\n";
- print FH (join "\n", get_sources_list) . "\n";
- close FH;
-
- my $config = '/etc/apt/apt.conf.d/23-debrebuild.conf';
- if (-e $config) {
- die "$config already exists -- refusing to overwrite\n";
- }
- open(FH, '>', $config) or die "cannot open $config: $!\n";
- foreach my $line (@common_aptopts) {
- print FH "$line\n";
- }
- close FH;
-
- 0 == system 'apt-get', 'update' or die "apt-get update failed\n";
-
- my @cmd
- = ('apt-get', 'install', '--no-install-recommends', '--yes', @install);
- 0 == system @cmd or die "apt-get install failed\n";
-
- 0 == system 'apt-get', 'source', '--only-source', '--download-only',
- "$srcpkgname=$srcpkgver"
- or die "apt-get source failed\n";
- unlink $sources or die "failed to unlink $sources\n";
- unlink $config or die "failed to unlink $config\n";
- make_path(dirname $custom_build_path);
- 0 == system 'dpkg-source', '--no-check', '--extract',
- $srcpkg->get_basename(1) . '.dsc', $custom_build_path
- or die "dpkg-source failed\n";
-
- if ($cdata->{"Binary-Only-Changes"}) {
- open my $infh, '<', "$custom_build_path/debian/changelog"
- or die "cannot open debian/changelog for reading: $!\n";
- my $changelogcontent = do { local $/; <$infh> };
- close $infh;
- open my $outfh, '>', "$custom_build_path/debian/changelog"
- or die "cannot open debian/changelog for writing: $!\n";
- my $logentry = $cdata->{"Binary-Only-Changes"};
- # due to storing the binnmu changelog entry in deb822 buildinfo, the
- # first character is an unwanted newline
- $logentry =~ s/^\n//;
- print $outfh $logentry;
- # while the linebreak at the beginning is wrong, there are two missing
- # at the end
- print $outfh "\n\n";
- print $outfh $changelogcontent;
- close $outfh;
- }
- 0 == system 'env', "--chdir=$custom_build_path", @environment,
- 'dpkg-buildpackage', '-uc', "--host-arch=$host_arch", "--build=$build"
- or die "dpkg-buildpackage failed\n";
- # we are not interested in the unpacked source directory
- 0 == system 'rm', '-r', $custom_build_path
- or die "failed to remove $custom_build_path: $?";
- # but instead we want the produced artifacts
- 0 == system 'dcmd', 'mv',
- (dirname $custom_build_path)
- . "/${srcpkgname}_${srcpkgbinver}_$changesarch.changes", $outdir
- or die "dcmd failed\n";
-} elsif ($builder eq "sbuild" or $builder eq "sbuild+unshare") {
- my $tarballpath = File::HomeDir->my_home
- . "/.cache/sbuild/$base_dist-$build_arch.tar.gz";
- if ($builder eq "sbuild+unshare") {
- if (!-e $tarballpath) {
- my $chrootdir = tempdir();
- 0 == system 'sbuild-createchroot', '--chroot-mode=unshare',
- '--make-sbuild-tarball', $tarballpath,
- $base_dist, $chrootdir, "$base_mirror/$build_date/"
- or die "sbuild-createchroot failed\n";
- !-e $chrootdir or die "$chrootdir wasn't removed\n";
- }
- }
-
- my @cmd = ('env', "--chdir=$outdir", @environment, 'sbuild');
- foreach my $line (get_sources_list) {
- push @cmd, "--extra-repository=$line";
- }
-
- # Release files from snapshots.d.o have often expired by the time
- # we fetch them. Include the option to work around that to assist
- # the user.
- push @cmd,
- '--chroot-setup-commands=echo '
- . (String::ShellQuote::shell_quote(join '\n', @common_aptopts))
- . ' | tee /etc/apt/apt.conf.d/23-debrebuild.conf';
-
- # sbuild chroots have build-essential already installed. This might
- # interfere with the packages that we need to install. Example:
- # libc6-dev : Breaks: libgcc-8-dev (< 8.4.0-2~) but 8.3.0-6 is to be inst..
- # Thus, we remove them beforehand -- the right versions will get installed
- # later anyways.
- # We have to list the packages manually instead of relying on autoremove
- # because debootstrap marks them all as manually installed.
- push @cmd,
- ( '--chroot-setup-commands=apt-get --yes remove build-essential'
- . ' libc6-dev gcc g++ make dpkg-dev');
- push @cmd, '--chroot-setup-commands=apt-get --yes autoremove';
-
- push @cmd, "--add-depends=" . (join ",", @install);
- push @cmd, "--build=$build_arch";
- push @cmd, "--host=$host_arch";
-
- if ($build_source) {
- push @cmd, '--source';
- } else {
- push @cmd, '--no-source';
- }
- if ($build_archany) {
- push @cmd, '--arch-any';
- } else {
- push @cmd, '--no-arch-any';
- }
- if ($build_archall) {
- push @cmd, '--arch-all';
- } else {
- push @cmd, '--no-arch-all';
- }
- if ($cdata->{"Binary-Only-Changes"}) {
- push @cmd, "--binNMU-changelog=$cdata->{'Binary-Only-Changes'}";
- }
- if ($builder eq "sbuild+unshare") {
- push @cmd, "--chroot=$tarballpath";
- push @cmd, "--chroot-mode=unshare";
- }
- push @cmd, "--dist=$base_dist";
- push @cmd, "--no-run-lintian";
- push @cmd, "--no-run-autopkgtest";
- push @cmd, "--no-apt-upgrade";
- push @cmd, "--no-apt-distupgrade";
- # disable the explainer
- push @cmd, "--bd-uninstallable-explainer=";
- # We need the aspcud resolver to install packages that are older than the
- # ones in the latest snapshot. Apt by default will only use the latest
- # package versions as candidates and sbuild uses a dummy package instead
- # of crafting an apt command line with the exact version requirements.
- push @cmd, "--build-dep-resolver=aspcud";
-
- if ($custom_build_path) {
- push @cmd, "--build-path=$custom_build_path";
- }
- push @cmd, "${srcpkgname}_$srcpkgver";
- print((join " ", @cmd) . "\n");
- 0 == system @cmd or die "sbuild failed\n";
-} elsif ($builder eq "mmdebstrap") {
-
- my @binnmucmds = ();
- if ($cdata->{"Binary-Only-Changes"}) {
- my $logentry = $cdata->{"Binary-Only-Changes"};
- # due to storing the binnmu changelog entry in deb822 buildinfo, the first
- # character is an unwanted newline
- $logentry =~ s/^\n//;
- # while the linebreak at the beginning is wrong, there are two missing at
- # the end
- $logentry .= "\n\n";
- push @binnmucmds,
- '{ printf "%s" '
- . (String::ShellQuote::shell_quote $logentry)
- . "; cat debian/changelog; } > debian/changelog.debrebuild",
- "mv debian/changelog.debrebuild debian/changelog";
- }
-
- my @cmd = (
- 'env', '-i',
- 'PATH=/usr/sbin:/usr/bin:/sbin:/bin',
- 'mmdebstrap',
- "--arch=$build_arch",
- "--variant=apt",
- (map { "--aptopt=$_" } @common_aptopts),
- '--include=' . (join ' ', @install),
- '--essential-hook=chroot "$1" sh -c "'
- . (
- join ' && ',
- 'rm /etc/apt/sources.list',
- 'echo '
- . (
- String::ShellQuote::shell_quote(
- (join "\n", get_sources_list) . "\n"
- ))
- . ' >> /etc/apt/sources.list',
- 'apt-get update'
- )
- . '"',
- '--customize-hook=chroot "$1" sh -c "'
- . (
- join ' && ',
- "apt-get source --only-source -d $srcpkgname=$srcpkgver",
- "mkdir -p "
- . (String::ShellQuote::shell_quote(dirname $custom_build_path)),
- "dpkg-source --no-check -x /"
- . $srcpkg->get_basename(1) . '.dsc '
- . (String::ShellQuote::shell_quote $custom_build_path),
- 'cd ' . (String::ShellQuote::shell_quote $custom_build_path),
- @binnmucmds,
-"env $environment dpkg-buildpackage -uc -a $host_arch --build=$build",
- 'cd /',
- 'rm -r ' . (String::ShellQuote::shell_quote $custom_build_path))
- . '"',
- '--customize-hook=sync-out '
- . (dirname $custom_build_path)
- . " $outdir",
- $base_dist,
- '/dev/null',
- "deb $base_mirror/$build_date/ $base_dist main"
- );
- print((join ' ', @cmd) . "\n");
-
- 0 == system @cmd or die "mmdebstrap failed\n";
-} else {
- die "unsupported builder: $builder\n";
-}
-
-# test if all checksums in the buildinfo file check out
-if ($builder ne "none") {
- print "build artifacts stored in $outdir\n";
-
- my $checksums = Dpkg::Checksums->new();
- $checksums->add_from_control($cdata);
- # remove the .dsc as we only did the binaries
- # - the .dsc cannot be reproduced anyways because we cannot reproduce its
- # signature
- # - binNMUs can only be done with --build=any
- foreach my $file ($checksums->get_files()) {
- if ($file !~ /\.dsc$/) {
- next;
- }
- $checksums->remove_file($file);
- }
-
- my $new_cdata
- = Dpkg::Control->new(type => CTRL_FILE_BUILDINFO, allow_pgp => 1);
- $new_cdata->load($new_buildinfo);
- my $new_checksums = Dpkg::Checksums->new();
- $new_checksums->add_from_control($new_cdata);
-
- my @files = $checksums->get_files();
- my @new_files = $new_checksums->get_files();
-
- if (scalar @files != scalar @new_files) {
- print("old buildinfo:\n" . (join "\n", @files) . "\n");
- print("new buildinfo:\n" . (join "\n", @new_files) . "\n");
- die "new buildinfo contains a different number of files\n";
- }
-
- for (my $i = 0 ; $i <= $#files ; $i++) {
- if ($files[$i] ne $new_files[$i]) {
- die "different checksum files at position $i\n";
- }
- if ($files[$i] =~ /\.dsc$/) {
- print("skipping $files[$i]\n");
- next;
- }
- print("checking $files[$i]: ");
- if ($checksums->get_size($files[$i])
- != $new_checksums->get_size($files[$i])) {
- die "size differs for $files[$i]\n";
- } else {
- print("size... ");
- }
- my $chksum = $checksums->get_checksum($files[$i], undef);
- my $new_chksum = $new_checksums->get_checksum($new_files[$i], undef);
- if (scalar keys %{$chksum} != scalar keys %{$new_chksum}) {
- die "different algos for $files[$i]\n";
- }
- foreach my $algo (keys %{$chksum}) {
- if (!exists $new_chksum->{$algo}) {
- die "$algo is not used in both buildinfo files\n";
- }
- if ($chksum->{$algo} ne $new_chksum->{$algo}) {
- die "value of $algo differs for $files[$i]\n";
- }
- print("$algo... ");
- }
- print("all OK\n");
- }
-}
=====================================
bin/reproducible_debian_rebuilder_prototype.sh deleted
=====================================
@@ -1,230 +0,0 @@
-#!/bin/bash
-# vim: set noexpandtab:
-
-# Copyright 2020-2021 Holger Levsen <holger at layer-acht.org>
-# released under the GPLv2
-
-cat << EOF
-
-###########################################################################################
-### ###
-### This prototype is meant to achieve several goals: ###
-### ###
-### One goal is to polish /usr/bin/debrebuild from src:devscripts to enable anyone to ###
-### independently verify that a distributed Debian binary package comes from the source ###
-### package it's said to be coming from. ###
-### ###
-### NOTE: we are using rb-debrebuild here which is a code copy from debrebuild from ###
-### devscripts.git - FIXME: use the version from bullseye. ###
-### ###
-### Once this goal has been achieved we can document these steps. currently the answer ###
-### to the question "how can I verify installed package X is reproducible?" is: "it's ###
-### really complicated" (and sometimes impossible or requires not yet written code). ###
-### ###
-### Another goal is to create json export to integrate in tracker.d.o and/or ###
-### packages.d.o as well as to provide statistics and graphs. ###
-### ###
-### The aim is to develop a 'real world' view about the reproducibility of all the ###
-### packages distributed via ftp.d.o. - so far tests.r-b.o/debian only shows the ###
-### 'theoretical' reproducibility of Debian packages. ###
-### ###
-### We'll leave out the problem of 'trust' here quite entirely. that's why it's called ###
-### a Debian rebuilder 'thing', to explore technical feasibility, duck taping our way ###
-### ahead, keeping our motto 'to allow anyone to independently verify...' in mind. ###
-### ###
-###########################################################################################
-
-EOF
-
-
-DEBUG=true
-. /srv/jenkins/bin/common-functions.sh
-common_init "$@"
-
-# common code for tests.reproducible-builds.org
-. /srv/jenkins/bin/reproducible_common.sh
-set -e
-
-output_echo() {
- echo "###########################################################################################"
- echo
- echo -e "$(date -u) - $1"
- echo
-}
-
-set_poolpath() {
- local PKG=$1
- if [ "${PKG:0:3}" = "lib" ] ; then
- POOLPATH="${PKG:0:4}"
- else
- POOLPATH="${PKG:0:1}"
- fi
-}
-
-#
-# define what to rebuild
-#
-
-#
-# find a random package in bullseye
-#
-ARCH=amd64
-SUITE=bullseye
-DISTNAME="$SUITE-$ARCH"
-PACKAGES=$(ls $CHPATH/$DISTNAME/var/lib/apt/lists/*_dists_${SUITE}_main_binary-${ARCH}_Packages)
-SOURCES=$(ls $CHPATH/$DISTNAME/var/lib/apt/lists/*_dists_${SUITE}_main_source_Sources)
-BINARY_PKG=$(grep-available . -s Package $PACKAGES | sort -R | head -1 | cut -d ' ' -f2)
-PKG=$( ( grep-available -X -S $BINARY_PKG -s Package $SOURCES || grep-available -X -P $BINARY_PKG -s Source $PACKAGES ) | cut -d ' ' -f2)
-VERSION=$(grep-available -X -S $PKG -s Version $SOURCES | head -1 | cut -d ' ' -f2)
-
-#
-# hardcoded test cases
-#
-#PKG=libofx ; BINARY_PKG=libofx-dev ; VERSION=1:0.9.15-3 # cannot find libdebconfclient0/0.250/amd64 in dumpavail -> impossible to recreate build environment
-#PKG=wpewebkit ; BINARY_PKG=wpewebkit-driver ; VERSION=2.28.0-1 # cannot find libc-bin/2.29-10/amd64 in dumpavail -> impossible to recreate build environment
-#PKG=libunibreak ; BINARY_PKG=libunibreak1 ; VERSION=1.1-2 # binNMU (on amd64 in bullseye, but not buster)
-#PKG=tomcat9 ; BINARY_PKG=tomcat9-user ; VERSION=9.0.37-3 # sbuild fails to build the package because dpkg-checkbuilddeps: error: Unmet build dependencies: - see job #924
-
-output_echo "let's try to rebuild $BINARY_PKG from src:$PKG ($VERSION) from $SUITE/$ARCH"
-EVERSION="$(echo $VERSION | cut -d ':' -f2)" # EPOCH_FREE_VERSION is too long
-
-# query .buildinfo file path (for that $PKG and $VERSION from bullseye...) from builtin-pho db
-BUILTINPHOPATH=$(psql -A -t -c "SELECT path FROM builds WHERE source='$PKG' AND source_version='$VERSION' AND ( arch_amd64=true OR arch_all=true ) LIMIT 1" buildinfo)
-if [ -z "$BUILTINPHOPATH" ] ; then
- output_echo "No .buildinfo file found for $PKG ($VERSION) for $ARCH - check https://tracker.debian.org/$PKG"
- OTHER_FILES=$(mktemp)
- psql -A -t -c "SELECT path FROM builds WHERE source='$PKG' AND (arch_amd64=true OR arch_all=true)" buildinfo > $OTHER_FILES
- if [ -s $OTHER_FILES ] ; then
- echo ".buildinfo files known for $PKG and (arch_amd64 or arch_all)"
- echo
- cat $OTHER_FILES
- else
- echo "Warning: no .buildinfo exists for $PKG ($VERSION) and (arch_amd64 or arch_all)."
- echo
- psql -A -t -c "SELECT path FROM builds WHERE source='$PKG'" buildinfo > $OTHER_FILES
- if [ -s $OTHER_FILES ] ; then
- echo ".buildinfo files known for $PKG:"
- echo
- cat $OTHER_FILES
- else
- echo "Warning: found no .buildinfo files for $PKG in builtin-pho db at all, let's query dak."
- rmadison -S -a $ARCH,all,source -s $SUITE,sid $PKG
- fi
- fi
- rm -f $OTHER_FILES
- output_echo "Warning: NBIFA - no .buildinfo file for $PKG ($VERSION on $ARCH) available. Exiting cleanly as this is out-of-scope."
- exit 0
-fi
-output_echo "Found $BUILTINPHOPATH"
-FILE="$(echo $BUILTINPHOPATH | cut -d '/' -f5-)"
-PKG="$(echo $FILE | cut -d '_' -f1)"
-BINARY_VERSION="$(echo $FILE | cut -d '_' -f2)"
-POOLPATH="" # declared as a global variable
-set_poolpath $PKG # so we can set it here with a function
-FTPMASTERPATH="$(echo $BUILTINPHOPATH | cut -d '/' -f2-4)"
-URLPATH="https://buildinfos.debian.net/ftp-master.debian.org/buildinfo/$FTPMASTERPATH"
-
-if [ "$VERSION" != "$BINARY_VERSION" ] && [ "$EVERSION" != "$BINARY_VERSION" ] ; then
- output_echo "Warning: .buildinfo file known for $PKG and $VERSION is $URL_PATH/$FILE: this indicates a binNMU because $VERSION != $BINARY_VERSION."
-elif [ "$VERSION" != "$EVERSION" ] ; then
- : # package has an epoch
-fi
-
-# FIXME: we need locking to enable concurrent job runs
-
-# FIXME: hack, should be done properly, with cleanup *after* the job run...
-mkdir -p $PWD/rebuilder-prototype
-BTPKG="$PWD/rebuilder-prototype/$PKG"
-mkdir $BTPKG 2>/dev/null || (rm $BTPKG -r ; mkdir $BTPKG)
-cd $BTPKG
-
-#
-# main: this is basically a description of the steps to use debrebuild today...
-#
-
-output_echo "downloading $URLPATH/$FILE"
-curl $URLPATH/$FILE > $FILE
-if head -1 $FILE | grep -q 'BEGIN PGP SIGNED MESSAGE' ; then
- output_echo "info: $URLPATH/$FILE with gpg signature stripped:"
-else
- output_echo "info: $URLPATH/$FILE is unsigned:"
-fi
-cat $FILE
-mv $FILE ..
-
-# prepare rebuild command
-set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html
-DEBREBUILD=$(mktemp -t debrebuild-cmd.XXXXXXXX)
-output_echo "trying to debrebuild $PKG ($BINARY_VERSION)"
-/srv/jenkins/bin/rb-debrebuild --builder=sbuild --timestamp=metasnap --buildresult=. ../$FILE 2>&1 | tee $DEBREBUILD
-RESULT=$?
-if [ "$RESULT" != "0" ] ; then
- output_echo "Warning: rb-debrebuild exited with $RESULT"
-fi
-
-# FIXME: drop if deemed unneeded with bullseye...
-# if egrep -q 'cannot find .* in dumpavail' $DEBREBUILD ; then
-# #FIXME: file bug, debrebuild should fail clearly on this (and not this subtile)
-# echo "The following build-dependencies are not available on snapshot.debian.org:"
-# echo
-# egrep 'cannot find .* in dumpavail' $DEBREBUILD
-# echo
-# echo "Warning: this should not happen and it's unclear why it did: $(egrep 'cannot find .* in dumpavail' $DEBREBUILD) - still exiting cleanly as this is out-of-scope."
-# exit 0
-# elif egrep -q 'Cannot locate the following packages via snapshots or the current repo/mirror' $DEBREBUILD ; then
-# echo "Some packages could not be located, check the log above."
-# echo
-# echo "Warning: this should not happen and it's unclear why it did - still exiting cleanly as this is out-of-scope."
-# exit 0
-# else
-# echo "Something went wrong with debrebuild, please take a look what exactly..."
-# echo
-# exit 1
-# fi
-
-# FIXME: drop if deemed unneeded with bullseye... (also drop the actuall schroot on disk and the sudo rights...)
-## create chroot for sbuild
-#if [ -d /schroots/debrebuild-$DISTRO-$ARCH ] ; then
-# output_echo "chroot for $DISTRO/$ARCH exists, good."
-#else
-# output_echo "preparing chroot for $DISTRO/$ARCH."
-# # FIXME: "|| true" is dummy code for regenerating this chroot every other week or so
-# sudo sbuild-createchroot $DISTRO /schroots/debrebuild-$DISTRO-$ARCH http://deb.debian.org/debian || true
-#
-# # I'm a bit surprised this was needed, as debrebuild has code for this...
-# # FIXME: a bug should probably be filed for this as well
-# echo 'Acquire::Check-Valid-Until "false";' | sudo tee /schroots/debrebuild-$DISTRO-$ARCH/etc/apt/apt.conf.d/23-rebuild
-#fi
-
-# show what we did/created
-output_echo "File artifacts:"
-ls -lart
-output_echo "Diff between .buildinfo files:"
-diff ../$FILE $FILE || true
-output_echo "The following binary packages could be rebuilt bit-by-bit identical to the ones distributed from ftp.debian.org:"
-# FIXME: this is irrelevant. instead the status of packages in the buildinfo file should be computed
-# (not the once we just build...)
-set +x # else there is too much debugging output
-BADDEBS=""
-for DEB in $(dcmd ls *.changes|egrep 'deb$' ) ; do
- SHASUM=$(sha256sum $DEB | awk '{ print $1 }')
- if grep $SHASUM $FILE.orig ; then
- # reproducible, yay!
- :
- # FIXME: NEXT: put this in the db and prevent rebuilds (way before)...
- else
- BADDEBS="$BADDEBS $DEB"
- fi
-done
-if [ -n "$BADDEBS" ] ; then
- output_echo "Unreproducible binary packages found:"
- for DEB in $BADDEBS ; do
- echo " $(egrep ' [a-z0-9]{64} ' $FILE.orig|grep $DEB | awk ' { print $1 " " $3 }') from ftp.debian.org"
- echo " $(sha256sum $DEB| sed 's# # #') from the current rebuild"
- echo "hmmmmmpppf."
- done
-fi
-
-# the end
-rm -f $DEBREBUILD
-output_echo "the end."
=====================================
bin/reproducible_html_nodes_info.sh
=====================================
@@ -1,7 +1,7 @@
#!/bin/bash
# vim: set noexpandtab:
-# Copyright © 2015-2020 Holger Levsen <holger at layer-acht.org>
+# Copyright © 2015-2021 Holger Levsen <holger at layer-acht.org>
# © 2018 Mattia Rizzolo <mattia at debian.org>
# released under the GPLv2
@@ -18,7 +18,7 @@ explain_nodes() {
case $JENKINS_NODENAME in
jenkins) ;;
ionos3) write_page "<br /><small>(twitterbot for irc:#reproducible-builds)</small>" ;;
- ionos7) write_page "<br /><small>(buildinfos.debian.net and debian rebuilder prototype)</small>" ;;
+ ionos7) write_page "<br /><small>(buildinfos.debian.net)</small>" ;;
ionos9) write_page "<br /><small>(jenkins.d.n rebootstrap jobs)</small>" ;;
ionos10) write_page "<br /><small>(jenkins.d.n chroot-installation jobs and http-proxy)</small>" ;;
osuosl167) write_page "<br /><small>(Debian janitor jobs and fakeroot foreign jobs and http-proxy for osuosl nodes)</small>" ;;
=====================================
bin/reproducible_maintenance.sh
=====================================
@@ -261,8 +261,7 @@ if [ ! -z "$http_proxy" ] ; then
pbuilder_http_proxy="--http-proxy $http_proxy"
fi
for s in $SUITES ; do
- # ionos7 is used also to run the prototype rebuilder, and it needs chdists
- for i in osuosl ionos3 ionos9 ionos10 ; do
+ for i in osuosl ionos3 ionos7 ionos9 ionos10 ; do
if [ "${HOSTNAME:0:${#i}}" = "$i" ]; then
# this node is not used to do Debian rebuilds, skip it all
continue 2
=====================================
job-cfg/reproducible.yaml
=====================================
@@ -788,12 +788,6 @@
my_description: 'Link files for buildinfos.debian.net from date to pool structure'
my_hname: 'ionos7'
my_timed: '10 */2 * * *'
- - 'debian_rebuilder_prototype':
- my_description: 'Prototype rebuilder for Debian'
- my_hname: 'ionos7'
- my_timed: ''
- my_parse_rules: '/srv/jenkins/logparse/reproducible.rules'
- my_disabled: true
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_hname: ''
my_timeout: '1440'
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/3814aa0073f681eeb2d1bdaa94c691986ac6384c
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/3814aa0073f681eeb2d1bdaa94c691986ac6384c
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20210930/906ba4b3/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list