[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] reproducible Debian, debrebuild: update code copy from updated...
Holger Levsen
gitlab at salsa.debian.org
Wed Dec 30 12:33:44 GMT 2020
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
2619b67b by Holger Levsen at 2020-12-30T13:33:35+01:00
reproducible Debian, debrebuild: update code copy from updated https://salsa.debian.org/debian/devscripts/-/merge_requests/212 - thanks josch
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/rb-debrebuild
Changes:
=====================================
bin/rb-debrebuild
=====================================
@@ -33,8 +33,10 @@ use File::Basename;
eval {
require LWP::Simple;
require LWP::UserAgent;
+ require URI::Escape; # libwww-perl depends on liburi-perl
no warnings;
- $LWP::Simple::ua = LWP::UserAgent->new(agent => 'LWP::UserAgent/srebuild');
+ $LWP::Simple::ua
+ = LWP::UserAgent->new(agent => 'LWP::UserAgent/debrebuild');
$LWP::Simple::ua->env_proxy();
};
if ($@) {
@@ -45,11 +47,11 @@ if ($@) {
}
}
-my $respect_build_path = 1;
-my $use_tor = 0;
-my $outdir = './';
-my $builder = 'none';
-my @required_timestamps = ();
+my $respect_build_path = 1;
+my $use_tor = 0;
+my $outdir = './';
+my $builder = 'none';
+my $timestamp = '';
my %OPTIONS = (
'help|h' => sub { usage(0); },
@@ -57,7 +59,7 @@ my %OPTIONS = (
'respect-build-path!' => \$respect_build_path,
'output|O=s' => \$outdir,
'builder=s' => \$builder,
- 'timestamp|t=s' => \@required_timestamps,
+ 'timestamp|t=s' => \$timestamp,
);
sub usage {
@@ -80,7 +82,9 @@ Options:
provided .buildinfo file.
--output, -O Directory for the build artifacts (default: ./)
--builder=BUILDER Which building software should be used. See section BUILDER
- --timestamp, -t The required timestamps from snapshot.d.o if you already know them, separated by commas
+ --timestamp, -t The required unstable main timestamps from snapshot.d.o if you
+ already know them, separated by commas, or one of the values
+ "first_seen" or "metasnap". See section TIMESTAMPS.
Note: $me can parse buildinfo files with and without a GPG signature. However,
the signature (if present) is discarded as debrebuild does not support verifying
@@ -120,6 +124,27 @@ bug #898446 for details. To enable user namespaces, run:
\$ sudo sysctl -w kernel.unprivileged_userns_clone=1
+TIMESTAMPS
+
+The --timestamp option allows one to skip the step of figuring out the correct
+set of required timestamps by listing them separated by commas in the same
+format used in the snapshot.d.o URL. The default is to use the "first_seen"
+attribute from the snapshot.d.o API and download multiple Packages files until
+all required timestamps are found. To explicitly select this mode, use
+--timestamp=first_seen. Lastly, the metasnap.d.n service can be used to figure
+out the right set of timestamps. This mode can be selected by using
+--timestamp=metasnap. In contrast to the "first_seen" mode, the metasnap.d.n
+service will always return a minimal set of timestamps if the package versions
+were at some point part of Debian unstable main.
+
+LIMITATIONS
+
+Currently, the code assumes that all packages were at some point part of Debian
+unstable main. This fails for packages from Debian ports, packages from
+experimental as well as for locally built packages or packages from third
+party repositories. Enabling support for Debian ports and experimental is
+conceptually possible and only needs somebody implementing it.
+
EOF
exit($exit_code);
@@ -127,9 +152,6 @@ EOF
GetOptions(%OPTIONS);
-# support timestamps being separated by a comma
- at required_timestamps = split(/,/, join(',', @required_timestamps));
-
my $buildinfo = shift @ARGV;
if (not defined($buildinfo)) {
print STDERR "ERROR: Missing mandatory buildinfo filename\n";
@@ -434,6 +456,50 @@ if (!defined($src_date)) {
die "cannot find .dsc\n";
}
+# support timestamps being separated by a comma
+my @required_timestamps = ();
+if ($timestamp eq "first_seen") {
+ # nothing to do, timestamps will be figured out later
+} elsif ($timestamp eq "metasnap") {
+ # acquire the required timestamps using metasnap.d.n
+ print "retrieving required timestamps from metasnap.d.n\n";
+ my $ua = LWP::UserAgent->new(timeout => 10);
+ $ua->env_proxy;
+ my @pkgs = ();
+ foreach my $pkg (@inst_build_deps) {
+ my $pkg_name = $pkg->{name};
+ my $pkg_ver = $pkg->{version};
+ my $pkg_arch = $pkg->{architecture};
+ if (defined $pkg_arch) {
+ push @pkgs,
+ URI::Escape::uri_escape("$pkg_name:$pkg_arch=$pkg_ver");
+ } else {
+ push @pkgs, URI::Escape::uri_escape("$pkg_name=$pkg_ver");
+ }
+ }
+ my $response
+ = $ua->get('https://metasnap.debian.net/cgi-bin/api'
+ . '?archive=debian'
+ . "&pkgs="
+ . (join "%2C", @pkgs)
+ . "&arch=$build_arch"
+ . '&suite=unstable'
+ . '&comp=main');
+ if (!$response->is_success) {
+ die "request to metasnap.d.n failed: $response->status_line";
+ }
+ foreach my $line (split /\n/, $response->decoded_content) {
+ my ($arch, $t) = split / /, $line, 2;
+ if ($arch ne $build_arch) {
+ die
+"debrebuild is currently unable to handle multiple architectures";
+ }
+ push @required_timestamps, $t;
+ }
+} else {
+ @required_timestamps = split(/,/, $timestamp);
+}
+
# setup a temporary apt directory
my $tempdir = tempdir(CLEANUP => 1);
@@ -505,7 +571,8 @@ open(FH, '>', $aptconf);
# commit 475f75506db48a7fa90711fce4ed129f6a14cc9a.
#
# Acquire::Check-Valid-Until has to be set to false because the snapshot
-# timestamps might be too far in the past to still be valid.
+# timestamps might be too far in the past to still be valid. This could be
+# fixed by a solution to https://bugs.debian.org/763419
#
# Acquire::Languages has to be set to prevent downloading of translations from
# the mirrors.
@@ -514,6 +581,14 @@ open(FH, '>', $aptconf);
# so that apt-get update fails if repositories cannot be authenticated. The
# default value of this option will change to true with apt from Debian
# Buster.
+#
+# We need APT::Get::allow-downgrades set to true, because even if we choose a
+# base distribution that was released before the state that "unstable"
+# currently is in, the package versions in that stable release might be newer
+# than what is in unstable due to security fixes. Choosing a stable release
+# from an older snapshot timestamp would fix this problem but would defeat the
+# purpose of a base distribution for builders like sbuild which can take
+# advantage of existing chroot environments.
print FH <<EOF;
Apt {
@@ -523,13 +598,19 @@ Apt {
Dir "$tempdir";
Dir::State::status "$tempdir/var/lib/dpkg/status";
-Acquire::Check-Valid-Until "false";
Acquire::Languages "none";
-Acquire::http::Dl-Limit "1000";
-Acquire::https::Dl-Limit "1000";
-Acquire::Retries "5";
Binary::apt-get::Acquire::AllowInsecureRepositories "false";
EOF
+my @common_aptopts = (
+ 'Acquire::Check-Valid-Until "false";',
+ 'Acquire::http::Dl-Limit "1000";',
+ 'Acquire::https::Dl-Limit "1000";',
+ 'Acquire::Retries "5";',
+ 'APT::Get::allow-downgrades "true";',
+);
+foreach my $line (@common_aptopts) {
+ print FH "$line\n";
+}
close FH;
# add the removed keys because they are not returned by Dpkg::Vendor
@@ -815,13 +896,6 @@ if ($builder ne "none") {
}
}
-my @aptopts = (
- 'Acquire::Check-Valid-Until "false";',
- 'Acquire::http::Dl-Limit "1000";',
- 'Acquire::https::Dl-Limit "1000";',
- 'Acquire::Retries "5";'
-);
-
if ($builder eq "none") {
print "\n";
print "Manual installation and build\n";
@@ -898,7 +972,7 @@ if ($builder eq "none") {
die "$config already exists -- refusing to overwrite\n";
}
open(FH, '>', $config) or die "cannot open $config: $!\n";
- foreach my $line (@aptopts) {
+ foreach my $line (@common_aptopts) {
print FH "$line\n";
}
close FH;
@@ -963,7 +1037,8 @@ if ($builder eq "none") {
'dpkg-buildpackage', '-uc', "--host-arch=$host_arch", "--build=$build"
or die "dpkg-buildpackage failed\n";
# we are not interested in the unpacked source directory
- 0 == system 'rm', '-r', $custom_build_path;
+ 0 == system 'rm', '-r', $custom_build_path
+ or die "failed to remove $custom_build_path: $?";
# but instead we want the produced artifacts
0 == system 'dcmd', 'mv',
(dirname $custom_build_path)
@@ -993,7 +1068,7 @@ if ($builder eq "none") {
# the user.
push @cmd,
'--chroot-setup-commands=echo '
- . (shellescape(join '\n', @aptopts))
+ . (shellescape(join '\n', @common_aptopts))
. ' | tee /etc/apt/apt.conf.d/23-debrebuild.conf';
my @add_depends = ();
@@ -1042,7 +1117,7 @@ if ($builder eq "none") {
push @cmd, "--build-path=$custom_build_path";
}
push @cmd, "${srcpkgname}_$srcpkgver";
- print(join " ", @cmd) . "\n";
+ print((join " ", @cmd) . "\n");
0 == system @cmd or die "sbuild failed\n";
} elsif ($builder eq "mmdebstrap") {
@@ -1091,10 +1166,7 @@ if ($builder eq "none") {
'mmdebstrap',
"--arch=$build_arch",
"--variant=apt",
- '--aptopt=Acquire::Check-Valid-Until "false"',
- '--aptopt=Acquire::http::Dl-Limit "1000";',
- '--aptopt=Acquire::https::Dl-Limit "1000";',
- '--aptopt=Acquire::Retries "5";',
+ (map { "--aptopt=$_" } @common_aptopts),
'--include=' . (join ' ', @install),
'--essential-hook=chroot "$1" sh -c "'
. (
@@ -1127,7 +1199,7 @@ if ($builder eq "none") {
'/dev/null',
"deb $base_mirror/$build_date/ $base_dist main"
);
- print(join ' ', @cmd) . "\n";
+ print((join ' ', @cmd) . "\n");
0 == system @cmd or die "mmdebstrap failed\n";
} else {
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/2619b67b36c77f2276eb46ee39ff7b8b97e443d2
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/2619b67b36c77f2276eb46ee39ff7b8b97e443d2
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20201230/19cb849d/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list