[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: reproducible Debian: update debrebuild to version from...
Holger Levsen
gitlab at salsa.debian.org
Thu Sep 10 22:15:05 BST 2020
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
6e318131 by Holger Levsen at 2020-09-10T23:10:00+02:00
reproducible Debian: update debrebuild to version from https://salsa.debian.org/debian/devscripts/-/merge_requests/203
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
872356ed by Holger Levsen at 2020-09-10T23:14:45+02:00
reproducible Debian: drop workaround for #955304 when using debrebuild
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
2 changed files:
- bin/rb-debrebuild
- bin/reproducible_debian_rebuilder_prototype.sh
Changes:
=====================================
bin/rb-debrebuild
=====================================
@@ -1,6 +1,7 @@
#!/usr/bin/perl
#
-# Copyright 2016 Johannes Schauer
+# Copyright © 2014-2016 Johannes Schauer <j.schauer at email.de>
+# Copyright © 2020 Niels Thykier <niels at thykier.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -14,6 +15,9 @@
use strict;
use warnings;
+use autodie;
+
+use Getopt::Long qw(:config gnu_getopt no_bundling no_auto_abbrev);
use Dpkg::Control;
use Dpkg::Index;
@@ -40,18 +44,97 @@ if ($@) {
}
}
+my $respect_build_path = 1;
+my $use_tor = 0;
+my $apt_tor_prefix = '';
+
+my %OPTIONS = (
+ 'help|h' => sub { usage(0); },
+ 'use-tor-proxy!' => \$use_tor,
+ 'respect-build-path!' => \$respect_build_path,
+);
+
+sub usage {
+ my ($exit_code) = @_;
+ my $me = basename($0);
+ $exit_code //= 0;
+ print <<EOF;
+Usage: $me [--no-respect-build-path] <buildinfo>
+ $me <--help|-h>
+
+Given a buildinfo file from a Debian package, generate instructions for
+attempting to reproduce the binary packages built from the associated source
+and build information.
+
+Options:
+ --help, -h Show this help and exit
+ --[no-]use-tor-proxy Whether to fetch resources via tor (socks://127.0.0.1:9050)
+ Assumes "apt-transport-tor" is installed both in host + chroot
+ --[no-]respect-build-path Whether to setup the build to use the Build-Path from the
+ provided .buildinfo file.
+
+Note: $me can parse buildinfo files with and without a GPG signature. However,
+the signature (if present) is discarded as debrebuild does not support verifying
+it. If the authenticity or integrity of the buildinfo files are important to
+you, checking these need to be done before invoking $me.
+EOF
+
+ exit($exit_code);
+}
+
+GetOptions(%OPTIONS);
+
my $buildinfo = shift @ARGV;
if (not defined($buildinfo)) {
- die "need buildinfo filename";
+ print STDERR "ERROR: Missing mandatory buildinfo filename\n";
+ print STDERR "\n";
+ usage(1);
+}
+if ($buildinfo eq '--help' or $buildinfo eq '-h') {
+ usage(0);
+}
+
+if ($buildinfo =~ m/^-/) {
+ print STDERR "ERROR: Unsupported option $buildinfo\n";
+ print STDERR "\n";
+ usage(1);
+}
+
+if (@ARGV) {
+ print STDERR "ERROR: This program requires exactly argument!\n";
+ print STDERR "\n";
+ usage(1);
+}
+
+if ($use_tor) {
+ $apt_tor_prefix = 'tor+';
+ eval {
+ $LWP::Simple::ua->proxy([qw(http https)] => 'socks://127.0.0.1:9050');
+ };
+ if ($@) {
+ if ($@ =~ m/Can\'t locate LWP/) {
+ die
+"Unable to use tor: the liblwp-protocol-socks-perl package is not installed\n";
+ } else {
+ die "Unable to use tor: Couldn't load socks proxy support: $@\n";
+ }
+ }
}
# buildinfo support in libdpkg-perl (>= 1.18.11)
-my $cdata = Dpkg::Control->new(type => CTRL_FILE_BUILDINFO);
+my $cdata = Dpkg::Control->new(type => CTRL_FILE_BUILDINFO, allow_pgp => 1);
if (not $cdata->load($buildinfo)) {
die "cannot load $buildinfo";
}
+if ($cdata->get_option('is_pgp_signed')) {
+ print
+"$buildinfo contained a GPG signature; it has NOT been validated (debrebuild does not support this)!\n";
+} else {
+ print "$buildinfo was unsigned\n";
+}
+
my @architectures = split /\s+/, $cdata->{"Architecture"};
my $build_source = (scalar(grep /^source$/, @architectures)) == 1;
my $build_archall = (scalar(grep /^all$/, @architectures)) == 1;
@@ -73,9 +156,50 @@ my $inst_build_deps = $cdata->{"Installed-Build-Depends"};
if (not defined($inst_build_deps)) {
die "need Installed-Build-Depends field";
}
+my $custom_build_path = $respect_build_path ? $cdata->{'Build-Path'} : undef;
+
+if (defined($custom_build_path)) {
+ if ($custom_build_path =~ m{['`\$\\"\(\)<>#]|(?:\a|/)[.][.](?:\z|/)}) {
+ warn(
+"Retry build with --no-respect-build-path to ignore the Build-Path field.\n"
+ );
+ die(
+"Refusing to use $custom_build_path as Build-Path: Looks too special to be true"
+ );
+ }
+
+ if ($custom_build_path eq '' or $custom_build_path !~ m{^/}) {
+ warn(
+"Retry build with --no-respect-build-path to ignore the Build-Path field.\n"
+ );
+ die(
+qq{Build-Path must be a non-empty absolute path (i.e. start with "/").\n}
+ );
+ }
+ print "Using defined Build-Path: ${custom_build_path}\n";
+} else {
+ if ($respect_build_path) {
+ print
+"No Build-Path defined; not setting a defined build path for this build.\n";
+ }
+}
+
+sub extract_source {
+ my ($source) = @_;
+ # In some cases such as binNMUs, the source field contains a version in
+ # the form:
+ # mscgen (0.20)
+ # This function strips the version leaving only the source package.
+ $source =~ s/\s+\(.+\)\s*//;
+ # Add a simple control check to avoid the worst surprises and stop obvious
+ # cases of garbage-in-garbage-out.
+ die("Unexpected source package name: ${source}\n")
+ if $source =~ m{[ \t_/\(\)<>!\n%&\$\#\@]};
+ return $source;
+}
my $srcpkg = Dpkg::Source::Package->new();
-$srcpkg->{fields}{'Source'} = $cdata->{"Source"};
+$srcpkg->{fields}{'Source'} = extract_source($cdata->{"Source"});
$srcpkg->{fields}{'Version'} = $cdata->{"Version"};
my $dsc_fname
= (dirname($buildinfo)) . '/' . $srcpkg->get_basename(1) . ".dsc";
@@ -205,7 +329,7 @@ foreach my $d ((
open(FH, '>', "$tempdir/etc/apt/sources.list");
print FH <<EOF;
-deb http://deb.debian.org/debian/ $base_dist main
+deb ${apt_tor_prefix}http://deb.debian.org/debian/ $base_dist main
EOF
close FH;
# FIXME - document what's dpkg's status for
@@ -214,7 +338,7 @@ open(FH, '>', "$tempdir/var/lib/dpkg/status");
close FH; #empty file
# Create apt.conf
my $aptconf = "$tempdir/etc/apt/apt.conf";
-open(FH, ">$aptconf");
+open(FH, '>', $aptconf);
# We create an apt.conf and pass it to apt via the APT_CONFIG environment
# variable instead of passing all options via the command line because
@@ -286,29 +410,37 @@ $ENV{'APT_CONFIG'} = $aptconf;
0 == system 'apt-get', 'update' or die "apt-get update failed\n";
-my $key_func = sub {
+sub dpkg_index_key_func {
return
$_[0]->{Package} . ' '
. $_[0]->{Version} . ' '
. $_[0]->{Architecture};
-};
-my $index = Dpkg::Index->new(get_key_func => $key_func);
+}
-open(my $fd, '-|', 'apt-get', 'indextargets', '--format', '$(FILENAME)',
- 'Created-By: Packages');
-while (my $fname = <$fd>) {
- chomp $fname;
- print "parsing $fname...\n";
- open(my $fd2, '-|', '/usr/lib/apt/apt-helper', 'cat-file', $fname);
- $index->parse($fd2, "pipe") or die "cannot parse Packages file\n";
- close($fd2);
+sub parse_all_packages_files {
+ my $dpkg_index = Dpkg::Index->new(get_key_func => \&dpkg_index_key_func);
+
+ open(my $fd, '-|', 'apt-get', 'indextargets', '--format', '$(FILENAME)',
+ 'Created-By: Packages');
+ while (my $fname = <$fd>) {
+ chomp $fname;
+ print "parsing $fname...\n";
+ open(my $fd2, '-|', '/usr/lib/apt/apt-helper', 'cat-file', $fname);
+ $dpkg_index->parse($fd2, "pipe") or die "cannot parse Packages file\n";
+ close($fd2);
+ }
+ close($fd);
+ return $dpkg_index;
}
-close($fd);
+
+my $index = parse_all_packages_files();
# go through all packages in the Installed-Build-Depends field and find out
# the timestamps at which they were first seen each
my %notfound_timestamps;
+my (%missing, $snapshots_needed);
+
foreach my $pkg (@inst_build_deps) {
my $pkg_name = $pkg->{name};
my $pkg_ver = $pkg->{version};
@@ -363,6 +495,10 @@ foreach my $pkg (@inst_build_deps) {
die
"package $pkg_name was implicitly requested for $pkg_arch but only $pkg->{architecture} was found\n";
}
+ # Ensure that $pkg_arch is defined from here as we want to look it up
+ # later in a Packages file from snapshot.d.o if it is not in the
+ # current Packages file
+ $pkg_arch = $pkg->{architecture};
} else {
# Since the package occurs more than once, we expect it to be of
# Architecture:any
@@ -395,8 +531,9 @@ foreach my $pkg (@inst_build_deps) {
die "no package with the right hash in Debian official\n";
}
my $date = $package_from_main[0]->{first_seen};
- $pkg->{first_seen} = $date;
- $notfound_timestamps{$date} = 1;
+ $pkg->{first_seen} = $date;
+ $notfound_timestamps{$date} = 1;
+ $missing{"${pkg_name}/${pkg_ver}/${pkg_arch}"} = 1;
}
# feed apt with timestamped snapshot.debian.org URLs until apt is able to find
@@ -416,22 +553,12 @@ while (0 < scalar keys %notfound_timestamps) {
my $snapshot_url = "http://snapshot.debian.org/archive/debian/$newest/";
open(FH, '>>', "$tempdir/etc/apt/sources.list");
- print FH "deb $snapshot_url unstable main\n";
+ print FH "deb ${apt_tor_prefix}${snapshot_url} unstable main\n";
close FH;
0 == system 'apt-get', 'update' or die "apt-get update failed";
- my $index = Dpkg::Index->new(get_key_func => $key_func);
- open(my $fd, '-|', 'apt-get', 'indextargets', '--format', '$(FILENAME)',
- 'Created-By: Packages');
- while (my $fname = <$fd>) {
- chomp $fname;
- print "parsing $fname...\n";
- open(my $fd2, '-|', '/usr/lib/apt/apt-helper', 'cat-file', $fname);
- $index->parse($fd2, "pipe") or die "cannot parse Packages file\n";
- close($fd2);
- }
- close($fd);
+ my $index = parse_all_packages_files();
foreach my $pkg (@inst_build_deps) {
my $pkg_name = $pkg->{name};
my $pkg_ver = $pkg->{version};
@@ -439,18 +566,34 @@ while (0 < scalar keys %notfound_timestamps) {
my $first_seen = $pkg->{first_seen};
my $cdata = $index->get_by_key("$pkg_name $pkg_ver $pkg_arch");
if (not defined($cdata->{"Package"})) {
- die "cannot find $pkg_name/$pkg_ver/$pkg_arch in dumpavail\n";
+ # Not present yet; we hope a later snapshot URL will locate it.
+ next;
}
+ $snapshots_needed = 1;
+ delete($missing{"${pkg_name}/${pkg_ver}/${pkg_arch}"});
if (defined $first_seen) {
delete $notfound_timestamps{$first_seen};
}
}
}
+if (%missing) {
+ print STDERR 'Cannot locate the following packages via snapshots'
+ . " or the current repo/mirror\n";
+ for my $key (sort(keys(%missing))) {
+ print STDERR " ${key}\n";
+ }
+ exit(1);
+}
+
print "\n";
print "Manual installation and build\n";
print "-----------------------------\n";
print "\n";
+if ($cdata->{"Binary-Only-Changes"}) {
+ print
+"The buildinfo appears to be for a binNMU; this is not fully supported yet.\n\n";
+}
print "The following sources.list contains all the required repositories:\n";
print "\n";
0 == system 'cat', "$tempdir/etc/apt/sources.list"
@@ -460,6 +603,13 @@ print "You can manually install the right dependencies like this:\n";
print "\n";
print "apt-get install --no-install-recommends";
+if ($snapshots_needed) {
+ # Release files from snapshots.d.o have often expired by the time
+ # we fetch them. Include the option to work around that to assist
+ # the user.
+ print " -oAcquire::Check-Valid-Until=false";
+}
+
foreach my $pkg (@inst_build_deps) {
my $pkg_name = $pkg->{name};
my $pkg_ver = $pkg->{version};
@@ -474,13 +624,29 @@ print "\n";
print "\n";
print "And then build your package:\n";
print "\n";
-print "dpkg-source -x $dsc_fname\n";
-print "cd packagedirectory\n";
+if ($custom_build_path) {
+ require Cwd;
+ my $custom_build_parent_dir = dirname($custom_build_path);
+ my $dsc_path = Cwd::realpath($dsc_fname)
+ // die("Cannot resolve ${dsc_fname}: $!\n");
+ print "mkdir -p \"${custom_build_parent_dir}\"\n";
+ print qq{dpkg-source -x "${dsc_path}" "${custom_build_path}"\n};
+ print "cd \"$custom_build_path\"\n";
+} else {
+ print qq{dpkg-source -x "${dsc_fname}"\n};
+ print "cd packagedirectory\n";
+}
print "$environment dpkg-buildpackage\n";
print "\n";
print "Using sbuild\n";
print "------------\n";
print "\n";
+
+if ($cdata->{"Binary-Only-Changes"}) {
+ print
+"The buildinfo appears to be for a binNMU; this is not fully supported yet.\n\n";
+}
+
print "You can try to build the package with sbuild like this:\n";
print "\n";
print "SBUILD_CMDLINE=$environment sbuild";
@@ -491,6 +657,13 @@ while (my $line = <FH>) {
print " --extra-repository=\"$line\"";
}
close FH;
+if ($snapshots_needed) {
+ # Release files from snapshots.d.o have often expired by the time
+ # we fetch them. Include the option to work around that to assist
+ # the user.
+ print
+q{--chroot-setup-commands='echo "Acquire::Check-Valid-Until \"false\";" | tee /etc/apt/apt.conf.d/23-debrebuild'};
+}
my @add_depends = ();
foreach my $pkg (@inst_build_deps) {
my $pkg_name = $pkg->{name};
@@ -524,5 +697,9 @@ if ($build_archall) {
print " --no-arch-all";
}
print " -d $base_dist";
+print " --no-run-lintian";
+if ($custom_build_path) {
+ print " --build-path='${custom_build_path}'";
+}
print " $dsc_fname\n";
print "BASE_DIST=$base_dist\n";
=====================================
bin/reproducible_debian_rebuilder_prototype.sh
=====================================
@@ -210,12 +210,10 @@ mv $FILE $FILE.orig
# actually run sbuild
# - workaround #955123 in devscripts: debrebuild: please provide --sbuild-output-only option
# - using tail
-# - workaround #955304 in devscripts: debrebuild: suggested sbuild command should use --no-run-lintian
-# - using sed
# - workaround yet unfiled bug in devscript by tail -2 | grep -v BASE_DIST... (see above)
# - no solution yet to rebuild using the same path as the original build: see #964722
output_echo "trying to re-sbuild $PKG..."
-SBUILD=$(tail -2 $DEBREBUILD | grep '^SBUILD_CMDLINE=' | cut -d '=' -f2- | sed 's# sbuild # sbuild --no-run-lintian #')
+SBUILD=$(tail -2 $DEBREBUILD | grep '^SBUILD_CMDLINE=' | cut -d '=' -f2- )
output_echo "using this sbuild command line:"
echo $SBUILD
echo
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/1c9e36305efd2ab382471756fcb62eab8ed1ca8f...872356eda87d198bf0273d778dce7e3c2b9f8aeb
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/1c9e36305efd2ab382471756fcb62eab8ed1ca8f...872356eda87d198bf0273d778dce7e3c2b9f8aeb
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20200910/0f5098ce/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list