[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: reproducible Debian: debrebuild as shipped in devscripts 2.20.2
Holger Levsen
gitlab at salsa.debian.org
Tue Mar 31 23:58:11 BST 2020
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
a4d3e0bc by Holger Levsen at 2020-04-01T00:52:16+02:00
reproducible Debian: debrebuild as shipped in devscripts 2.20.2
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
4119fa16 by Holger Levsen at 2020-04-01T00:57:58+02:00
reproducible Debian: take patch from devscripts 2.20.3: debrebuild: stop using reproducible-builds.org apt repo. Closes: #955280
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- + bin/rb-debrebuild
Changes:
=====================================
bin/rb-debrebuild
=====================================
@@ -0,0 +1,525 @@
+#!/usr/bin/perl
+#
+# Copyright 2016 Johannes Schauer
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+use strict;
+use warnings;
+
+use Dpkg::Control;
+use Dpkg::Index;
+use Dpkg::Deps;
+use Dpkg::Source::Package;
+use File::Temp qw(tempdir);
+use File::Path qw(make_path);
+use JSON::PP;
+use Time::Piece;
+use File::Basename;
+
+eval {
+ require LWP::Simple;
+ require LWP::UserAgent;
+ no warnings;
+ $LWP::Simple::ua = LWP::UserAgent->new(agent => 'LWP::UserAgent/srebuild');
+ $LWP::Simple::ua->env_proxy();
+};
+if ($@) {
+ if ($@ =~ m/Can\'t locate LWP/) {
+ die "Unable to run: the libwww-perl package is not installed";
+ } else {
+ die "Unable to run: Couldn't load LWP::Simple: $@";
+ }
+}
+
+my $buildinfo = shift @ARGV;
+if (not defined($buildinfo)) {
+ die "need buildinfo filename";
+}
+
+# buildinfo support in libdpkg-perl (>= 1.18.11)
+my $cdata = Dpkg::Control->new(type => CTRL_FILE_BUILDINFO);
+
+if (not $cdata->load($buildinfo)) {
+ die "cannot load $buildinfo";
+}
+
+my @architectures = split /\s+/, $cdata->{"Architecture"};
+my $build_source = (scalar(grep /^source$/, @architectures)) == 1;
+my $build_archall = (scalar(grep /^all$/, @architectures)) == 1;
+ at architectures = grep { !/^source$/ && !/^all$/ } @architectures;
+if (scalar @architectures > 1) {
+ die "more than one architecture in Architecture field";
+}
+my $build_archany = (scalar @architectures) == 1;
+my $host_arch = undef;
+if ($build_archany) {
+ $host_arch = $architectures[0];
+}
+
+my $build_arch = $cdata->{"Build-Architecture"};
+if (not defined($build_arch)) {
+ die "need Build-Architecture field";
+}
+my $inst_build_deps = $cdata->{"Installed-Build-Depends"};
+if (not defined($inst_build_deps)) {
+ die "need Installed-Build-Depends field";
+}
+
+my $srcpkg = Dpkg::Source::Package->new();
+$srcpkg->{fields}{'Source'} = $cdata->{"Source"};
+$srcpkg->{fields}{'Version'} = $cdata->{"Version"};
+my $dsc_fname
+ = (dirname($buildinfo)) . '/' . $srcpkg->get_basename(1) . ".dsc";
+
+my $environment = $cdata->{"Environment"};
+if (not defined($environment)) {
+ die "need Environment field";
+}
+$environment =~ s/\n/ /g; # remove newlines
+$environment =~ s/^ //; # remove leading whitespace
+
+my $checksums = Dpkg::Checksums->new();
+$checksums->add_from_control($cdata);
+my @files = $checksums->get_files();
+
+# gather all installed build-depends and figure out the version of base-files
+# and dpkg
+my $base_files_version;
+my $dpkg_version;
+my @inst_build_deps = ();
+$inst_build_deps
+ = deps_parse($inst_build_deps, reduce_arch => 0, build_dep => 0);
+if (!defined $inst_build_deps) {
+ die "deps_parse failed\n";
+}
+
+foreach my $pkg ($inst_build_deps->get_deps()) {
+ if (!$pkg->isa('Dpkg::Deps::Simple')) {
+ die "dependency disjunctions are not allowed\n";
+ }
+ if (not defined($pkg->{package})) {
+ die "name undefined";
+ }
+ if (defined($pkg->{relation})) {
+ if ($pkg->{relation} ne "=") {
+ die "wrong relation";
+ }
+ if (not defined($pkg->{version})) {
+ die "version undefined";
+ }
+ } else {
+ die "no version";
+ }
+ if ($pkg->{package} eq "dpkg") {
+ if (defined($dpkg_version)) {
+ die "more than one dpkg\n";
+ }
+ $dpkg_version = $pkg->{version};
+ }
+ if ($pkg->{package} eq "base-files") {
+ if (defined($base_files_version)) {
+ die "more than one base-files\n";
+ }
+ $base_files_version = $pkg->{version};
+ }
+ push @inst_build_deps,
+ {
+ name => $pkg->{package},
+ architecture => $pkg->{archqual},
+ version => $pkg->{version} };
+}
+
+if (!defined($base_files_version)) {
+ die "no base-files\n";
+}
+if (!defined($dpkg_version)) {
+ die "no dpkg\n";
+}
+
+# figure out the debian release from the version of base-files and dpkg
+# FIXME - this really shouldn't be hardcoded here
+my $base_dist;
+
+my %base_files_map = (
+ "6" => "squeeze",
+ "7" => "wheezy",
+ "8" => "jessie",
+ "9" => "stretch",
+ "10" => "buster",
+ "11" => "bullseye",
+);
+my %dpkg_map = (
+ "15" => "squeeze",
+ "16" => "wheezy",
+ "17" => "jessie",
+ "18" => "stretch",
+ "19" => "buster",
+ "20" => "bullseye",
+);
+
+$base_files_version =~ s/^(\d+).*/$1/;
+#$dpkg_version =~ s/1\.(\d+)\..*/$1/;
+
+$base_dist = $base_files_map{$base_files_version};
+
+if (!defined $base_dist) {
+ die "base-files version didn't map to any Debian release";
+}
+
+#if ($base_dist ne $dpkg_map{$dpkg_version}) {
+# die "base-files and dpkg versions point to different Debian releases\n";
+#}
+
+# test if all checksums in the buildinfo file check out
+
+#foreach my $fname ($checksums->get_files()) {
+# # Re-adding existing files to the checksum object is the current way to
+# # ask Dpkg to check the checksums for us
+# $checksums->add_from_file((dirname($buildinfo)) . '/' . $fname);
+#}
+
+# setup a temporary apt directory
+
+my $tempdir = tempdir(CLEANUP => 1);
+
+foreach my $d ((
+ '/etc/apt', '/etc/apt/apt.conf.d',
+ '/etc/apt/preferences.d', '/etc/apt/trusted.gpg.d',
+ '/etc/apt/sources.list.d', '/var/lib/apt/lists/partial',
+ '/var/cache/apt/archives/partial', '/var/lib/dpkg',
+ )
+) {
+ make_path("$tempdir/$d");
+}
+
+open(FH, '>', "$tempdir/etc/apt/sources.list");
+print FH <<EOF;
+deb http://httpredir.debian.org/debian/ $base_dist main
+EOF
+close FH;
+# FIXME - document what's dpkg's status for
+# Create dpkg status
+open(FH, '>', "$tempdir/var/lib/dpkg/status");
+close FH; #empty file
+# Create apt.conf
+my $aptconf = "$tempdir/etc/apt/apt.conf";
+open(FH, ">$aptconf");
+
+# We create an apt.conf and pass it to apt via the APT_CONFIG environment
+# variable instead of passing all options via the command line because
+# otherwise apt will read the system's config first and might get unwanted
+# configuration options from there. See apt.conf(5) for the order in which
+# configuration options are read.
+#
+# While we are at it, we also set all other options through our custom
+# apt.conf.
+#
+# Apt::Architecture has to be set because otherwise apt will default to the
+# architecture apt was compiled for.
+#
+# Apt::Architectures has to be set or otherwise apt will use dpkg to find all
+# foreign architectures of the system running apt.
+#
+# Dir::State::status has to be set even though Dir is set because Dir::State
+# is set to var/lib/apt, so Dir::State::status would be below that but really
+# isn't and without an absolute path, Dir::State::status would be constructed
+# from Dir + Dir::State + Dir::State::status. This has been fixed in apt
+# commit 475f75506db48a7fa90711fce4ed129f6a14cc9a.
+#
+# Acquire::Check-Valid-Until has to be set to false because the snapshot
+# timestamps might be too far in the past to still be valid.
+#
+# Acquire::Languages has to be set to prevent downloading of translations from
+# the mirrors.
+#
+# Binary::apt-get::Acquire::AllowInsecureRepositories has to be set to false
+# so that apt-get update fails if repositories cannot be authenticated. The
+# default value of this option will change to true with apt from Debian
+# Buster.
+
+print FH <<EOF;
+Apt {
+ Architecture "$build_arch";
+ Architectures "$build_arch";
+};
+
+Dir "$tempdir";
+Dir::State::status "$tempdir/var/lib/dpkg/status";
+Acquire::Check-Valid-Until "false";
+Acquire::Languages "none";
+Binary::apt-get::Acquire::AllowInsecureRepositories "false";
+EOF
+close FH;
+
+# add the removed keys because they are not returned by Dpkg::Vendor
+# we don't need the Ubuntu vendor now but we already put the comments to
+# possibly extend this script to other Debian derivatives
+my @keyrings = ();
+my $debianvendor = Dpkg::Vendor::Debian->new();
+push @keyrings, $debianvendor->run_hook('archive-keyrings');
+push @keyrings, $debianvendor->run_hook('archive-keyrings-historic');
+#my $ubuntuvendor = Dpkg::Vendor::Ubuntu->new();
+#push @keyrings, $ubuntuvendor->run_hook('archive-keyrings');
+#push @keyrings, $ubuntuvendor->run_hook('archive-keyrings-historic');
+
+foreach my $keyring (@keyrings) {
+ my $base = basename $keyring;
+ print "$keyring\n";
+ if (-f $keyring) {
+ print "linking $tempdir/etc/apt/trusted.gpg.d/$base to $keyring\n";
+ symlink $keyring, "$tempdir/etc/apt/trusted.gpg.d/$base";
+ }
+}
+
+$ENV{'APT_CONFIG'} = $aptconf;
+
+0 == system 'apt-get', 'update' or die "apt-get update failed\n";
+
+my $key_func = sub {
+ return
+ $_[0]->{Package} . ' '
+ . $_[0]->{Version} . ' '
+ . $_[0]->{Architecture};
+};
+my $index = Dpkg::Index->new(get_key_func => $key_func);
+
+open(my $fd, '-|', 'apt-get', 'indextargets', '--format', '$(FILENAME)',
+ 'Created-By: Packages');
+while (my $fname = <$fd>) {
+ chomp $fname;
+ print "parsing $fname...\n";
+ open(my $fd2, '-|', '/usr/lib/apt/apt-helper', 'cat-file', $fname);
+ $index->parse($fd2, "pipe") or die "cannot parse Packages file\n";
+ close($fd2);
+}
+close($fd);
+
+# go through all packages in the Installed-Build-Depends field and find out
+# the timestamps at which they were first seen each
+my %notfound_timestamps;
+
+foreach my $pkg (@inst_build_deps) {
+ my $pkg_name = $pkg->{name};
+ my $pkg_ver = $pkg->{version};
+ my $pkg_arch = $pkg->{architecture};
+
+ # check if we really need to acquire this package from snapshot.d.o or if
+ # it already exists in the cache
+ if (defined $pkg->{architecture}) {
+ if ($index->get_by_key("$pkg_name $pkg_ver $pkg_arch")) {
+ print "skipping $pkg_name $pkg_ver\n";
+ next;
+ }
+ } else {
+ if ($index->get_by_key("$pkg_name $pkg_ver $build_arch")) {
+ $pkg->{architecture} = $build_arch;
+ print "skipping $pkg_name $pkg_ver\n";
+ next;
+ }
+ if ($index->get_by_key("$pkg_name $pkg_ver all")) {
+ $pkg->{architecture} = "all";
+ print "skipping $pkg_name $pkg_ver\n";
+ next;
+ }
+ }
+
+ print "retrieving snapshot.d.o data for $pkg_name $pkg_ver\n";
+ my $json_url
+ = "http://snapshot.debian.org/mr/binary/$pkg_name/$pkg_ver/binfiles?fileinfo=1";
+ my $content = LWP::Simple::get($json_url);
+ die "cannot retrieve $json_url" unless defined $content;
+ my $json = JSON::PP->new();
+ # json options taken from debsnap
+ my $json_text = $json->allow_nonref->utf8->relaxed->decode($content);
+ die "cannot decode json" unless defined $json_text;
+ my $pkg_hash;
+ if (scalar @{ $json_text->{result} } == 1) {
+ # if there is only a single result, then the package must either be
+ # Architecture:all, be the build architecture or match the requested
+ # architecture
+ $pkg_hash = ${ $json_text->{result} }[0]->{hash};
+ $pkg->{architecture} = ${ $json_text->{result} }[0]->{architecture};
+ # if a specific architecture was requested, it should match
+ if (defined $pkg_arch && $pkg_arch ne $pkg->{architecture}) {
+ die
+"package $pkg_name was explicitly requested for $pkg_arch but only $pkg->{architecture} was found\n";
+ }
+ # if no specific architecture was requested, it should be the build
+ # architecture
+ if ( !defined $pkg_arch
+ && $build_arch ne $pkg->{architecture}
+ && "all" ne $pkg->{architecture}) {
+ die
+"package $pkg_name was implicitly requested for $pkg_arch but only $pkg->{architecture} was found\n";
+ }
+ } else {
+ # Since the package occurs more than once, we expect it to be of
+ # Architecture:any
+ #
+ # If no specific architecture was requested, look for the build
+ # architecture
+ if (!defined $pkg_arch) {
+ $pkg_arch = $build_arch;
+ }
+ foreach my $result (@{ $json_text->{result} }) {
+ if ($result->{architecture} eq $pkg_arch) {
+ $pkg_hash = $result->{hash};
+ last;
+ }
+ }
+ if (!defined($pkg_hash)) {
+ die "cannot find package in architecture $pkg_arch\n";
+ }
+ # we now know that this package is not architecture:all but has a
+ # concrete architecture
+ $pkg->{architecture} = $pkg_arch;
+ }
+ # FIXME - assumption: package is from Debian official (and not ports)
+ my @package_from_main = grep { $_->{archive_name} eq "debian" }
+ @{ $json_text->{fileinfo}->{$pkg_hash} };
+ if (scalar @package_from_main > 1) {
+ die "more than one package with the same hash in Debian official\n";
+ }
+ if (scalar @package_from_main == 0) {
+ die "no package with the right hash in Debian official\n";
+ }
+ my $date = $package_from_main[0]->{first_seen};
+ $pkg->{first_seen} = $date;
+ $notfound_timestamps{$date} = 1;
+}
+
+# feed apt with timestamped snapshot.debian.org URLs until apt is able to find
+# all the required package versions. We start with the most recent timestamp,
+# check which packages cannot be found at that timestamp, add the timestamp of
+# the most recent not-found package and continue doing this iteratively until
+# all versions can be found.
+
+while (0 < scalar keys %notfound_timestamps) {
+ print "left to check: " . (scalar keys %notfound_timestamps) . "\n";
+ my @timestamps = map { Time::Piece->strptime($_, '%Y%m%dT%H%M%SZ') }
+ (sort keys %notfound_timestamps);
+ my $newest = $timestamps[$#timestamps];
+ $newest = $newest->strftime("%Y%m%dT%H%M%SZ");
+ delete $notfound_timestamps{$newest};
+
+ my $snapshot_url = "http://snapshot.debian.org/archive/debian/$newest/";
+
+ open(FH, '>>', "$tempdir/etc/apt/sources.list");
+ print FH "deb $snapshot_url unstable main\n";
+ close FH;
+
+ 0 == system 'apt-get', 'update' or die "apt-get update failed";
+
+ my $index = Dpkg::Index->new(get_key_func => $key_func);
+ open(my $fd, '-|', 'apt-get', 'indextargets', '--format', '$(FILENAME)',
+ 'Created-By: Packages');
+ while (my $fname = <$fd>) {
+ chomp $fname;
+ print "parsing $fname...\n";
+ open(my $fd2, '-|', '/usr/lib/apt/apt-helper', 'cat-file', $fname);
+ $index->parse($fd2, "pipe") or die "cannot parse Packages file\n";
+ close($fd2);
+ }
+ close($fd);
+ foreach my $pkg (@inst_build_deps) {
+ my $pkg_name = $pkg->{name};
+ my $pkg_ver = $pkg->{version};
+ my $pkg_arch = $pkg->{architecture};
+ my $first_seen = $pkg->{first_seen};
+ my $cdata = $index->get_by_key("$pkg_name $pkg_ver $pkg_arch");
+ if (not defined($cdata->{"Package"})) {
+ die "cannot find $pkg_name/$pkg_ver/$pkg_arch in dumpavail\n";
+ }
+ if (defined $first_seen) {
+ delete $notfound_timestamps{$first_seen};
+ }
+ }
+}
+
+print "\n";
+print "Manual installation and build\n";
+print "-----------------------------\n";
+print "\n";
+print "The following sources.list contains all the required repositories:\n";
+print "\n";
+0 == system 'cat', "$tempdir/etc/apt/sources.list"
+ or die "cannot cat $tempdir/etc/apt/sources.list";
+print "\n";
+print "You can manually install the right dependencies like this:\n";
+print "\n";
+print "apt-get install --no-install-recommends";
+
+foreach my $pkg (@inst_build_deps) {
+ my $pkg_name = $pkg->{name};
+ my $pkg_ver = $pkg->{version};
+ my $pkg_arch = $pkg->{architecture};
+ if ($pkg_arch eq "all" || $pkg_arch eq $build_arch) {
+ print " $pkg_name=$pkg_ver";
+ } else {
+ print " $pkg_name:$pkg_arch=$pkg_ver";
+ }
+}
+print "\n";
+print "\n";
+print "And then build your package:\n";
+print "\n";
+print "dpkg-source -x $dsc_fname\n";
+print "cd packagedirectory\n";
+print "$environment dpkg-buildpackage\n";
+print "\n";
+print "Using sbuild\n";
+print "------------\n";
+print "\n";
+print "You can try to build the package with sbuild like this:\n";
+print "\n";
+print "$environment sbuild";
+open(FH, '<', "$tempdir/etc/apt/sources.list");
+
+while (my $line = <FH>) {
+ chomp $line;
+ print " --extra-repository=\"$line\"";
+}
+close FH;
+my @add_depends = ();
+foreach my $pkg (@inst_build_deps) {
+ my $pkg_name = $pkg->{name};
+ my $pkg_ver = $pkg->{version};
+ my $pkg_arch = $pkg->{architecture};
+ if ($pkg_arch eq "all" || $pkg_arch eq $build_arch) {
+ push @add_depends, "$pkg_name (= $pkg_ver)";
+ } else {
+ push @add_depends, "$pkg_name:$pkg_arch (= $pkg_ver)";
+ }
+}
+print " --add-depends=\"" . (join ",", @add_depends) . "\"";
+print " --build-dep-resolver=aptitude";
+print " --build=$build_arch";
+if (defined $host_arch) {
+ print " --host=$host_arch";
+}
+if ($build_source) {
+ print " --source";
+} else {
+ print " --no-source";
+}
+if ($build_archany) {
+ print " --arch-any";
+} else {
+ print " --no-arch-any";
+}
+if ($build_archall) {
+ print " --arch-all";
+} else {
+ print " --no-arch-all";
+}
+print " -d $base_dist";
+print " $dsc_fname\n";
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/e052157165c20f04f0a217af182131a9fad4757d...4119fa167d09920cca5f257f2777030c9dc090bb
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/e052157165c20f04f0a217af182131a9fad4757d...4119fa167d09920cca5f257f2777030c9dc090bb
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20200331/a058ba77/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list