[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] reproducible openwrt: add snapshot build script
Holger Levsen
gitlab at salsa.debian.org
Wed Sep 25 21:10:23 BST 2019
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
034491ea by Paul Spooren at 2019-09-25T20:09:58Z
reproducible openwrt: add snapshot build script
The script downloads all buildinfo and checksums from the OpenWrt
download server and tries to rebuild them. Once done the upstream
checksums are used to verify the newly created files.
A website based on templates-mustache/openwrt/ is rendered.
Use locally installed diffoscope.
Store created and origin packages in database.
Signed-off-by: Paul Spooren <mail at aparcar.org>
- - - - -
6 changed files:
- bin/reproducible_openwrt_package_parser.py
- + bin/reproducible_openwrt_rebuild.py
- + mustache-templates/openwrt/footer.mustache
- + mustache-templates/openwrt/header.mustache
- + mustache-templates/openwrt/index.mustache
- + mustache-templates/openwrt/target.mustache
Changes:
=====================================
bin/reproducible_openwrt_package_parser.py
=====================================
@@ -21,7 +21,7 @@ def parse_packages(package_list_fp):
packages = []
for line in package_list_fp:
- if line == '\n':
+ if line == '\n' or line == '':
parser = email.parser.Parser()
package = parser.parsestr(linebuffer)
packages.append(package)
=====================================
bin/reproducible_openwrt_rebuild.py
=====================================
@@ -0,0 +1,322 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import pystache
+import subprocess
+import hashlib
+from urllib.request import urlopen
+from tempfile import mkdtemp, NamedTemporaryFile
+from multiprocessing import cpu_count
+from multiprocessing import Pool
+from time import strftime, gmtime
+import shutil
+import importlib
+
+# target to be build
+target = os.environ.get("TARGET", "ath79/generic")
+# version to be build
+version = os.environ.get("VERSION", "SNAPSHOT")
+# where to store rendered html and diffoscope output
+output_dir = os.environ.get("OUTPUT_DIR", "/srv/reproducible-results")
+# where to (re)build openwrt
+temporary_dir = os.environ.get("TMP_DIR", mkdtemp(dir="/srv/workspace/chroots/"))
+# where to find mustache templates
+template_dir = os.environ.get(
+ "TEMPLATE_DIR", "/srv/jenkins/mustache-templates/reproducible"
+)
+# where to find the origin builds
+openwrt_url = (
+ os.environ.get("ORIGIN_URL", "https://downloads.openwrt.org/snapshots/targets/")
+ + target
+)
+
+# dir of the version + target
+target_dir = os.path.join(output_dir, version, target)
+# dir where openwrt actually stores binary files
+rebuild_dir = temporary_dir + "/bin/targets/" + target
+# where to get the openwrt source git
+openwrt_git = os.environ.get("OPENWRT_GIT", "https://github.com/openwrt/openwrt.git")
+
+# run a command in shell
+def run_command(cmd, cwd=".", ignore_errors=False):
+ print("Running {} in {}".format(cmd, cwd))
+ proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE)
+ response = ""
+ # print and store the output at the same time
+ while True:
+ line = proc.stdout.readline().decode("utf-8")
+ if line == "" and proc.poll() != None:
+ break
+ response += line
+ print(line, end="", flush=True)
+
+ if proc.returncode and not ignore_errors:
+ print("Error running {}".format(cmd))
+ quit()
+ return response
+
+
+# files not to check via diffoscope
+meta_files = re.compile(
+ "|".join(
+ [
+ ".+\.buildinfo",
+ ".+\.manifest",
+ "openwrt-imagebuilder",
+ "openwrt-sdk",
+ "sha256sums",
+ "kernel-debug.tar.bz2",
+ ]
+ )
+)
+
+# the context to fill the mustache tempaltes
+context = {
+ "targets": [
+ {"version": "SNAPSHOT", "name": "ath79/generic"},
+ {"version": "SNAPSHOT", "name": "x86/64"},
+ {"version": "SNAPSHOT", "name": "ramips/mt7621"},
+ ],
+ "version": version,
+ "commit_string": "",
+ "images_repro": 0,
+ "images_repro_percent": 0,
+ "images_total": 0,
+ "packages_repro": 0,
+ "packages_repro_percent": 0,
+ "packages_total": 0,
+ "today": strftime("%Y-%m-%d", gmtime()),
+ "diffoscope_version": run_command(["diffoscope", "--version"]).split()[1],
+ "target": target,
+ "images": [],
+ "packages": [],
+ "git_log_oneline": "",
+ "missing": [],
+}
+
+# download file from openwrt server and compare it, store output in target_dir
+def diffoscope(origin_name):
+ file_origin = NamedTemporaryFile()
+
+ if get_file(openwrt_url + "/" + origin_name, file_origin.name):
+ print("Error downloading {}".format(origin_name))
+ return
+
+ run_command(
+ [
+ "diffoscope",
+ file_origin.name,
+ rebuild_dir + "/" + origin_name,
+ "--html",
+ target_dir + "/" + origin_name + ".html",
+ ],
+ ignore_errors=True,
+ )
+ file_origin.close()
+
+# return sha256sum of given path
+def sha256sum(path):
+ with open(path, "rb") as hash_file:
+ return hashlib.sha256(hash_file.read()).hexdigest()
+
+
+# return content of online file or stores it locally if path is given
+def get_file(url, path=None):
+ print("downloading {}".format(url))
+ try:
+ content = urlopen(url).read()
+ except:
+ return 1
+
+ if path:
+ print("storing to {}".format(path))
+ with open(path, "wb") as file_b:
+ file_b.write(content)
+ return 0
+ else:
+ return content.decode("utf-8")
+
+# parse the origin sha256sums file from openwrt
+def parse_origin_sha256sums():
+ sha256sums = get_file(openwrt_url + "/sha256sums")
+ return re.findall(r"(.+?) \*(.+?)\n", sha256sums)
+
+
+# not required for now
+# def exchange_signature(origin_path, rebuild_path):
+# file_sig = NamedTemporaryFile()
+# # extract original signatur in temporary file
+# run_command(
+# "./staging_dir/host/bin/fwtool -s {} {}".format(file_sig.name, origin_path),
+# temporary_dir,
+# )
+# # remove random signatur of rebuild
+# run_command(
+# "./staging_dir/host/bin/fwtool -t -s /dev/null {}".format(rebuild_path),
+# temporary_dir,
+# )
+# # add original signature to rebuild file
+# run_command(
+# "./staging_dir/host/bin/fwtool -S {} {}".format(file_sig.name, rebuild_path),
+# temporary_dir,
+# )
+# file_sig.close()
+
+
+# initial clone of openwrt.git
+run_command(["git", "clone", openwrt_git, temporary_dir])
+
+# download buildinfo files
+get_file(openwrt_url + "/config.buildinfo", temporary_dir + "/.config")
+with open(temporary_dir + "/.config", "a") as config_file:
+ # extra options used by the buildbot
+ config_file.writelines(
+ [
+ "CONFIG_CLEAN_IPKG=y\n",
+ "CONFIG_TARGET_ROOTFS_TARGZ=y\n",
+ "CONFIG_CLEAN_IPKG=y\n",
+ 'CONFIG_KERNEL_BUILD_USER="builder"\n',
+ 'CONFIG_KERNEL_BUILD_DOMAIN="buildhost"\n',
+ ]
+ )
+
+# insecure private key to build the images
+with open(temporary_dir + "/key-build", "w") as key_build_file:
+ key_build_file.write(
+ "Local build key\nRWRCSwAAAAB12EzgExgKPrR4LMduadFAw1Z8teYQAbg/EgKaN9SUNrgteVb81/bjFcvfnKF7jS1WU8cDdT2VjWE4Cp4cxoxJNrZoBnlXI+ISUeHMbUaFmOzzBR7B9u/LhX3KAmLsrPc="
+ )
+
+# spoof the official openwrt public key to prevent adding another key in the binary
+with open(temporary_dir + "/key-build.pub", "w") as key_build_pub_file:
+ key_build_pub_file.write(
+ "OpenWrt snapshot release signature\nRWS1BD5w+adc3j2Hqg9+b66CvLR7NlHbsj7wjNVj0XGt/othDgIAOJS+"
+ )
+# this specific key is odly chmodded to 600
+os.chmod(temporary_dir + "/key-build.pub", 0o600)
+
+# download origin buildinfo file containing the feeds
+get_file(openwrt_url + "/feeds.buildinfo", temporary_dir + "/feeds.conf")
+
+# get current commit_string to show in website banner
+context["commit_string"] = get_file(openwrt_url + "/version.buildinfo")[:-1]
+# ... and parse the actual commit to checkout
+commit = context["commit_string"].split("-")[1]
+
+# checkout the desired commit
+run_command(["git", "checkout", "-f", commit, temporary_dir])
+
+# show the last 20 commit to have an idea what was changed lately
+context["git_log_oneline"] = run_command(
+ ["git", "log", "--oneline", "-n", "20"], temporary_dir
+)
+
+# do as the buildbots do
+run_command(["./scripts/feeds", "update"], temporary_dir)
+run_command(["./scripts/feeds", "install", "-a"], temporary_dir)
+run_command(["make", "defconfig"], temporary_dir)
+# actually build everything
+run_command(
+ ["make", "IGNORE_ERRORS='n m y'", "BUILD_LOG=1", "-j", str(cpu_count() + 1)],
+ temporary_dir,
+)
+
+# flush the current website dir of target
+shutil.rmtree(target_dir, ignore_errors=True)
+
+# and recreate it here
+os.makedirs(target_dir + "/packages", exist_ok=True)
+
+# iterate over all sums in origin sha256sums and check rebuild files
+for origin in parse_origin_sha256sums():
+ origin_sum, origin_name = origin
+ # except the meta files defined above
+ if meta_files.match(origin_name):
+ print("Skipping meta file {}".format(origin_name))
+ continue
+
+ rebuild_path = temporary_dir + "/bin/targets/" + target + "/" + origin_name
+ # report missing files
+ if not os.path.exists(rebuild_path):
+ context["missing"].append({"name": origin_name})
+ else:
+ rebuild_info = {
+ "name": origin_name,
+ "size": os.path.getsize(rebuild_path),
+ "sha256sum": sha256sum(rebuild_path),
+ "repro": False,
+ }
+
+ # files ending with ipk are considered packages
+ if origin_name.endswith(".ipk"):
+ if rebuild_info["sha256sum"] == origin_sum:
+ rebuild_info["repro"] = True
+ context["packages_repro"] += 1
+ context["packages"].append(rebuild_info)
+ else:
+ #everything else should be images
+ if rebuild_info["sha256sum"] == origin_sum:
+ rebuild_info["repro"] = True
+ context["images_repro"] += 1
+ context["images"].append(rebuild_info)
+
+# calculate how many images are reproducible
+context["images_total"] = len(context["images"])
+if context["images_total"]:
+ context["images_repro_percent"] = round(
+ context["images_repro"] / context["images_total"] * 100.0, 2
+ )
+
+# calculate how many packages are reproducible
+context["packages_total"] = len(context["packages"])
+if context["packages_total"]:
+ context["packages_repro_percent"] = round(
+ context["packages_repro"] / context["packages_total"] * 100.0, 2
+ )
+
+# now render the website
+renderer = pystache.Renderer()
+mustache_header = renderer.load_template(template_dir + "/header")
+mustache_footer = renderer.load_template(template_dir + "/footer")
+mustache_target = renderer.load_template(template_dir + "/target")
+mustache_index = renderer.load_template(template_dir + "/index")
+
+index_html = renderer.render(mustache_header, context)
+index_html += renderer.render(mustache_index, context)
+index_html += renderer.render(mustache_footer, context)
+
+target_html = renderer.render(mustache_header, context)
+target_html += renderer.render(mustache_target, context)
+target_html += renderer.render(mustache_footer, context)
+
+# and store the files
+with open(output_dir + "/index.html", "w") as index_file:
+ index_file.write(index_html)
+
+with open(target_dir + "/index.html", "w") as target_file:
+ target_file.write(target_html)
+
+# get the origin manifest
+origin_manifest = get_file(openwrt_url + "/packages/Packages.manifest")
+
+# and store it in the databse
+ropp = importlib.import_module("reproducible_openwrt_package_parser")
+with open(rebuild_dir + "/packages/Packages.manifest") as rebuild_manifest:
+ result = ropp.show_list_difference(origin_manifest, rebuild_manifest.readlines())
+ ropp.insert_into_db(result, "{}-rebuild".format(version))
+
+# run diffoscope over non reproducible files in all available threads
+pool = Pool(cpu_count() + 1)
+pool.map(
+ diffoscope,
+ map(
+ lambda x: x["name"],
+ filter(lambda x: not x["repro"], context["images"] + context["packages"]),
+ ),
+)
+
+# debug option to keep build dir
+if not os.environ.get("KEEP_BUILD_DIR"):
+ print("removing build dir")
+ shutil.rmtree(temporary_dir)
+print("all done")
=====================================
mustache-templates/openwrt/footer.mustache
=====================================
@@ -0,0 +1,25 @@
+ </div>
+ <hr id="footer_separator" />
+ <p style="font-size:0.9em;">
+ <div id="page_footer">
+ This page was built by the jenkins job <a href="https://jenkins.debian.net/job/reproducible_openwrt-target-ar71xx/"> reproducible_openwrt-target-ar71xx</a>
+ which is configured via this <a href="https://salsa.debian.org/qa/jenkins.debian.net">git repo</a>.
+ There is more information
+ <a href="/userContent/about.html">about jenkins.debian.net</a>
+ and about <a href="https://wiki.debian.org/ReproducibleBuilds"> reproducible builds of Debian</a>
+ available elsewhere.
+ <br /> Please send technical feedback about jenkins to <a href="mailto:qa-jenkins-dev at lists.alioth.debian.org">the Debian jenkins development list</a>,
+ or as a <a href="https://www.debian.org/Bugs/Reporting">bug report against the <tt>jenkins.debian.org</tt> package</a>.
+ Feedback about specific job results should go to their respective lists and/or the BTS.
+ <br />
+ The code of <a href="https://salsa.debian.org/qa/jenkins.debian.net">jenkins.debian.net</a>
+ is mostly GPL-2 licensed. The weather icons are public domain and were taken from the
+ <a href="http://tango.freedesktop.org/Tango_Icon_Library" target=_blank> Tango Icon Library</a>.
+ Copyright 2014-2019 <a href="mailto:holger at layer-acht.org">Holger Levsen</a>
+ and <a href="https://jenkins.debian.net/userContent/thanks.html">many others</a>.
+ </div>
+ </p>
+ </div>
+</body>
+
+</html>
=====================================
mustache-templates/openwrt/header.mustache
=====================================
@@ -0,0 +1,130 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width">
+ <title>Reproducible OpenWrt ?</title>
+ <link rel='stylesheet' id='kamikaze-style-css' href='cascade.css?ver=4.0' type='text/css' media='all'>
+</head>
+<style type="text/css">
+ html,
+ body {
+ margin: 0;
+ padding: 0;
+ height: 100%;
+ }
+
+ body {
+ color: #333;
+ padding-top: 2em;
+ font-family: Helvetica, Arial, sans-serif;
+ width: 90%;
+ min-width: 700px;
+ max-width: 1100px;
+ margin: auto;
+ font-size: 120%;
+ background-color: #ddd;
+ }
+
+ h1 {
+ font-size: 120%;
+ line-height: 1em;
+ }
+
+ h2 {
+ font-size: 100%;
+ line-height: 1em;
+ }
+
+ table {
+ width: 100%;
+ box-shadow: 0 0 0.5em #999;
+ margin: 0;
+ border: none !important;
+ margin-bottom: 2em;
+ border-collapse: collapse;
+ border-spacing: 0;
+ }
+
+ th {
+ background: #000;
+ background: -webkit-linear-gradient(top, #444, #000);
+ background: -moz-linear-gradient(top, #444, #000);
+ background: -ms-linear-gradient(top, #444, #000);
+ background: -o-linear-gradient(top, #444, #000);
+ background: linear-gradient(top, #444, #000);
+ font-size: 14px;
+ line-height: 24px;
+ border: none;
+ text-align: left;
+ color: #fff;
+ }
+
+ tr {
+ background: rgba(255, 255, 255, 0.8);
+ }
+
+ tr:hover {
+ background: rgba(255, 255, 255, 0.6);
+ }
+
+ p,
+ th,
+ td {
+ font-size: 14px;
+ }
+
+ th,
+ td {
+ height: 20px;
+ vertical-align: middle;
+ white-space: nowrap;
+ padding: 0.2em 0.5em;
+ border: 1px solid #ccc;
+ }
+
+ a:link,
+ a:visited {
+ color: #337ab7;
+ font-weight: bold;
+ text-decoration: none;
+ }
+
+ a:hover,
+ a:active,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ .s {
+ text-align: right;
+ width: 15%;
+ }
+
+ .d {
+ text-align: center;
+ width: 15%;
+ }
+
+ .sh {
+ font-family: monospace;
+ }
+</style>
+
+<body>
+ <div id="content">
+ <pre>
+ _______ ________ __
+ | |.-----.-----.-----.| | | |.----.| |_
+ | - || _ | -__| || | | || _|| _|
+ |_______|| __|_____|__|__||________||__| |____|
+ |__| W I R E L E S S F R E E D O M
+ -----------------------------------------------------
+ OpenWrt {{ version }}, {{ commit_string }}
+ -----------------------------------------------------
+ </pre>
+ </div>
+ <div id="main-content">
+ <h1>OpenWrt - <em>reproducible</em> wireless freedom!</h1>
=====================================
mustache-templates/openwrt/index.mustache
=====================================
@@ -0,0 +1,42 @@
+ <p>
+ <em>Reproducible builds</em> enable anyone to reproduce bit by bit
+ identical binary packages from a given source, so that anyone can verify
+ that a given binary derived from the source it was said to be derived.
+ There is more information about
+ <a href="https://wiki.debian.org/ReproducibleBuilds">reproducible builds on the Debian wiki</a>
+ and on
+ <a href="https://reproducible-builds.org">https://reproducible-builds.org</a>.
+ These pages explain in more depth why this is
+ useful, what common issues exist and which
+ workarounds and solutions are known.
+ </p>
+ <p>
+ <em>Reproducible OpenWrt</em> is an effort to apply this to OpenWrt. Thus each OpenWrt target is build twice, with a few variations added and then the resulting images and packages from the two builds are compared using <a href="https://tracker.debian.org/diffoscope">diffoscope</a>. OpenWrt generates many different types of raw <code>.bin</code> files, and diffoscope does not know how to parse these. Thus the resulting diffoscope output is not nearly as clear as it could be - hopefully this limitation will be overcome eventually, but in the meanwhile the input components (uImage kernel file, rootfs.tar.gz, and/or rootfs squashfs) can be inspected. Also please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>
+ <p>
+ There is a weekly run <a href="https://jenkins.debian.net/view/reproducible/job/reproducible_openwrt/">jenkins
+ job</a> to test the <code>master</code> branch of <a href="https://github.com/openwrt/openwrt.git">OpenWrt.git</a>. The
+ jenkins job is running <a href="https://salsa.debian.org/qa/jenkins.debian.net/tree/master/bin/reproducible_openwrt.sh">reproducible_openwrt.sh</a>
+ in a Debian environment and this script is solely responsible for creating
+ this page. Feel invited to join <code>#reproducible-builds</code> (on
+ irc.oftc.net) to request job runs whenever sensible. Patches and other <a href="mailto:reproducible-builds at lists.alioth.debian.org">feedback</a> are
+ very much appreciated - if you want to help, please start by looking at the
+ <a href="https://jenkins.debian.net/userContent/todo.html#_reproducible_openwrt">ToDo
+ list for OpenWrt</a>, you might find something easy to contribute. <br />Thanks to <a href="https://www.profitbricks.co.uk">Profitbricks</a> for
+ donating the virtual machines this is running on!</p>
+ <p>
+ <ul>
+ {{ #targets }}
+ <li><a href="{{ version }}/{{ name}}">{{ version }} - {{ name }}</a></li>
+ {{ /targets }}
+ </ul>
+ </p>
+ <table>
+ <tr>
+ <th>git commit log</th>
+ </tr>
+ <tr>
+ <td>
+ <pre>{{ git_log_oneline }}</pre>
+ </td>
+ </tr>
+ </table>
=====================================
mustache-templates/openwrt/target.mustache
=====================================
@@ -0,0 +1,60 @@
+ <p>
+ {{ images_repro }} ({{ images_repro_percent }}%) out of {{ images_total }}
+ built images and {{ packages_repro }} ({{ packages_repro_percent }}%) out of
+ {{ packages_total }} built packages were reproducible in our test setup.
+ </p>
+ <p>
+ These tests were last run on {{ today }} for version using diffoscope {{
+ diffoscope_version }}.
+ </p>
+ <p style="clear:both;">
+ </p>
+ <table>
+ <tr>
+ <th>Images for <code>{{ target }}</code></th>
+ </tr>
+ {{ #images }}
+ <tr>
+ {{ #repro }}
+ <td><img src="/userContent/reproducible/static/weather-clear.png" alt="reproducible icon" /> {{ name }} ({{ sha256sum }}, {{ size }}K) is reproducible.</td>
+ {{ /repro }}
+ {{ ^repro }}
+ <td><a href="/{{ version }}/{{ target }}/{{ name }}.html"><img src="/userContent/reproducible/static/weather-showers-scattered.png" alt="unreproducible icon"> {{ name }}</a> ({{ size }}K) is unreproducible.</td>
+ {{ /repro }}
+ </tr>
+ {{ /images }}
+ </table>
+ <table>
+ <tr>
+ <th>Unreproducible and otherwise broken packages</th>
+ </tr>
+ {{ #packages }}
+ {{ ^repro }}
+ <tr>
+ <td><a href="/{{ version }}/{{ target }}/{{ name }}.html"><img src="/userContent/reproducible/static/weather-showers-scattered.png" alt="unreproducible icon"> {{ name }}</a> ({{ size }}K) is unreproducible.</td>
+ </tr>
+ {{ /repro }}
+ {{ /packages }}
+ </table>
+ <table>
+ <tr>
+ <th>Reproducible packages</th>
+ </tr>
+ {{ #packages }}
+ {{ #repro }}
+ <tr>
+ <td><img src="/userContent/reproducible/static/weather-clear.png" alt="reproducible icon" /> {{ name }} ({{ sha256sum }}, {{ size }}K) is reproducible.</td>
+ </tr>
+ {{ /repro }}
+ {{ /packages }}
+ </table>
+ <table>
+ <tr>
+ <th>Missing files after rebuild</th>
+ </tr>
+ {{ #missing}}
+ <tr>
+ <td>{{ name }}</td>
+ </tr>
+ {{ /missing }}
+ </table>
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/034491eacdc86d0bc5857062192bbb697192e061
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/034491eacdc86d0bc5857062192bbb697192e061
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20190925/86019434/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list