[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] reproducible: move diffoscope_compare() into its own class
Mattia Rizzolo
gitlab at salsa.debian.org
Mon Dec 2 17:38:26 GMT 2019
Mattia Rizzolo pushed to branch master at Debian QA / jenkins.debian.net
Commits:
70e1568b by Mattia Rizzolo at 2019-12-02T17:38:07Z
reproducible: move diffoscope_compare() into its own class
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
3 changed files:
- + bin/rblib/commands.py
- − bin/reproducible_common.py
- bin/reproducible_openwrt_rebuild.py
Changes:
=====================================
bin/rblib/commands.py
=====================================
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2015-2019 Mattia Rizzolo <mattia at debian.org>
+# Copyright © 2015-2017 Holger Levsen <holger at layer-acht.org>
+# Copyright © 2019 Paul Spooren <mail at aparcar.org>
+#
+# Licensed under GPL-2
+
+import os
+import subprocess
+import resource
+
+
+class Schroot():
+ def __init__(self, chroot, directory=None):
+ self.chroot = chroot
+ self._cmd = ['schroot', '-c', chroot]
+ self.set_directory(directory)
+ self._limits = []
+
+ def set_directory(self, directory):
+ if not directory:
+ self.directory = '/tmp'
+ self._cmd.extend(('--directory', self.directory))
+
+ def add_limit(self, what, how):
+ self._limits.append((what, how))
+
+ def set_default_limits(self):
+ # 10 GB of actually used memory
+ self._limits.append(
+ resource.RLIMIT_AS, (10 * 1024 ** 3, resource.RLIM_INFINITY)
+ )
+
+ def _preexec_limiter(self):
+ for limit in self._limits:
+ resource.setrlimit(*limit)
+
+ def run(self, command, *, check=False, timeout=False):
+ # separate the command name frome the optiosn with --
+ self._cmd.append(command[0])
+ self._cmd.append('--')
+ self._cmd.extend(command[1:])
+ return subprocess.run(
+ self._cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ check=check,
+ timeout=timeout,
+ text=check,
+ preexec_fn=self._preexec_limiter,
+ )
+
+
+class Diffoscope(Schroot):
+ def __init__(self, chroot=None, suite=None):
+ if suite is None:
+ suite = 'unstable'
+ if chroot is None:
+ chroot = f'jenkins-reproducible-{suite}-diffoscope'
+ super.__init__(chroot=chroot)
+ self.set_default_limits()
+
+ def version(self):
+ try:
+ p = self.run(('diffoscope', '--version'), check=True)
+ except subprocess.CalledProcessError:
+ return 'cannot get version'
+ return p.stdout.strip()
+
+ def compare_files(self, a, b, html):
+ self.set_directory(os.path.dirname(html))
+ diff_cmd = ['diffoscope', '--html', html]
+ msg = f'diffoscope {self.version()}'
+ timeout = 30*60 # 30 minutes
+ try:
+ p = self.run(diff_cmd + (a, b), timeout=timeout)
+ if p.returncode == 0:
+ print(f'{msg}: {a} is reproducible, yay!')
+ elif p.returncode == 1:
+ print(f'{msg}: {a} has issue, please investigate')
+ elif p.returncode == 2:
+ with open(html, 'w') as f:
+ f.write(f'{msg} had errors comparing the two builds.\n')
+ f.write(f'{diff_cmd}\n')
+ f.write(p.stdout)
+ except subprocess.TimeoutExpired:
+ if os.path.exits(html):
+ text = (f'{msg} produced not output and was killed after '
+ f'running into timeout after {timeout}.')
+ with open(html, 'w') as f:
+ f.write(f'text\n')
+ f.write(f'{diff_cmd}\n')
+ f.write(p.stdout)
+ print(text)
+ else:
+ print(f'{msg} was killed after running into timeout after '
+ f'{timeout}, but there still {html}')
=====================================
bin/reproducible_common.py deleted
=====================================
@@ -1,100 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# Copyright © 2019 Paul Spooren <mail at aparcar.org>
-#
-# Inspired by reproducible_common.sh
-# © Holger Levsen <holger at layer-acht.org>
-#
-# Released under the GPLv2
-
-import os
-import subprocess
-import resource
-
-
-def e(var, default):
- """Return env variable or default"""
- return os.environ.get(var, default)
-
-
-# DBSUITE which version of diffoscope to use
-timeout = e("TIMEOUT", 30 * 60) # 30m
-# DIFFOSCOPE_VIRT_LIMIT max RAM usage
-ds_virt_limit = int(e("DIFFOSCOPE_VIRT_LIMIT", 10 * 1024 ** 3)) # 10GB
-# TIMEOUT timeout for diffoscope in seconds
-dbsuite = e("DBDSUITE", "unstable")
-# SCHROOT to use
-schroot = e("SCHROOT", f"source:jenkins-reproducible-{dbsuite}-diffoscope")
-
-
-def limit_resources():
- resource.setrlimit(resource.RLIMIT_CPU, (1, 1))
- resource.setrlimit(resource.RLIMIT_AS, (ds_virt_limit, resource.RLIM_INFINITY))
-
-
-def diffoscope_version():
- cmd = []
- if schroot:
- cmd.extend(["schroot", "--directory", "/tmp", "-c", schroot])
-
- cmd.extend(["diffoscope", "--", "--version"])
- print(cmd)
- return (
- subprocess.run(cmd, capture_output=True, text=True, preexec_fn=limit_resources)
- .stdout.strip()
- .split()[1]
- )
-
-
-def diffoscope_compare(path_a, path_b, path_output_html):
- """
- Run diffoscope in a schroot environment
-
- Args:
- - path_a path to first file to compare
- - path_b path to second file a to compare
- - path_output_html path where to store result html
- """
- cmd = []
- if schroot:
- cmd.extend(
- ["schroot", "--directory", os.path.dirname(path_output_html), "-c", schroot]
- )
-
- try:
- cmd.extend(["diffoscope", "--", "--html", path_output_html, path_a, path_b])
- result = subprocess.run(
- cmd,
- timeout=timeout,
- capture_output=True,
- text=True,
- preexec_fn=limit_resources,
- )
- msg = f"diffoscope {diffoscope_version()} "
- if result.returncode == 0:
- print(msg + f"{path_a} reproducible, yay!")
- else:
- if result.returncode == 1:
- print(msg + f"found issues, please investigate {path_a}")
- elif result.returncode == 2:
- with open(path_output_html, "w") as output_html_file:
- output_html_file.write(
- msg
- + f"""had trouble comparing the two builds. Please
- investigate {path_a}"""
- )
-
- except subprocess.TimeoutExpired:
- if os.path.exists(path_output_html):
- print(
- msg
- + f"""produced no output comparing {path_a} with {path_b} and
- was killed after running into timeout after {timeout}..."""
- )
- else:
- print(
- msg
- + """was killed after running into timeout after $TIMEOUT, but
- there is still {path_output_html}"""
- )
=====================================
bin/reproducible_openwrt_rebuild.py
=====================================
@@ -23,7 +23,7 @@ from time import strftime, gmtime
import shutil
import json
-from reproducible_common import diffoscope_compare, diffoscope_version
+from .rblib.commands import Diffoscope
from reproducible_openwrt_package_parser import insert_into_db, show_list_difference
# target to be build
@@ -108,7 +108,7 @@ context = {
"packages_repro_percent": 0,
"packages_total": 0,
"today": strftime("%Y-%m-%d", gmtime()),
- "diffoscope_version": diffoscope_version(),
+ "diffoscope_version": Diffoscope().version(),
"target": target,
"images": [],
"packages": [],
@@ -125,10 +125,10 @@ def diffoscope(origin_name):
print("Error downloading {}".format(origin_name))
return
- diffoscope_compare(
+ Diffoscope().compare(
file_origin.name,
target_dir + "/" + origin_name,
- results_target_dir + "/" + origin_name + ".html",
+ html=results_target_dir + "/" + origin_name + ".html",
)
file_origin.close()
@@ -145,7 +145,7 @@ def get_file(url, path=None):
print("downloading {}".format(url))
try:
content = urlopen(url).read()
- except:
+ except Exception:
return 1
if path:
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/70e1568b8f2cec495b89c14267837761728dbf15
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/commit/70e1568b8f2cec495b89c14267837761728dbf15
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20191202/e0bfd155/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list