[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 3 commits: add reproducible_openwrt_package_parser.py
Holger Levsen
gitlab at salsa.debian.org
Thu Mar 7 14:33:10 GMT 2019
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
02dd59fd by Alexander Couzens at 2019-03-07T14:32:47Z
add reproducible_openwrt_package_parser.py
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
d6d1cac8 by Alexander Couzens at 2019-03-07T14:32:47Z
reproducible_openwrt_package_parser.py: add DB interface based on opensuse
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
b556fbf7 by Alexander Couzens at 2019-03-07T14:32:47Z
reproducible_openwrt.py: add OpenWrt to the database
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
4 changed files:
- bin/reproducible.ini
- bin/reproducible_db_maintenance.py
- bin/reproducible_openwrt.sh
- + bin/reproducible_openwrt_package_parser.py
Changes:
=====================================
bin/reproducible.ini
=====================================
@@ -44,3 +44,11 @@ suites = default
archs = x86_64
defaultsuite = default
defaultarch = x86_64
+
+[openwrt]
+distro_root = openwrt
+landing_page = openwrt.html
+suites = trunk
+archs = mips_24kc mipsel_24kc i386_pentium4 x86_64
+defaultsuite = trunk
+defaultarch = x86_64
=====================================
bin/reproducible_db_maintenance.py
=====================================
@@ -684,6 +684,10 @@ schema_updates = {
SELECT id FROM distributions WHERE name='archlinux')
WHERE suite LIKE 'archlinux_%%'"""
],
+ 42: [ # add OpenWrt
+ "INSERT INTO distributions (name) VALUES "
+ "('openwrt')"
+ ],
}
=====================================
bin/reproducible_openwrt.sh
=====================================
@@ -644,6 +644,25 @@ else
MAGIC_SIGN="?"
fi
+# parse the OpenWrt Packages.manifest files to generate results to fill into the database
+for i in * ; do
+ cd "$i"
+
+ # search packages in both paths to find non-existing ones
+ PKGS1=$(find -- * -type f -name "Packages.manifest" | sort -u )
+ pushd "$RESULTSDIR/b2/$i"
+ PKGS2=$(find -- * -type f -name "Packages.manifest" | sort -u )
+ popd
+
+ for j in $(printf "%s\n%s" "$PKGS1" "$PKGS2" | sort -u ) ; do
+ if [ ! -f "$RESULTSDIR/b1/$i/$j" ] || [ ! -f "$RESULTSDIR/b2/$i/$j" ] ; then
+ echo "One Packages.manifest list does not exist! Either $RESULTSDIR/b1/$i/$j or $RESULTSDIR/b2/$i/$j"
+ fi
+ echo "Running reproducible_openwrt_package_parser.py"
+ /srv/jenkins/bin/reproducible_openwrt_package_parser.py "$PKGS1" "$PKGS2" || true
+ done
+done
+
write_openwrt_page_header(){
cat > "$PAGE" <<- EOF
<!DOCTYPE html>
=====================================
bin/reproducible_openwrt_package_parser.py
=====================================
@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+# Copyright Alexander Couzens <lynxis at fe80.eu> 2018
+#
+# under the GPL2
+#
+# Parse two Packages.manifest or Packages list from two builds and
+# fill the database of reproducible-builds with the results
+
+import email.parser
+
+def download_manifest():
+ import requests
+
+ url = "https://downloads.openwrt.org/releases/18.06.1/targets/ramips/mt76x8/packages/Packages.manifest"
+ response = requests.get(url)
+ return str(response.content, 'utf-8')
+
+def parse_packages(package_list_fp):
+ linebuffer = ""
+ packages = []
+
+ for line in package_list_fp:
+ if line == '\n':
+ parser = email.parser.Parser()
+ package = parser.parsestr(linebuffer)
+ packages.append(package)
+ linebuffer = ""
+ else:
+ linebuffer += line
+ return packages
+
+def show_list_difference(list_a, list_b):
+ """ get two list of manifest and generate a result """
+ pkg_a = parse_packages(list_a)
+ pkg_b = parse_packages(list_b)
+
+ # packages which does not have the same pkg in B
+ alone_a = {}
+
+ # packages which does not have the same pkg in A
+ alone_b = {}
+
+ # package which are not reproducible
+ differ = {}
+ same = {}
+
+ dict_a = {}
+ dict_b = {}
+
+ for pkg in pkg_a:
+ dict_a[pkg['Package']] = pkg
+
+ for pkg in pkg_b:
+ dict_b[pkg['Package']] = pkg
+
+ for name in dict_a:
+ if name not in dict_b:
+ alone_a[name] = dict_a[name]
+ else:
+ if dict_a[name]['SHA256sum'] != dict_b[name]['SHA256sum']:
+ differ[name] = [dict_a[name], dict_b[name]]
+ else:
+ same[name] = [dict_a[name], dict_b[name]]
+ del dict_b[name]
+
+ for name in dict_b:
+ alone_b[name] = dict_b[name]
+
+ return (same, alone_a, alone_b, differ)
+
+def insert_into_db(result, suite='trunk'):
+ """ takes the result tuple and insert it into the database """
+ from sqlalchemy import select, and_, bindparam
+ from rblib import conn_db, query_db, db_table
+ from rblib.confparse import log
+ from datetime import datetime
+
+ same, alone_a, alone_b, differ = result
+
+ distributions = db_table('distributions')
+ results = db_table('results')
+ sources = db_table('sources')
+
+ distro_id = query_db(
+ select([distributions.c.id]).where(distributions.c.name == 'openwrt')
+ )[0][0]
+
+ # Delete all old data
+ transaction = conn_db.begin()
+ d = results.delete(results.c.package_id.in_(
+ select([sources.c.id]).select_from(sources).where(sources.c.distribution == distro_id)
+ ))
+ query_db(d)
+ d = sources.delete(sources.c.distribution == distro_id)
+ query_db(d)
+ transaction.commit()
+
+ # create new data
+ pkgs = []
+ pkgs_b = {}
+ now = datetime.now()
+
+ def insert_pkg_list(pkg_list, state, timestamp):
+ # Add new data
+ for pkg in same:
+ p = {
+ 'name': pkg['Package'],
+ 'version': pkg['Version'],
+ 'suite': suite,
+ 'architecture': pkg['Architecture'],
+ 'distribution': distro_id
+ }
+ pkgs.append(p)
+ data = {
+ 'status': pkg['status'],
+ 'build_date': timestamp,
+ 'build_duration': 2342,
+ }
+ pkgs_b[(pkg['package'], pkg['version'])] = data
+
+ insert_pkg_list(same, "reproducible", now)
+ insert_pkg_list(alone_a, "FTBFS on B", now)
+ insert_pkg_list(alone_b, "FTBFS on A", now)
+ insert_pkg_list(differ, "unreproducible", now)
+
+ log.info('Injecting new source packages…')
+ transaction = conn_db.begin()
+ conn_db.execute(sources.insert(), pkgs)
+ transaction.commit()
+
+ log.info('Injecting build results…')
+ cur_pkgs = select(
+ [sources.c.id, sources.c.name, sources.c.version,
+ sources.c.suite, sources.c.architecture]
+ ).select_from(
+ sources.join(distributions)
+ ).where(
+ and_(
+ distributions.c.name == 'openwrt',
+ sources.c.suite == bindparam('suite'),
+ sources.c.architecture == bindparam('arch')
+ )
+ )
+ cur_pkgs = query_db(cur_pkgs.params({'suite': 'factory', 'arch': 'x86_64'}))
+
+ builds = []
+ for pkg in cur_pkgs:
+ # (id, name, version, suite, architecture)
+ data = pkgs_b[(pkg[1], pkg[2])]
+ p = {
+ 'package_id': pkg[0],
+ 'version': pkg[2],
+ 'status': pkg['status'],
+ 'build_date': data['build_date'],
+ 'build_duration': data['build_duration'],
+ 'job': 'external',
+ }
+ builds.append(p)
+ if builds:
+ transaction = conn_db.begin()
+ conn_db.execute(results.insert(), builds)
+ transaction.commit()
+
+def example():
+ import io
+
+ package_list = io.StringIO(download_manifest)
+ packages = parse_packages(package_list)
+
+ for pkg in packages:
+ print(pkg['Filename'])
+
+def main():
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('packagea', nargs=1, type=argparse.FileType('r'),
+ help='The first package list')
+ parser.add_argument('packageb', nargs=1, type=argparse.FileType('r'),
+ help='The second package list')
+ args = parser.parse_args()
+
+ result = show_list_difference(args.packagea[0], args.packageb[0])
+ insert_into_db(result)
+
+if __name__ == "__main__":
+ main()
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/74fc1f1de0e79fcb643cb8c4f428e5a37138b24e...b556fbf70cbf6b40550b07c591f7c4e52d641198
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/74fc1f1de0e79fcb643cb8c4f428e5a37138b24e...b556fbf70cbf6b40550b07c591f7c4e52d641198
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20190307/adecda54/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list