[Qa-jenkins-scm] [jenkins.debian.net] 01/01: reproducible: new db table: removed_packages, to track removed package and subsequently clean up files

[Holger Levsen]

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch master
in repository jenkins.debian.net.

commit 36798da6b02677f0d8d3808bb0541148d8e1c314
Author: Mattia Rizzolo <mattia at mapreri.org>
Date:   Fri Apr 10 12:39:12 2015 +0200

    reproducible: new db table: removed_packages, to track removed package and subsequently clean up files
    
    the db now stores tuple of (pkgname, suite, architecture) of every removed
    packages, then reproducible_maintenance.sh pick up entries from it and
    find+remove files.
    
    For now the reproducible_maintenance only output what he would have remove.
---
 TODO                               |  1 -
 bin/reproducible_db_maintenance.py |  7 +++++++
 bin/reproducible_maintenance.sh    | 29 ++++++++++++++---------------
 bin/reproducible_scheduler.py      |  5 +++++
 logparse/reproducible.rules        |  1 +
 5 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/TODO b/TODO
index ecd949a..69b46f4 100644
--- a/TODO
+++ b/TODO
@@ -138,7 +138,6 @@ properties:
 ** meld bin/reproducible_setup_schroot.sh into bin/schroot-create.sh and alias the former to the latter
 ** "fork" etc/schroot/default into etc/schroot/reproducible
 ** use one css for all, not two minimally different ones
-** restore the "find packages which have been removed from sid" part of reproducible_maintenance.sh
 ** repo-comparison: check for binaries without source
 ** turn job-cfg/g-i.yaml into .yaml.py
 
diff --git a/bin/reproducible_db_maintenance.py b/bin/reproducible_db_maintenance.py
index 1774039..003ae32 100755
--- a/bin/reproducible_db_maintenance.py
+++ b/bin/reproducible_db_maintenance.py
@@ -354,6 +354,13 @@ schema_updates = {
             url TEXT,
             PRIMARY KEY (name))''',
         'INSERT INTO rb_schema VALUES ("10", "' + now + '")'],
+    11: [ # table with removed packages, to enable the maintenance job to do clean up
+        '''CREATE TABLE removed_packages (
+            name TEXT NOT NULL,
+            suite TEXT NOT NULL,
+            architecture TEXT NOT NULL,
+            PRIMARY KEY (name, suite, architecture))''',
+        'INSERT INTO rb_schema VALUES ("11", "' + now + '")'],
 }
 
 
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index 53799b2..a436265 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -185,26 +185,25 @@ fi
 rm $PACKAGES
 
 # find packages which have been removed from unstable
-# commented out for now. This can't be done using the database anymore
-QUERY="SELECT source_packages.name FROM source_packages
-		WHERE source_packages.name NOT IN
-		(SELECT sources.name FROM sources)
-	LIMIT 25"
-#PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY")
-PACKAGES=''
+QUERY="SELECT name, suite, architecture FROM removed_packages
+		LIMIT 25"
+PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY")
 if [ ! -z "$PACKAGES" ] ; then
+	DIRTY=true
 	echo
+	echo "Warning: found files relative to old packages, no more in the archive:"
 	echo "Removing these removed packages from database:"
 	echo $PACKAGES
 	echo
-	QUERY="DELETE FROM source_packages
-			WHERE source_packages.name NOT IN
-			(SELECT sources.name FROM sources)
-		LIMIT 25"
-	sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY"
-	cd /var/lib/jenkins/userContent
-	for i in PACKAGES ; do
-		find rb-pkg/ rbuild/ notes/ dbd/ -name "${i}_*" -exec rm -v {} \;
+	for pkg in "$PACKAGES" ; do
+		PKGNAME=$(echo "$pkg" | cut -f '|' -d 1)
+		SUITE=$(echo "$pkg" | cut -f '|' -d 2)
+		ARCH=$(echo "$pkg" | cut -f '|' -d 3)
+		QUERY="DELETE FROM removed_packages
+			WHERE name='$PKGNAME' AND suite='$SUITE' AND architecture='$ARCH'"
+		sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY"
+		cd /var/lib/jenkins/userContent
+		find rb-pkg/$SUITE/$ARCH  rbuild/$SUITE/$ARCH notes/ dbd/$SUITE/$ARCH buildinfo/$SUITE/$ARCH -name "${PKGNAME}_*" | xargs echo rm -v
 	done
 	cd -
 fi
diff --git a/bin/reproducible_scheduler.py b/bin/reproducible_scheduler.py
index bb465ea..78d6fcb 100755
--- a/bin/reproducible_scheduler.py
+++ b/bin/reproducible_scheduler.py
@@ -102,11 +102,13 @@ def update_sources_tables(suite):
     rmed_pkgs = [x for x in cur_pkgs_name if x not in new_pkgs_name]
     log.info('Now deleting ' + str(len(rmed_pkgs)) + ' removed packages: ' + str(rmed_pkgs))
     rmed_pkgs_id = []
+    pkgs_to_rm = []
     for pkg in rmed_pkgs:
         result = query_db(('SELECT id FROM sources ' +
                           'WHERE name="{name}" ' +
                           'AND suite="{suite}"').format(name=pkg, suite=suite))
         rmed_pkgs_id.extend(result)
+        pkgs_to_rm.extend([(x[0], x[1], 'amd64') for x in result])
     log.debug('removed packages ID: ' + str([str(x[0]) for x in rmed_pkgs_id]))
     cursor.executemany('DELETE FROM sources ' +
                        'WHERE id=?', rmed_pkgs_id)
@@ -114,6 +116,9 @@ def update_sources_tables(suite):
                        'WHERE package_id=?', rmed_pkgs_id)
     cursor.executemany('DELETE FROM schedule ' +
                        'WHERE package_id=?', rmed_pkgs_id)
+    cursor.executemany('INSERT INTO removed_packages '  +
+                       '(name, suite, architecture) ' +
+                       'VALUES (?, ?, ?)', pkgs_to_rm)
     conn_db.commit()
     # finally check whether the db has the correct number of packages
     pkgs_end = query_db('SELECT count(*) FROM sources WHERE suite="%s"' % suite)
diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules
index 176836c..b9a5c56 100644
--- a/logparse/reproducible.rules
+++ b/logparse/reproducible.rules
@@ -9,6 +9,7 @@ warning /Warning: lock .+ still exists, exiting./
 warning /Warning: old schroots found in /schroots, which have been deleted:/
 warning /Warning: processes found which should not be there/
 warning /Warning: packages found where the build was started more than 24h ago:/
+warning /Warning: found files relative to old packages, no more in the archive:/
 warning /Warning: Found files with bad permissions.+/
 warning /Warning: .+ could not be fully removed./
 warning /Warning: could not download.+/

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git