[Piuparts-commits] [piuparts] 11/12: split the generate_daily_report cronjob

[Holger Levsen]

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch develop
in repository piuparts.

commit e5fee40412cd3b10dcb00651a94e0c6853f1d252
Author: Andreas Beckmann <anbe at debian.org>
Date:   Wed Oct 4 05:58:55 2017 +0200

    split the generate_daily_report cronjob
    
    Signed-off-by: Andreas Beckmann <anbe at debian.org>
    Signed-off-by: Holger Levsen <holger at layer-acht.org>
---
 conf/crontab-master.in              | 12 ++++++++----
 debian/changelog                    | 10 ++++++++++
 master-bin/generate_daily_report.in | 15 ++-------------
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/conf/crontab-master.in b/conf/crontab-master.in
index 0ad5d6f..ad36447 100644
--- a/conf/crontab-master.in
+++ b/conf/crontab-master.in
@@ -1,7 +1,8 @@
 # m h dom mon dow (0|7=sun,1=mon)    command
 
 # set PATH, variable substitution is not supported (PATH=$PATH:/my/bin)
-PATH=@sharedir@/piuparts/master:/usr/bin:/bin
+PATH=@sharedir@/piuparts/master:@sharedir@/piuparts:/usr/bin:/bin
+PYTHONPATH=@libdir@/python2.7/dist-packages
 
 #
 # cleanup $HTDOCS/daily.lock
@@ -12,7 +13,10 @@ PATH=@sharedir@/piuparts/master:/usr/bin:/bin
 # generate reports twice a day
 #  (dinstall runs 1|7|13|19:52, so this is long after mirror pushes...)
 #
-0  0,12 * * * generate_daily_report
+0  4-23/12 * * * piuparts-analyze
+0  2-23/3 * * * detect_well_known_errors
+0  0-23/3 * * * piuparts-report
+15 23 * * * generate_daily_report
 
 #
 # reschedule old logs twice a day
@@ -24,8 +28,8 @@ PATH=@sharedir@/piuparts/master:/usr/bin:/bin
 # - these may result in packages being retested
 # - if that's not the case, run them from within generate_daily_report
 #
-30 3-21/6 * * * detect_network_issues
-45 3-21/6 * * * detect_piuparts_issues
+30 2-23/3 * * * detect_network_issues
+45 2-23/3 * * * detect_piuparts_issues
 0  10,22 * * * detect_archive_issues
 30 22 * * * report_untestable_packages
 0  23 * * * report_stale_reserved_packages
diff --git a/debian/changelog b/debian/changelog
index f950f55..f5d1c64 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,15 @@ piuparts (0.83) UNRELEASED; urgency=medium
     - Use global locks to allow only one running instance of each script.
     - Acquire section locks to prevent concurrent processing of a section,
       including piuparts-master access.
+    - Run as three separate cronjobs.
+  * conf/crontab-master:
+    - Run detect_well_known_errors every three hours to avoid accumulating a
+      large backlog of new logfiles.
+    - Run piuparts-analyze twice per day.
+    - Try piuparts-report every three hours s.t. we start 2-3 runs per day
+      even with running times above 8-12 hours.
+    - Run detect_{piuparts,network}_issues a short time before trying
+      piuparts-report to exclude (most of) these logfiles from the report.
 
  -- Holger Levsen <holger at debian.org>  Sat, 07 Oct 2017 17:42:12 +0200
 
@@ -45,6 +54,7 @@ piuparts (0.81) unstable; urgency=medium
     failing.  (Closes: #877417)
   * master-bin/distill_alternatives_log: Parse piuparts.conf to find the
     sections that use scripts-log-alternatives.
+  * master-bin/generate_daily_report: Most parts are now separate cronjobs.
   * Add bug template for stable -> sid file overwrite errors.
   * Use copyright notices without FSF street address.
 
diff --git a/master-bin/generate_daily_report.in b/master-bin/generate_daily_report.in
index 05adfa1..caa7375 100644
--- a/master-bin/generate_daily_report.in
+++ b/master-bin/generate_daily_report.in
@@ -41,7 +41,7 @@ R_STARTDATE=$(date -u +%s)
 # various scripts to detect issues and cleanup
 #
 OUTPUT=$(mktemp)
-for SCRIPT in detect_network_issues detect_piuparts_issues archive_old_logs report_newly_bugged_packages ; do
+for SCRIPT in archive_old_logs ; do
   STARTDATE=$(date -u +%s)
   @sharedir@/piuparts/master/$SCRIPT 2>&1 > $OUTPUT
   FINALDATE=$(date -u +%s)
@@ -79,17 +79,6 @@ done | sed s#^$MASTER#$URLBASE# >> $DAILYREPORT
 
 echo "" >> $DAILYREPORT
 date >> $DAILYREPORT
- at sharedir@/piuparts/master/detect_well_known_errors 2>&1 >> $DAILYREPORT
-
-echo "" >> $DAILYREPORT
-date >> $DAILYREPORT
-nice \
-	@sharedir@/piuparts/piuparts-report \
-	>> $DAILYREPORT 2>&1
-date >> $DAILYREPORT
-
-echo "" >> $DAILYREPORT
-date >> $DAILYREPORT
 echo "Expiring old .html files:" >> $DAILYREPORT
 find $HTDOCS -name '* *.html' -mtime +30 -ls -delete >> $DAILYREPORT
 expire=$(mktemp)
@@ -103,7 +92,7 @@ rm $expire
 FINALDATE=$(date -u +%s)
 RUNTIME=$(date -u -d "0 $FINALDATE seconds - $R_STARTDATE seconds" +%T)
 
-( echo "Runtime: $RUNTIME" ; echo ; cat $DAILYREPORT ) | mail -s piuparts-report $LOGNAME
+( echo "Runtime: $RUNTIME" ; echo ; cat $DAILYREPORT ) | mail -s generate_daily_report $LOGNAME
 
 mv $FAILURESTAMP.new $FAILURESTAMP
 rm -f $LOCKFILE

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/piuparts/piuparts.git