[debian-mysql] Bug#736452: apt-get upgrade fails because mysql is too slow starting
Matija Nalis
mnalis-debianbug at tomsoft.hr
Thu Jan 23 18:59:38 UTC 2014
Package: mysql-server-5.5
Version: 5.5.35+dfsg-0+wheezy1
Severity: normal
apt-get upgrade fails on mysql-server-5.5 with following output outputs:
Reading package lists... Done
Building dependency tree
Reading state information... Done
Calculating upgrade... Done
0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
1 not fully installed or removed.
After this operation, 0 B of additional disk space will be used.
Setting up mysql-server-5.5 (5.5.35+dfsg-0+wheezy1) ...
Stopping MySQL database server: mysqld
..
Starting MySQL database server: mysqld . . . . . . . . . . . . . . failed!
invoke-rc.d: initscript mysql, action "start" failed.
dpkg: error processing mysql-server-5.5 (--configure):
subprocess installed post-installation script returned error exit status 1
Errors were encountered while processing:
mysql-server-5.5
Looking at what happens, is that "/etc/init.d/mysql start" fails because it
takes 16 seconds (which is longer than defaukt 14 seconds), which makes upgrade fail
too.
Database server in question has about 500GB local files in /var/lib/mysql
spread in 750 databases.
mysql log shows:
140123 19:27:49 mysqld_safe Starting mysqld daemon with databases from /var/lib/mysql
140123 19:27:49 [Note] Plugin 'FEDERATED' is disabled.
140123 19:27:49 InnoDB: Mutexes and rw_locks use GCC atomic builtins
140123 19:27:49 InnoDB: Compressed tables use zlib 1.2.7
140123 19:27:49 InnoDB: Using Linux native AIO
140123 19:27:49 InnoDB: Initializing buffer pool, size = 20.0G
140123 19:27:50 InnoDB: Completed initialization of buffer pool
140123 19:27:50 InnoDB: highest supported file format is Barracuda.
140123 19:28:05 InnoDB: Waiting for the background threads to start
140123 19:28:06 InnoDB: 5.5.35 started; log sequence number 2192961446708
As a quick kludge, I've changed "sleep 1" to "sleep 5" in /etc/init.d/mysql to allow installation to finish normally.
Maybe it should be possible to specify number of seconds to wait in /etc/default/mysql-server or somewhere, and mention
it in documentation, so admins can change it in upgrade-resistant way? or just up the default to 30 seconds or something
to accomodate larger DBs?
-- System Information:
Debian Release: 7.3
APT prefers stable-updates
APT policy: (500, 'stable-updates'), (500, 'stable')
Architecture: amd64 (x86_64)
Kernel: Linux 3.2.0-4-amd64 (SMP w/8 CPU cores)
Locale: LANG=C, LC_CTYPE=C (charmap=ANSI_X3.4-1968)
Shell: /bin/sh linked to /bin/dash
Versions of packages mysql-server-5.5 depends on:
ii adduser 3.113+nmu3
ii debconf [debconf-2.0] 1.5.49
ii initscripts 2.88dsf-41+deb7u1
ii libc6 2.13-38
ii libdbi-perl 1.622-1
ii libgcc1 1:4.7.2-5
ii libstdc++6 4.7.2-5
ii lsb-base 4.1+Debian8+deb7u1
ii mysql-client-5.5 5.5.35+dfsg-0+wheezy1
ii mysql-common 5.5.35+dfsg-0+wheezy1
ii mysql-server-core-5.5 5.5.35+dfsg-0+wheezy1
ii passwd 1:4.1.5.1-1
ii perl 5.14.2-21+deb7u1
ii psmisc 22.19-1+deb7u1
ii zlib1g 1:1.2.7.dfsg-13
Versions of packages mysql-server-5.5 recommends:
ii heirloom-mailx [mailx] 12.5-2
ii libhtml-template-perl 2.91-1
Versions of packages mysql-server-5.5 suggests:
pn tinyca <none>
-- Configuration Files:
/etc/init.d/mysql changed:
set -e
set -u
${DEBIAN_SCRIPT_DEBUG:+ set -v -x}
test -x /usr/bin/mysqld_safe || exit 0
.. /lib/lsb/init-functions
SELF=$(cd $(dirname $0); pwd -P)/$(basename $0)
CONF=/etc/mysql/my.cnf
MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf"
ERR_LOGGER="logger -p daemon.err -t /etc/init.d/mysql -i"
cd /
umask 077
export HOME=/etc/mysql/
mysqld_get_param() {
/usr/sbin/mysqld --print-defaults \
| tr " " "\n" \
| grep -- "--$1" \
| tail -n 1 \
| cut -d= -f2
}
sanity_checks() {
# check for config file
if [ ! -r /etc/mysql/my.cnf ]; then
log_warning_msg "$0: WARNING: /etc/mysql/my.cnf cannot be read. See README.Debian.gz"
echo "WARNING: /etc/mysql/my.cnf cannot be read. See README.Debian.gz" | $ERR_LOGGER
fi
# check for diskspace shortage
datadir=`mysqld_get_param datadir`
if LC_ALL=C BLOCKSIZE= df --portability $datadir/. | tail -n 1 | awk '{ exit ($4>4096) }'; then
log_failure_msg "$0: ERROR: The partition with $datadir is too full!"
echo "ERROR: The partition with $datadir is too full!" | $ERR_LOGGER
exit 1
fi
}
mysqld_status () {
ping_output=`$MYADMIN ping 2>&1`; ping_alive=$(( ! $? ))
ps_alive=0
pidfile=`mysqld_get_param pid-file`
if [ -f "$pidfile" ] && ps `cat $pidfile` >/dev/null 2>&1; then ps_alive=1; fi
if [ "$1" = "check_alive" -a $ping_alive = 1 ] ||
[ "$1" = "check_dead" -a $ping_alive = 0 -a $ps_alive = 0 ]; then
return 0 # EXIT_SUCCESS
else
if [ "$2" = "warn" ]; then
echo -e "$ps_alive processes alive and '$MYADMIN ping' resulted in\n$ping_output\n" | $ERR_LOGGER -p daemon.debug
fi
return 1 # EXIT_FAILURE
fi
}
case "${1:-''}" in
'start')
sanity_checks;
# Start daemon
log_daemon_msg "Starting MySQL database server" "mysqld"
if mysqld_status check_alive nowarn; then
log_progress_msg "already running"
log_end_msg 0
else
# Could be removed during boot
test -e /var/run/mysqld || install -m 755 -o mysql -g root -d /var/run/mysqld
# Start MySQL!
/usr/bin/mysqld_safe > /dev/null 2>&1 &
# 6s was reported in #352070 to be too few when using ndbcluster
for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14; do
sleep 5
if mysqld_status check_alive nowarn ; then break; fi
log_progress_msg "."
done
if mysqld_status check_alive warn; then
log_end_msg 0
# Now start mysqlcheck or whatever the admin wants.
output=$(/etc/mysql/debian-start)
[ -n "$output" ] && log_action_msg "$output"
else
log_end_msg 1
log_failure_msg "Please take a look at the syslog"
fi
fi
;;
'stop')
# * As a passwordless mysqladmin (e.g. via ~/.my.cnf) must be possible
# at least for cron, we can rely on it here, too. (although we have
# to specify it explicit as e.g. sudo environments points to the normal
# users home and not /root)
log_daemon_msg "Stopping MySQL database server" "mysqld"
if ! mysqld_status check_dead nowarn; then
set +e
shutdown_out=`$MYADMIN shutdown 2>&1`; r=$?
set -e
if [ "$r" -ne 0 ]; then
log_end_msg 1
[ "$VERBOSE" != "no" ] && log_failure_msg "Error: $shutdown_out"
log_daemon_msg "Killing MySQL database server by signal" "mysqld"
killall -15 mysqld
server_down=
for i in 1 2 3 4 5 6 7 8 9 10; do
sleep 5
if mysqld_status check_dead nowarn; then server_down=1; break; fi
done
if test -z "$server_down"; then killall -9 mysqld; fi
fi
fi
if ! mysqld_status check_dead warn; then
log_end_msg 1
log_failure_msg "Please stop MySQL manually and read /usr/share/doc/mysql-server-5.5/README.Debian.gz!"
exit -1
else
log_end_msg 0
fi
;;
'restart')
set +e; $SELF stop; set -e
$SELF start
;;
'reload'|'force-reload')
log_daemon_msg "Reloading MySQL database server" "mysqld"
$MYADMIN reload
log_end_msg 0
;;
'status')
if mysqld_status check_alive nowarn; then
log_action_msg "$($MYADMIN version)"
else
log_action_msg "MySQL is stopped."
exit 3
fi
;;
*)
echo "Usage: $SELF start|stop|restart|reload|force-reload|status"
exit 1
;;
esac
/etc/logrotate.d/mysql-server changed:
/var/log/mysql/mysql.err /var/log/mysql.log /var/log/mysql/mysql.log /var/log/mysql/mysql-slow.log {
daily
rotate 7
missingok
create 640 mysql adm
compress
sharedscripts
postrotate
test -x /usr/bin/mysqladmin || exit 0
# If this fails, check debian.conf!
MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf"
if [ -z "`$MYADMIN ping 2>/dev/null`" ]; then
# Really no mysqld or rather a missing debian-sys-maint user?
# If this occurs and is not a error please report a bug.
#if ps cax | grep -q mysqld; then
if killall -q -s0 -umysql mysqld; then
exit 1
fi
else
$MYADMIN flush-logs
fi
endscript
}
-- debconf information:
mysql-server/error_setting_password:
mysql-server-5.5/postrm_remove_databases: false
mysql-server-5.5/start_on_boot: true
mysql-server-5.5/nis_warning:
mysql-server-5.5/really_downgrade: false
mysql-server/password_mismatch:
mysql-server/no_upgrade_when_using_ndb:
More information about the pkg-mysql-maint
mailing list