pmp-check-mysql-deleted-files

#!/bin/sh

# ########################################################################
# This program is part of Percona Monitoring Plugins
# License: GPL License (see COPYING)
# Authors:
#  Baron Schwartz
# ########################################################################

# ########################################################################
# Redirect STDERR to STDOUT; Nagios doesn't handle STDERR.
# ########################################################################
exec 2>&1

# ########################################################################
# Set up constants, etc.
# ########################################################################
STATE_OK=0
STATE_WARNING=1
STATE_CRITICAL=2
STATE_UNKNOWN=3
STATE_DEPENDENT=4
EXITSTATUS=$STATE_UNKNOWN

# ########################################################################
# Run the program.
# ########################################################################
main() {
   # Get options
   for o; do
      case "${o}" in
         -c)              shift; OPT_CRIT="${1}"; shift; ;;
         --defaults-file) shift; OPT_DEFT="${1}"; shift; ;;
         -H)              shift; OPT_HOST="${1}"; shift; ;;
         -l)              shift; OPT_USER="${1}"; shift; ;;
         -p)              shift; OPT_PASS="${1}"; shift; ;;
         -P)              shift; OPT_PORT="${1}"; shift; ;;
         -S)              shift; OPT_SOCK="${1}"; shift; ;;
         -t)              shift; OPT_TMPDIR="${1}"; shift; ;;
         -w)              shift; OPT_WARN="${1}"; shift; ;;
         --version)       grep -A2 '^=head1 VERSION' "$0" | tail -n1; exit 0 ;;
         --help)          perl -00 -ne 'm/^  Usage:/ && print' "$0"; exit 0 ;;
         -*)              echo "Unknown option ${o}.  Try --help."; exit 1; ;;
      esac
   done
   if [ -e '/etc/nagios/mysql.cnf' ]; then
      OPT_DEFT="${OPT_DEFT:-/etc/nagios/mysql.cnf}"
   fi

   # If any connection option was given, then try to log in to find the server's
   # tmpdir.
   if [ "${OPT_HOST}${OPT_USER}${OPT_PASS}${OPT_PORT}${OPT_SOCK}" ]; then
      OPT_TMPDIR=$(mysql_exec "SELECT @@tmpdir") || exit $?
   elif [ -z "${OPT_TMPDIR}" ]; then
      OPT_TMPDIR="${TMPDIR:-/tmp/}"
   fi

   # TODO: We could auto-check every running instance, not just one.
   local NOTE="OK no deleted files"
   local PROC_ID=$(_pidof mysqld | head -n1)
   if [ "${PROC_ID}" ]; then
      local TEMP=$(mktemp "/tmp/${0##*/}.XXXX") || exit $?
      trap 'rm -rf "${TEMP}" >/dev/null 2>&1' EXIT
      if _lsof "${PROC_ID}" > "${TEMP}" ; then
         # If lsof exists, but you run it as non-root, you'll get a file with a
         # bunch of this stuff:
         # mysqld 15287 ... /proc/15287/cwd (readlink: Permission denied)
         # We have to detect this and return UNK.
         if grep -v -e denied -e COMMAND "${TEMP}" >/dev/null 2>&1; then
            local FILES=$(check_deleted_files "${TEMP}" "${OPT_TMPDIR}")
            if [ "${FILES}" -a -z "${OPT_WARN}" ]; then
               NOTE="CRIT open but deleted files: ${FILES}"
               EXITSTATUS=$STATE_CRITICAL
            elif [ "${FILES}" ]; then
               NOTE="WARN open but deleted files: ${FILES}"
               EXITSTATUS=$STATE_WARNING
            fi
         else
            NOTE="UNK You must execute lsof with root privileges"
            EXITSTATUS=$STATE_UNKNOWN
         fi
      else
         NOTE="UNK could not list MySQL's open files"
         EXITSTATUS=$STATE_UNKNOWN
      fi
   else
      NOTE="UNK could not find a mysqld process"
      EXITSTATUS=$STATE_UNKNOWN
   fi
   echo $NOTE
   exit $EXITSTATUS
}

# ########################################################################
# A wrapper around pidof, which might not exist. The first argument is the
# command name to match.
# ########################################################################
_pidof() {
   if ! pidof "${1}" 2>/dev/null; then
      ps -eo pid,ucomm | awk -v comm="${1}" '$2 == comm { print $1 }'
   fi
}

# ########################################################################
# Execute a MySQL command.
# ########################################################################
mysql_exec() {
   mysql ${OPT_DEFT:+--defaults-file="${OPT_DEFT}"} ${OPT_HOST:+-h"${OPT_HOST}"} ${OPT_USER:+-u"${OPT_USER}"} \
      ${OPT_PASS:+-p"${OPT_PASS}"} ${OPT_SOCK:+-S"${OPT_SOCK}"} ${OPT_PORT:+-P"${OPT_PORT}"} \
      -ss -e "$1"
}

# ########################################################################
# A wrapper around lsof, which might not exist. The first argument is the
# process ID to match.  This is Linux-centric; the fallback of listing
# /proc/pid/fd will probably only work on Linux.
# ########################################################################
_lsof() {
   PATH="$PATH:/usr/sbin:/sbin"
   if ! lsof -p $1 2>/dev/null; then
      /bin/ls -l /proc/$1/fd 2>/dev/null
   fi
}

# ########################################################################
# Generate a list of file handles that MySQL has open, but which are deleted,
# and are not temp files such as /tmp/ib* files (InnoDB) or /tmp/ML* files
# (binary logging).  The first argument is a file containing the output of lsof
# or ls -l for the open files.  The second argument is the server's tmpdir.
# ########################################################################
check_deleted_files() {
   awk -v tmpdir="${2}" '
   /\(deleted\)/ { if ( index($0, tmpdir) == 0 ) {
      if ( $NF ~ /deleted/ ) {
         lf=NF-1;
      }
      else {
         lf=NF;
      }
      print $lf;
   }}' "${1}"
}

# ########################################################################
# Execute the program if it was not included from another file.
# This makes it possible to include without executing, and thus test.
# ########################################################################
if    [ "${0##*/}" = "pmp-check-mysql-deleted-files" ] \
   || [ "${0##*/}" = "bash" -a "$_" = "$0" ]; then
   main "$@"
fi

# ############################################################################
# Documentation
# ############################################################################
: <<'DOCUMENTATION'
=pod

=head1 NAME

pmp-check-mysql-deleted-files - Alert when MySQL's files are deleted.

=head1 SYNOPSIS

  Usage: pmp-check-mysql-deleted-files [OPTIONS]
  Options:
    -c CRIT         Critical threshold; ignored.
    --defaults-file FILE Only read mysql options from the given file.
                    Defaults to /etc/nagios/mysql.cnf if it exists.
    -H HOST         MySQL hostname.
    -l USER         MySQL username.
    -p PASS         MySQL password.
    -P PORT         MySQL port.
    -S SOCKET       MySQL socket file.
    -w WARN         Warning threshold; changes the alert to WARN instead of CRIT.
    --help          Print help and exit.
    --version       Print version and exit.
  Options must be given as --option value, not --option=value or -Ovalue.
  Use perldoc to read embedded documentation with more details.

=head1 DESCRIPTION

This Nagios plugin looks at the files that the mysqld process has open, and
warns if any of them are deleted that shouldn't be.  This typically happens when
there is a poorly written logrotate script or when a human makes a mistake at
the command line.  This can cause several bad effects. If a table has been
deleted, of course, it is a serious matter.  Such a file can also potentially
fill up the disk invisibly.  If the file is the server's log, it might mean that
logging is effectively broken and any problems the server experiences could be
undiagnosable.

The plugin accepts the -w and -c options for compatibility with standard Nagios
plugin conventions, but they are not based on a threshold. Instead, the plugin
raises a critical alert by default, and if the -w option is given, it raises a
warning instead, regardless of the option's value.

This plugin doesn't alert about deleted temporary files, which are not a
problem.  By default, this plugin assumes that the server's temporary directory
is either the TMPDIR environment variable, or if that is not set, then /tmp/.
If you specify MySQL authentication options, the value will log into the
specified MySQL instance and look at the C<tmpdir> variable to find the
temporary directory.

This plugin looks at the first running instance of MySQL, as found in the
system process table, so it will not work on systems that have multiple
instances running. It probably works best on Linux, though it might work on
other operating systems. It relies on either lsof or the ability to list the
files in the process's /proc/pid/fd directory.

=head1 PRIVILEGES

This plugin executes the following commands against MySQL:

=over

=item *

C<SELECT> the system variable C<@@tmpdir>.

=back

This plugin executes the following UNIX commands that may need special privileges:

=over

=item *

ps

=item *

lsof

=item *

ls /proc/$pid/fd

=back

=head1 COPYRIGHT, LICENSE, AND WARRANTY

This program is copyright 2012 Baron Schwartz, 2012 Percona Inc.
Feedback and improvements are welcome.

THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation, version 2.  You should have received a copy of the GNU General
Public License along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.

=head1 VERSION

Percona Monitoring Plugins pmp-check-mysql-deleted-files 0.9.0

=cut

DOCUMENTATION